From fa57681b3403e801046f91e66c61969aae8240ed Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 18:29:00 -0400 Subject: [PATCH 01/12] feat(asap-common): RQE expression tree type and validation Add `asap_types::rqe_expr`: a serde type for RQE expression trees (rqe leaves, agg, binary, scalar nodes), a `validate` check run at load, and `leaves` / `output_labels` helpers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../dependencies/rs/asap_types/src/lib.rs | 2 + .../rs/asap_types/src/rqe_expr.rs | 573 ++++++++++++++++++ 2 files changed, 575 insertions(+) create mode 100644 asap-common/dependencies/rs/asap_types/src/rqe_expr.rs diff --git a/asap-common/dependencies/rs/asap_types/src/lib.rs b/asap-common/dependencies/rs/asap_types/src/lib.rs index 167c753a..cdba5a38 100644 --- a/asap-common/dependencies/rs/asap_types/src/lib.rs +++ b/asap-common/dependencies/rs/asap_types/src/lib.rs @@ -7,6 +7,7 @@ pub mod inference_config; pub mod promql_schema; pub mod query_config; pub mod query_requirements; +pub mod rqe_expr; pub mod streaming_config; pub mod traits; pub mod utils; @@ -19,4 +20,5 @@ pub use inference_config::*; pub use promql_schema::*; pub use query_config::*; pub use query_requirements::*; +pub use rqe_expr::*; pub use streaming_config::*; diff --git a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs new file mode 100644 index 00000000..05e6000d --- /dev/null +++ b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs @@ -0,0 +1,573 @@ +//! RQE expression tree: self-contained RQE leaves combined by aggregations and +//! binary ops. Each JSON node is an object identified by its key: +//! +//! ```json +//! {"agg": "avg", "by": ["job"], "of": +//! {"op": "/", "lhs": {"rqe": {...}}, "rhs": {"scalar": 2.0}}} +//! ``` + +use std::collections::BTreeSet; + +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RqeCapability { + Sum, + Quantile, + Cardinality, + TopkByValue, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RqeLeaf { + pub id: String, + /// Metric name (PromQL) or table name (SQL). + pub metric: String, + pub capability: RqeCapability, + pub lookback_ms: u64, + pub grouping: Vec, + /// Quantile rank; set iff `capability` is `quantile`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub phi: Option, + /// Top-k size; set iff `capability` is `topk_by_value`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub k: Option, + /// SQL value column. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum RqeAggOp { + Avg, + Min, + Max, + Sum, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub enum RqeBinaryOp { + #[serde(rename = "+")] + Add, + #[serde(rename = "-")] + Sub, + #[serde(rename = "*")] + Mul, + #[serde(rename = "/")] + Div, +} + +// Serialized untagged (each variant's field names are the JSON keys); +// deserialized through `RawNode` so shape errors name the offending key. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged, try_from = "RawNode")] +pub enum RqeExpr { + Rqe { + rqe: RqeLeaf, + }, + Agg { + agg: RqeAggOp, + by: Vec, + of: Box, + }, + Binary { + op: RqeBinaryOp, + lhs: Box, + rhs: Box, + }, + Scalar { + scalar: f64, + }, +} + +/// Union of all node keys; `TryFrom` picks the kind and rejects mixed keys. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct RawNode { + rqe: Option, + agg: Option, + by: Option>, + of: Option>, + op: Option, + lhs: Option>, + rhs: Option>, + scalar: Option, +} + +impl TryFrom for RqeExpr { + type Error = String; + + fn try_from(raw: RawNode) -> Result { + match raw { + RawNode { + rqe: Some(rqe), + agg: None, + by: None, + of: None, + op: None, + lhs: None, + rhs: None, + scalar: None, + } => Ok(RqeExpr::Rqe { rqe }), + RawNode { + rqe: None, + agg: Some(agg), + by: Some(by), + of: Some(of), + op: None, + lhs: None, + rhs: None, + scalar: None, + } => Ok(RqeExpr::Agg { agg, by, of }), + RawNode { + rqe: None, + agg: None, + by: None, + of: None, + op: Some(op), + lhs: Some(lhs), + rhs: Some(rhs), + scalar: None, + } => Ok(RqeExpr::Binary { op, lhs, rhs }), + RawNode { + rqe: None, + agg: None, + by: None, + of: None, + op: None, + lhs: None, + rhs: None, + scalar: Some(scalar), + } => Ok(RqeExpr::Scalar { scalar }), + _ => Err("RQE expression node must be exactly one of \ + {\"rqe\"}, {\"agg\", \"by\", \"of\"}, {\"op\", \"lhs\", \"rhs\"}, {\"scalar\"}" + .to_string()), + } + } +} + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum RqeExprError { + #[error("RQE {id}: quantile requires phi in [0, 1], got {phi:?}")] + InvalidPhi { id: String, phi: Option }, + #[error("RQE {id}: phi is only allowed for quantile")] + UnexpectedPhi { id: String }, + #[error("RQE {id}: topk_by_value requires k > 0, got {k:?}")] + InvalidK { id: String, k: Option }, + #[error("RQE {id}: k is only allowed for topk_by_value")] + UnexpectedK { id: String }, + #[error("RQE {id}: topk_by_value is only allowed at the root")] + TopkNotAtRoot { id: String }, + #[error("agg by {by:?} is not a subset of child labels {child:?}")] + AggByNotSubset { by: Vec, child: Vec }, + #[error("binary op needs at least one non-scalar side")] + BinaryBothScalar, + #[error("binary op sides have different labels: {lhs:?} vs {rhs:?}")] + BinaryLabelMismatch { lhs: Vec, rhs: Vec }, +} + +impl RqeExpr { + /// Checks the whole tree; call once after loading. + pub fn validate(&self) -> Result<(), RqeExprError> { + self.validate_node(true) + } + + fn validate_node(&self, is_root: bool) -> Result<(), RqeExprError> { + match self { + RqeExpr::Rqe { rqe } => rqe.validate(is_root), + RqeExpr::Agg { by, of, .. } => { + of.validate_node(false)?; + let child = label_set(of.output_labels()); + if !label_set(by).is_subset(&child) { + return Err(RqeExprError::AggByNotSubset { + by: by.clone(), + child: of.output_labels().to_vec(), + }); + } + Ok(()) + } + RqeExpr::Binary { lhs, rhs, .. } => { + lhs.validate_node(false)?; + rhs.validate_node(false)?; + match (lhs.is_scalar(), rhs.is_scalar()) { + (true, true) => Err(RqeExprError::BinaryBothScalar), + (false, false) + if label_set(lhs.output_labels()) != label_set(rhs.output_labels()) => + { + Err(RqeExprError::BinaryLabelMismatch { + lhs: lhs.output_labels().to_vec(), + rhs: rhs.output_labels().to_vec(), + }) + } + _ => Ok(()), + } + } + RqeExpr::Scalar { .. } => Ok(()), + } + } + + /// Labels of this node's result: leaf grouping, agg `by`, the non-scalar + /// side of a binary op (lhs if both), none for a scalar. + pub fn output_labels(&self) -> &[String] { + match self { + RqeExpr::Rqe { rqe } => &rqe.grouping, + RqeExpr::Agg { by, .. } => by, + RqeExpr::Binary { lhs, rhs, .. } => { + if lhs.is_scalar() { + rhs.output_labels() + } else { + lhs.output_labels() + } + } + RqeExpr::Scalar { .. } => &[], + } + } + + /// All RQE leaves, left to right. + pub fn leaves(&self) -> Vec<&RqeLeaf> { + let mut out = Vec::new(); + self.collect_leaves(&mut out); + out + } + + fn collect_leaves<'a>(&'a self, out: &mut Vec<&'a RqeLeaf>) { + match self { + RqeExpr::Rqe { rqe } => out.push(rqe), + RqeExpr::Agg { of, .. } => of.collect_leaves(out), + RqeExpr::Binary { lhs, rhs, .. } => { + lhs.collect_leaves(out); + rhs.collect_leaves(out); + } + RqeExpr::Scalar { .. } => {} + } + } + + fn is_scalar(&self) -> bool { + matches!(self, RqeExpr::Scalar { .. }) + } +} + +impl RqeLeaf { + fn validate(&self, is_root: bool) -> Result<(), RqeExprError> { + let id = || self.id.clone(); + match (self.capability, self.phi) { + (RqeCapability::Quantile, Some(phi)) if (0.0..=1.0).contains(&phi) => {} + (RqeCapability::Quantile, phi) => { + return Err(RqeExprError::InvalidPhi { id: id(), phi }) + } + (_, Some(_)) => return Err(RqeExprError::UnexpectedPhi { id: id() }), + (_, None) => {} + } + match (self.capability, self.k) { + (RqeCapability::TopkByValue, Some(k)) if k > 0 => {} + (RqeCapability::TopkByValue, k) => return Err(RqeExprError::InvalidK { id: id(), k }), + (_, Some(_)) => return Err(RqeExprError::UnexpectedK { id: id() }), + (_, None) => {} + } + if self.capability == RqeCapability::TopkByValue && !is_root { + return Err(RqeExprError::TopkNotAtRoot { id: id() }); + } + Ok(()) + } +} + +fn label_set(labels: &[String]) -> BTreeSet<&str> { + labels.iter().map(String::as_str).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{json, Value}; + + fn leaf_json(id: &str, capability: &str, grouping: &[&str]) -> Value { + json!({"rqe": { + "id": id, "metric": "http_requests", "capability": capability, + "lookback_ms": 60000, "grouping": grouping, + }}) + } + + fn sum_leaf(id: &str, grouping: &[&str]) -> Value { + leaf_json(id, "sum", grouping) + } + + fn parse(v: Value) -> RqeExpr { + serde_json::from_value(v).expect("valid JSON shape") + } + + fn parse_err(v: Value) -> String { + serde_json::from_value::(v) + .expect_err("should not deserialize") + .to_string() + } + + fn assert_round_trip(v: Value) { + let expr = parse(v.clone()); + assert_eq!(serde_json::to_value(&expr).unwrap(), v); + } + + // --- JSON round trip --- + + #[test] + fn round_trip_leaf_minimal() { + assert_round_trip(sum_leaf("q0", &["job"])); + } + + #[test] + fn round_trip_leaf_all_optional_fields() { + assert_round_trip(json!({"rqe": { + "id": "q1", "metric": "events", "capability": "quantile", + "lookback_ms": 300000, "grouping": [], "phi": 0.99, "value": "latency", + }})); + assert_round_trip(json!({"rqe": { + "id": "q2", "metric": "events", "capability": "topk_by_value", + "lookback_ms": 1000, "grouping": ["user_id"], "k": 10, + }})); + assert_round_trip(leaf_json("q3", "cardinality", &["job"])); + } + + #[test] + fn round_trip_agg() { + for op in ["avg", "min", "max", "sum"] { + assert_round_trip( + json!({"agg": op, "by": ["job"], "of": sum_leaf("q", &["job", "instance"])}), + ); + } + } + + #[test] + fn round_trip_binary() { + for op in ["+", "-", "*", "/"] { + assert_round_trip( + json!({"op": op, "lhs": sum_leaf("a", &["job"]), "rhs": sum_leaf("b", &["job"])}), + ); + } + } + + #[test] + fn round_trip_scalar() { + assert_round_trip(json!({"scalar": 2.5})); + } + + #[test] + fn round_trip_nested() { + assert_round_trip(json!({"agg": "avg", "by": ["job"], "of": { + "op": "/", "lhs": sum_leaf("a", &["job", "instance"]), "rhs": sum_leaf("b", &["instance", "job"]), + }})); + } + + // --- deserialization errors --- + + #[test] + fn rejects_unknown_capability() { + for cap in ["rate", "topk_by_count"] { + let err = parse_err(leaf_json("q", cap, &[])); + assert!(err.contains(&format!("unknown variant `{cap}`")), "{err}"); + } + } + + #[test] + fn rejects_unknown_ops() { + let err = parse_err(json!({"agg": "count", "by": [], "of": sum_leaf("q", &[])})); + assert!(err.contains("unknown variant `count`"), "{err}"); + let err = parse_err(json!({"op": "%", "lhs": sum_leaf("a", &[]), "rhs": {"scalar": 1.0}})); + assert!(err.contains("unknown variant `%`"), "{err}"); + } + + #[test] + fn rejects_mixed_or_incomplete_node() { + let mixed = json!({"scalar": 1.0, "op": "+"}); + let incomplete = json!({"agg": "sum", "of": sum_leaf("q", &[])}); + for v in [mixed, incomplete, json!({})] { + assert!(parse_err(v).contains("must be exactly one of")); + } + } + + #[test] + fn rejects_unknown_keys() { + assert!(parse_err(json!({"scalar": 1.0, "extra": 1})).contains("unknown field `extra`")); + let mut leaf = sum_leaf("q", &[]); + leaf["rqe"]["filter"] = json!("x"); + assert!(parse_err(leaf).contains("unknown field `filter`")); + } + + // --- validation: phi / k --- + + #[test] + fn quantile_phi_rules() { + let with_phi = |phi: Value| { + let mut v = leaf_json("q", "quantile", &[]); + v["rqe"]["phi"] = phi; + parse(v).validate() + }; + assert_eq!(with_phi(json!(0.5)), Ok(())); + assert_eq!(with_phi(json!(0.0)), Ok(())); + assert_eq!(with_phi(json!(1.0)), Ok(())); + assert!(matches!( + with_phi(json!(1.5)), + Err(RqeExprError::InvalidPhi { .. }) + )); + assert!(matches!( + with_phi(json!(-0.1)), + Err(RqeExprError::InvalidPhi { .. }) + )); + assert!(matches!( + parse(leaf_json("q", "quantile", &[])).validate(), + Err(RqeExprError::InvalidPhi { phi: None, .. }) + )); + } + + #[test] + fn rejects_phi_on_non_quantile() { + let mut v = sum_leaf("q", &[]); + v["rqe"]["phi"] = json!(0.5); + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::UnexpectedPhi { .. }) + )); + } + + #[test] + fn topk_k_rules() { + let with_k = |k: Value| { + let mut v = leaf_json("q", "topk_by_value", &[]); + v["rqe"]["k"] = k; + parse(v).validate() + }; + assert_eq!(with_k(json!(5)), Ok(())); + assert!(matches!( + with_k(json!(0)), + Err(RqeExprError::InvalidK { k: Some(0), .. }) + )); + assert!(matches!( + parse(leaf_json("q", "topk_by_value", &[])).validate(), + Err(RqeExprError::InvalidK { k: None, .. }) + )); + } + + #[test] + fn rejects_k_on_non_topk() { + let mut v = leaf_json("q", "cardinality", &[]); + v["rqe"]["k"] = json!(3); + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::UnexpectedK { .. }) + )); + } + + // --- validation: top-k placement --- + + #[test] + fn rejects_topk_below_root() { + let mut topk = leaf_json("t", "topk_by_value", &["job"]); + topk["rqe"]["k"] = json!(3); + let agg = json!({"agg": "sum", "by": ["job"], "of": topk.clone()}); + let binary = json!({"op": "*", "lhs": topk, "rhs": {"scalar": 2.0}}); + for v in [agg, binary] { + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::TopkNotAtRoot { .. }) + )); + } + } + + // --- validation: agg --- + + #[test] + fn agg_by_subset_rules() { + let ok = json!({"agg": "max", "by": ["job"], "of": sum_leaf("q", &["instance", "job"])}); + assert_eq!(parse(ok).validate(), Ok(())); + let empty_by = json!({"agg": "sum", "by": [], "of": sum_leaf("q", &["job"])}); + assert_eq!(parse(empty_by).validate(), Ok(())); + let bad = json!({"agg": "max", "by": ["pod"], "of": sum_leaf("q", &["job"])}); + assert!(matches!( + parse(bad).validate(), + Err(RqeExprError::AggByNotSubset { .. }) + )); + } + + #[test] + fn agg_validates_child() { + let v = json!({"agg": "sum", "by": [], "of": leaf_json("q", "quantile", &[])}); + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::InvalidPhi { .. }) + )); + } + + // --- validation: binary --- + + #[test] + fn binary_label_set_rules() { + let same_set = json!({"op": "-", "lhs": sum_leaf("a", &["job", "instance"]), "rhs": sum_leaf("b", &["instance", "job"])}); + assert_eq!(parse(same_set).validate(), Ok(())); + let differ = + json!({"op": "-", "lhs": sum_leaf("a", &["job"]), "rhs": sum_leaf("b", &["instance"])}); + assert!(matches!( + parse(differ).validate(), + Err(RqeExprError::BinaryLabelMismatch { .. }) + )); + } + + #[test] + fn binary_scalar_operand_rules() { + let scalar_rhs = + json!({"op": "/", "lhs": sum_leaf("a", &["job"]), "rhs": {"scalar": 1000.0}}); + assert_eq!(parse(scalar_rhs).validate(), Ok(())); + let scalar_lhs = json!({"op": "-", "lhs": {"scalar": 1.0}, "rhs": sum_leaf("a", &["job"])}); + assert_eq!(parse(scalar_lhs).validate(), Ok(())); + let both = json!({"op": "+", "lhs": {"scalar": 1.0}, "rhs": {"scalar": 2.0}}); + assert_eq!(parse(both).validate(), Err(RqeExprError::BinaryBothScalar)); + } + + #[test] + fn binary_validates_children() { + let bad_child = json!({"agg": "sum", "by": ["pod"], "of": sum_leaf("q", &["job"])}); + let v = json!({"op": "+", "lhs": sum_leaf("a", &["pod"]), "rhs": bad_child}); + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::AggByNotSubset { .. }) + )); + } + + // --- nested example and helpers --- + + fn nested_example() -> RqeExpr { + parse(json!({"agg": "avg", "by": ["job"], "of": { + "op": "/", "lhs": sum_leaf("a", &["job", "instance"]), "rhs": sum_leaf("b", &["instance", "job"]), + }})) + } + + #[test] + fn nested_example_validates() { + assert_eq!(nested_example().validate(), Ok(())); + } + + #[test] + fn leaves_in_order() { + let ids: Vec<_> = nested_example() + .leaves() + .iter() + .map(|l| l.id.clone()) + .collect(); + assert_eq!(ids, ["a", "b"]); + assert!(parse(json!({"scalar": 1.0})).leaves().is_empty()); + } + + #[test] + fn output_labels_per_kind() { + let expr = nested_example(); + assert_eq!(expr.output_labels(), ["job"]); + let RqeExpr::Agg { of, .. } = &expr else { + unreachable!() + }; + assert_eq!(of.output_labels(), ["job", "instance"]); + let scalar_lhs = + parse(json!({"op": "*", "lhs": {"scalar": 2.0}, "rhs": sum_leaf("a", &["pod"])})); + assert_eq!(scalar_lhs.output_labels(), ["pod"]); + assert!(parse(json!({"scalar": 2.0})).output_labels().is_empty()); + } +} From d8681d5efe3d56d99664ae2fc6cbe6b1ce7ec66d Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 18:49:55 -0400 Subject: [PATCH 02/12] feat(planner): --planner saved reads RQE trees and writes rqe_configs An e2e query's optional `expr` (an RQE expression tree, validated at load) now names the plan RQEs it reads. Each tree leaf becomes one more optimizer item and Raqe, served by its plan pick like a text leaf, and gets an entry in a new `rqe_configs` section of the inference config: RQE id -> the aggregations it reads and, for an exact roll-up, the query-time stage that regroups to the RQE's grouping. `query_configs` are unchanged; a table without trees converts as before and writes no `rqe_configs`. A query may have only a tree: its language is that of the metrics its leaves read (records -> SQL). The engine's `InferenceConfig` loads `rqe_configs` (empty when absent); nothing reads it yet. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../rs/asap_types/src/inference_config.rs | 216 +++++- asap-planner-rs/src/generator.rs | 116 ++- asap-planner-rs/src/optimizer/milp.rs | 13 + asap-planner-rs/src/optimizer/milp_output.rs | 214 ++++-- asap-planner-rs/src/optimizer/saved_plan.rs | 723 +++++++++++++++++- .../tests/saved_plan_e2e_tables.rs | 153 ++++ .../synthetic-templatesall-shared1.json | 1 + ...nthetic-templatesall-shared1.plans.json.gz | Bin 0 -> 39806 bytes .../src/engines/simple_engine/mod.rs | 5 + .../src/engines/simple_engine/promql.rs | 4 + .../src/engines/simple_engine/sql.rs | 6 + .../src/tests/capability_matching_tests.rs | 2 + .../exact_window_grid_adversarial_tests.rs | 1 + .../src/tests/native_range_query_tests.rs | 4 + .../src/tests/prometheus_forwarding_tests.rs | 1 + .../query_time_aggregation_public_tests.rs | 5 + .../src/tests/range_query_arithmetic_tests.rs | 3 + asap-query-engine/src/tests/rollup_tests.rs | 2 + .../src/tests/sql_pattern_matching_tests.rs | 1 + ...stage_e_instant_range_equivalence_tests.rs | 2 + .../src/tests/structural_matching_tests.rs | 1 + .../tests/test_utilities/config_builders.rs | 3 + .../tests/test_utilities/engine_factories.rs | 6 + .../window_semantics_consistency_tests.rs | 5 + .../tests/e2e_precompute_equivalence.rs | 3 + 25 files changed, 1339 insertions(+), 151 deletions(-) create mode 100644 asap-planner-rs/tests/saved_plan_e2e_tables.rs create mode 100644 asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json create mode 100644 asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz diff --git a/asap-common/dependencies/rs/asap_types/src/inference_config.rs b/asap-common/dependencies/rs/asap_types/src/inference_config.rs index cb674149..3b96e037 100644 --- a/asap-common/dependencies/rs/asap_types/src/inference_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/inference_config.rs @@ -1,6 +1,6 @@ use anyhow::Result; use serde_yaml::Value; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::BufReader; @@ -21,11 +21,21 @@ pub enum SchemaConfig { ElasticSQL(SQLSchema), } +/// How the engine answers one RQE leaf directly: the aggregations it reads +/// and the query-time stages that regroup them to the RQE's grouping. +#[derive(Debug, Clone)] +pub struct RqeConfig { + pub aggregations: Vec, + pub query_time_aggregations: Vec, +} + #[derive(Debug, Clone)] pub struct InferenceConfig { pub schema: SchemaConfig, pub query_configs: Vec, pub cleanup_policy: CleanupPolicy, + /// RQE id -> how to answer that RQE leaf; empty unless planned from RQE trees. + pub rqe_configs: HashMap, } impl InferenceConfig { @@ -42,6 +52,7 @@ impl InferenceConfig { schema, query_configs: Vec::new(), cleanup_policy, + rqe_configs: HashMap::new(), } } @@ -75,11 +86,13 @@ impl InferenceConfig { let cleanup_policy = Self::parse_cleanup_policy(data)?; let query_configs = Self::parse_query_configs(data, cleanup_policy)?; + let rqe_configs = Self::parse_rqe_configs(data, cleanup_policy)?; Ok(Self { schema, query_configs, cleanup_policy, + rqe_configs, }) } @@ -263,43 +276,11 @@ impl InferenceConfig { .map_err(anyhow::Error::from) })?; - let aggregations = if let Some(aggregations_data) = - query_data.get("aggregations").and_then(|v| v.as_sequence()) - { - let mut agg_refs = Vec::new(); - for agg_data in aggregations_data { - let aggregation_id = agg_data - .get("aggregation_id") - .and_then(|v| v.as_u64()) - .ok_or_else(|| { - anyhow::anyhow!("Missing aggregation_id in aggregation") - })?; - - let agg_ref = match cleanup_policy { - CleanupPolicy::CircularBuffer => { - let num_aggregates_to_retain = agg_data - .get("num_aggregates_to_retain") - .and_then(|v| v.as_u64()); - AggregationReference::new(aggregation_id, num_aggregates_to_retain) - } - CleanupPolicy::ReadBased => { - let read_count_threshold = agg_data - .get("read_count_threshold") - .and_then(|v| v.as_u64()); - AggregationReference::with_read_count_threshold( - aggregation_id, - read_count_threshold, - ) - } - CleanupPolicy::NoCleanup => { - AggregationReference::new(aggregation_id, None) - } - }; - agg_refs.push(agg_ref); + let aggregations = match query_data.get("aggregations") { + Some(aggregations) => { + Self::parse_aggregation_refs(aggregations, cleanup_policy)? } - agg_refs - } else { - Vec::new() + None => Vec::new(), }; let config = @@ -316,11 +297,97 @@ impl InferenceConfig { }; Ok(query_configs) } + + /// A sequence of `{aggregation_id, num_aggregates_to_retain | read_count_threshold}`. + fn parse_aggregation_refs( + aggregations_data: &Value, + cleanup_policy: CleanupPolicy, + ) -> Result> { + let Some(aggregations_data) = aggregations_data.as_sequence() else { + return Ok(Vec::new()); + }; + let mut agg_refs = Vec::new(); + for agg_data in aggregations_data { + let aggregation_id = agg_data + .get("aggregation_id") + .and_then(|v| v.as_u64()) + .ok_or_else(|| anyhow::anyhow!("Missing aggregation_id in aggregation"))?; + + let agg_ref = match cleanup_policy { + CleanupPolicy::CircularBuffer => { + let num_aggregates_to_retain = agg_data + .get("num_aggregates_to_retain") + .and_then(|v| v.as_u64()); + AggregationReference::new(aggregation_id, num_aggregates_to_retain) + } + CleanupPolicy::ReadBased => { + let read_count_threshold = agg_data + .get("read_count_threshold") + .and_then(|v| v.as_u64()); + AggregationReference::with_read_count_threshold( + aggregation_id, + read_count_threshold, + ) + } + CleanupPolicy::NoCleanup => AggregationReference::new(aggregation_id, None), + }; + agg_refs.push(agg_ref); + } + Ok(agg_refs) + } + + /// The optional `rqe_configs` mapping: RQE id -> `{aggregations, + /// query_time_aggregations}`, both required. + fn parse_rqe_configs( + data: &Value, + cleanup_policy: CleanupPolicy, + ) -> Result> { + let Some(rqe_configs) = data.get("rqe_configs") else { + return Ok(HashMap::new()); + }; + let rqe_configs = rqe_configs + .as_mapping() + .ok_or_else(|| anyhow::anyhow!("rqe_configs must map RQE ids to configs"))?; + let mut configs = HashMap::new(); + for (id, config) in rqe_configs { + let id = id + .as_str() + .ok_or_else(|| anyhow::anyhow!("rqe_configs key {id:?} is not a string"))?; + let aggregations = config + .get("aggregations") + .ok_or_else(|| anyhow::anyhow!("RQE {id:?}: missing aggregations")) + .and_then(|value| Self::parse_aggregation_refs(value, cleanup_policy))?; + if aggregations.is_empty() { + anyhow::bail!("RQE {id:?}: aggregations is empty"); + } + let query_time_aggregations = config + .get("query_time_aggregations") + .ok_or_else(|| anyhow::anyhow!("RQE {id:?}: missing query_time_aggregations")) + .and_then(|value| { + serde_yaml::from_value::>(value.clone()) + .map_err(anyhow::Error::from) + })?; + for stage in &query_time_aggregations { + stage + .validate() + .map_err(|error| anyhow::anyhow!("RQE {id:?}: {error}"))?; + } + configs.insert( + id.to_string(), + RqeConfig { + aggregations, + query_time_aggregations, + }, + ); + } + Ok(configs) + } } #[cfg(test)] mod tests { use super::*; + use crate::query_config::QueryTimeAggregationOperator; #[test] fn rejects_a_query_without_a_planned_subquery() { @@ -390,4 +457,79 @@ queries: assert!(error.to_string().contains("quantile")); } + + #[test] + fn loads_rqe_configs_with_their_roll_up() { + let data: Value = serde_yaml::from_str( + r#" +cleanup_policy: + name: circular_buffer +metrics: {} +queries: [] +rqe_configs: + sum_by_job: + aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 5 + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: [job] + parameter: null +"#, + ) + .unwrap(); + + let config = InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).unwrap(); + + let rqe = &config.rqe_configs["sum_by_job"]; + let [aggregation] = rqe.aggregations.as_slice() else { + panic!("{:?}", rqe.aggregations); + }; + assert_eq!(aggregation.aggregation_id, 2); + assert_eq!(aggregation.num_aggregates_to_retain, Some(5)); + let [stage] = rqe.query_time_aggregations.as_slice() else { + panic!("{:?}", rqe.query_time_aggregations); + }; + assert!(matches!(stage.operator, QueryTimeAggregationOperator::Sum)); + assert_eq!(stage.grouping.labels, ["job"]); + } + + #[test] + fn a_config_without_rqe_configs_has_none() { + let data: Value = + serde_yaml::from_str("cleanup_policy: {name: no_cleanup}\nmetrics: {}\n").unwrap(); + let config = InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).unwrap(); + assert!(config.rqe_configs.is_empty()); + } + + #[test] + fn rejects_an_rqe_config_missing_a_field_or_with_an_invalid_stage() { + for (rqe, reason) in [ + ("{query_time_aggregations: []}", "missing aggregations"), + ( + "{aggregations: [], query_time_aggregations: []}", + "aggregations is empty", + ), + ( + "{aggregations: [{aggregation_id: 1}]}", + "missing query_time_aggregations", + ), + ( + "{aggregations: [{aggregation_id: 1}], query_time_aggregations: \ + [{operator: sum, grouping: {mode: by, labels: []}, parameter: null}]}", + "by grouping must name at least one label", + ), + ] { + let yaml = format!( + "cleanup_policy: {{name: no_cleanup}}\nmetrics: {{}}\nrqe_configs: {{r: {rqe}}}\n" + ); + let data: Value = serde_yaml::from_str(&yaml).unwrap(); + let error = + InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).expect_err("refused"); + assert!(error.to_string().contains(reason), "{error}"); + assert!(error.to_string().contains("\"r\""), "{error}"); + } + } } diff --git a/asap-planner-rs/src/generator.rs b/asap-planner-rs/src/generator.rs index ba045540..e0137e34 100644 --- a/asap-planner-rs/src/generator.rs +++ b/asap-planner-rs/src/generator.rs @@ -1,7 +1,7 @@ use indexmap::IndexMap; use serde_json::Value as JsonValue; use serde_yaml::Value as YamlValue; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use asap_types::enums::CleanupPolicy; use asap_types::query_config::QueryTimeAggregation; @@ -31,6 +31,7 @@ pub(crate) const KEY_QUERIES: &str = "queries"; pub(crate) const KEY_QUERY: &str = "query"; pub(crate) const KEY_QUERY_TIME_AGGREGATIONS: &str = "query_time_aggregations"; pub(crate) const KEY_READ_COUNT_THRESHOLD: &str = "read_count_threshold"; +pub(crate) const KEY_RQE_CONFIGS: &str = "rqe_configs"; pub(crate) const KEY_SLIDE_INTERVAL_MS: &str = "slideIntervalMs"; pub(crate) const KEY_SPATIAL_FILTER: &str = "spatialFilter"; pub(crate) const KEY_TABLE_NAME: &str = "table_name"; @@ -49,6 +50,14 @@ pub struct QueryPlanEntry { pub query_time_aggregations: Vec, } +/// How the engine answers one RQE leaf: the aggregations it reads, and the +/// query-time stages that regroup them to the RQE's grouping. +#[derive(Debug, Clone)] +pub struct RqePlanEntry { + pub aggregation_keys: Vec<(String, Option)>, + pub query_time_aggregations: Vec, +} + impl QueryPlanEntry { pub fn fully_planned(query: String, aggregation_keys: Vec<(String, Option)>) -> Self { Self { @@ -144,6 +153,50 @@ pub fn build_aggregation_entry(id: u32, cfg: &IntermediateAggConfig) -> YamlValu YamlValue::Mapping(map) } +/// The `aggregations` list of a query or RQE config: each aggregation's id +/// and, by cleanup policy, how many aggregates it retains or reads. +fn aggregation_refs_yaml( + cleanup_policy: CleanupPolicy, + aggregation_keys: &[(String, Option)], + id_map: &HashMap, +) -> YamlValue { + YamlValue::Sequence( + aggregation_keys + .iter() + .map(|(key, cleanup_param)| { + let agg_id = id_map[key]; + let mut agg_map = serde_yaml::Mapping::new(); + agg_map.insert( + YamlValue::String(KEY_AGGREGATION_ID.to_string()), + YamlValue::Number(agg_id.into()), + ); + if let Some(param) = cleanup_param { + match cleanup_policy { + CleanupPolicy::CircularBuffer => { + agg_map.insert( + YamlValue::String(KEY_NUM_AGG_TO_RETAIN.to_string()), + YamlValue::Number((*param).into()), + ); + } + CleanupPolicy::ReadBased => { + agg_map.insert( + YamlValue::String(KEY_READ_COUNT_THRESHOLD.to_string()), + YamlValue::Number((*param).into()), + ); + } + CleanupPolicy::NoCleanup => {} + } + } + YamlValue::Mapping(agg_map) + }) + .collect(), + ) +} + +fn query_time_aggregations_yaml(stages: &[QueryTimeAggregation]) -> YamlValue { + serde_yaml::to_value(stages).expect("query-time aggregation pipeline should serialize to YAML") +} + pub fn build_queries_yaml( cleanup_policy: CleanupPolicy, query_plan_map: &IndexMap, @@ -152,41 +205,10 @@ pub fn build_queries_yaml( query_plan_map .iter() .map(|(query_str, plan)| { - let aggregations: Vec = plan - .aggregation_keys - .iter() - .map(|(key, cleanup_param)| { - let agg_id = id_map[key]; - let mut agg_map = serde_yaml::Mapping::new(); - agg_map.insert( - YamlValue::String(KEY_AGGREGATION_ID.to_string()), - YamlValue::Number(agg_id.into()), - ); - if let Some(param) = cleanup_param { - match cleanup_policy { - CleanupPolicy::CircularBuffer => { - agg_map.insert( - YamlValue::String(KEY_NUM_AGG_TO_RETAIN.to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::ReadBased => { - agg_map.insert( - YamlValue::String(KEY_READ_COUNT_THRESHOLD.to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::NoCleanup => {} - } - } - YamlValue::Mapping(agg_map) - }) - .collect(); - let mut q_map = serde_yaml::Mapping::new(); q_map.insert( YamlValue::String(KEY_AGGREGATIONS.to_string()), - YamlValue::Sequence(aggregations), + aggregation_refs_yaml(cleanup_policy, &plan.aggregation_keys, id_map), ); q_map.insert( YamlValue::String(KEY_QUERY.to_string()), @@ -198,14 +220,38 @@ pub fn build_queries_yaml( ); q_map.insert( YamlValue::String(KEY_QUERY_TIME_AGGREGATIONS.to_string()), - serde_yaml::to_value(&plan.query_time_aggregations) - .expect("query-time aggregation pipeline should serialize to YAML"), + query_time_aggregations_yaml(&plan.query_time_aggregations), ); YamlValue::Mapping(q_map) }) .collect() } +/// The `rqe_configs` mapping: RQE id -> its aggregations and query-time stages. +pub fn build_rqe_configs_yaml( + cleanup_policy: CleanupPolicy, + rqe_plans: &BTreeMap, + id_map: &HashMap, +) -> YamlValue { + YamlValue::Mapping( + rqe_plans + .iter() + .map(|(id, plan)| { + let mut rqe_map = serde_yaml::Mapping::new(); + rqe_map.insert( + YamlValue::String(KEY_AGGREGATIONS.to_string()), + aggregation_refs_yaml(cleanup_policy, &plan.aggregation_keys, id_map), + ); + rqe_map.insert( + YamlValue::String(KEY_QUERY_TIME_AGGREGATIONS.to_string()), + query_time_aggregations_yaml(&plan.query_time_aggregations), + ); + (YamlValue::String(id.clone()), YamlValue::Mapping(rqe_map)) + }) + .collect(), + ) +} + pub fn params_to_yaml(params: &HashMap) -> YamlValue { if params.is_empty() { return YamlValue::Mapping(serde_yaml::Mapping::new()); diff --git a/asap-planner-rs/src/optimizer/milp.rs b/asap-planner-rs/src/optimizer/milp.rs index 9d3ae6cb..3fd99894 100644 --- a/asap-planner-rs/src/optimizer/milp.rs +++ b/asap-planner-rs/src/optimizer/milp.rs @@ -327,6 +327,19 @@ pub struct MilpWorkload { pub raqe_items: Vec, } +impl MilpWorkload { + /// Appends `item` as a single Raqe named `id`, returning the Raqe's index. + pub(crate) fn push_item(&mut self, item: OptimizerItem, id: &str) -> Result { + self.raqes.push(Raqe { + id: id.to_string(), + ..item_to_raqe(&item)? + }); + self.raqe_items.push(self.items.len()); + self.items.push(item); + Ok(self.raqes.len() - 1) + } +} + /// One Raqe per query occurrence: a `Raqe` has no frequency weight, so two /// dashboards running the same query at the same cadence become two /// identical Raqes, which the MILP serves from one shared deployment. diff --git a/asap-planner-rs/src/optimizer/milp_output.rs b/asap-planner-rs/src/optimizer/milp_output.rs index 5e33ce98..008460d6 100644 --- a/asap-planner-rs/src/optimizer/milp_output.rs +++ b/asap-planner-rs/src/optimizer/milp_output.rs @@ -1,9 +1,11 @@ //! Turns a solved MILP plan into the planner's streaming and inference YAML. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use asap_types::enums::{CleanupPolicy, WindowType}; -use asap_types::query_config::QueryTimeAggregationOperator; +use asap_types::query_config::{ + QueryTimeAggregation, QueryTimeAggregationOperator, QueryTimeGrouping, QueryTimeGroupingMode, +}; use indexmap::map::Entry; use indexmap::IndexMap; use promql_parser::label::Labels; @@ -16,7 +18,9 @@ use serde_json::Value; use thiserror::Error; use crate::error::ControllerError; -use crate::generator::{GeneratorOutput, QueryPlanEntry}; +use crate::generator::{ + build_rqe_configs_yaml, GeneratorOutput, QueryPlanEntry, RqePlanEntry, KEY_RQE_CONFIGS, +}; use crate::planner::agg_config::IntermediateAggConfig; use crate::planner::labels::set_subpopulation_labels; use crate::planner::promql::{aggregation_stage, parse_binary_arms, strip_parens}; @@ -143,16 +147,19 @@ pub fn plan_to_planner_output( .map(|planned| planned.deployment.clone()) .collect(); let served_by: Vec = solution.raqes.iter().map(|r| r.deployment).collect(); - deployments_to_planner_output(config, workload, &deployments, &served_by) + deployments_to_planner_output(config, workload, &deployments, &served_by, &BTreeMap::new()) } /// [`plan_to_planner_output`] for a plan given as its deployments and, per -/// Raqe of `workload`, the index of the deployment serving it. +/// Raqe of `workload`, the index of the deployment serving it. Each +/// `rqe_raqes` entry (RQE id -> index of the Raqe that is that RQE leaf) gets +/// an `rqe_configs` entry, written only when there are any. pub fn deployments_to_planner_output( config: MilpConfig, workload: &MilpWorkload, deployments: &[Deployment], served_by: &[usize], + rqe_raqes: &BTreeMap, ) -> Result { assert_eq!( served_by.len(), @@ -228,12 +235,27 @@ pub fn deployments_to_planner_output( } } + let mut rqe_plans: BTreeMap = BTreeMap::new(); + for (id, &raqe) in rqe_raqes { + let index = served_by[raqe]; + let key = &item_keys[&(index, workload.raqe_items[raqe])]; + rqe_plans.insert( + id.clone(), + rqe_plan_entry( + key, + &workload.raqes[raqe], + &deployments[index], + aggregations[key].aggregation_type, + )?, + ); + } + let id_map: HashMap = aggregations .keys() .enumerate() .map(|(index, key)| (key.clone(), index as u32 + 1)) .collect(); - let (streaming_yaml, inference_yaml) = match config { + let (streaming_yaml, mut inference_yaml) = match config { MilpConfig::Promql(config) => { let schema = config.schema_from_hints(); ( @@ -246,6 +268,15 @@ pub fn deployments_to_planner_output( build_sql_inference_yaml(config, CleanupPolicy::CircularBuffer, &queries, &id_map)?, ), }; + if !rqe_plans.is_empty() { + let serde_yaml::Value::Mapping(root) = &mut inference_yaml else { + unreachable!("the inference YAML is a mapping"); + }; + root.insert( + KEY_RQE_CONFIGS.into(), + build_rqe_configs_yaml(CleanupPolicy::CircularBuffer, &rqe_plans, &id_map), + ); + } Ok(PlannerOutput::from_output(GeneratorOutput { punted_queries: Vec::new(), streaming_yaml, @@ -272,61 +303,96 @@ fn check_grouping(raqe: &Raqe, deployment: &Deployment) -> Result<(), MilpOutput }) } -/// How `query` (an occurrence of `raqe`) reads the aggregation `key`. -/// -/// A Raqe grouped by a strict subset of its deployment's grouping is a -/// roll-up. The engine merges single-subpopulation sketches across the -/// deployment's groups itself, so those name the aggregation as is. Exact -/// keyed accumulators are read at the deployment's grouping and regrouped -/// at query time, which is exact for sum, count, min and max, unless -/// `stage_blocker` says `query` can't run a query-time stage. -fn query_plan_entry( +/// The closed windows of `deployment` that `raqe`'s lookback reads. The +/// store holds closed windows only; open ones live in the precompute engine. +fn retained_windows( query: &str, - key: &str, raqe: &Raqe, deployment: &Deployment, - aggregation_type: AggregationType, - stage_blocker: Option, -) -> Result { - // The store holds closed windows only; open ones live in the precompute - // engine. - let retained = deployment +) -> Result { + deployment .closed_instance_count(raqe.lookback_ms) .ok_or_else(|| MilpOutputError::PartialWindowLookback { query: query.to_string(), lookback_ms: raqe.lookback_ms, window_ms: deployment.window_ms, slide_ms: deployment.slide_ms, - })?; - let aggregation_keys = vec![(key.to_string(), Some(retained))]; + }) +} + +/// Whether `raqe` reads `deployment` through a query-time stage. +/// +/// A Raqe grouped by a strict subset of its deployment's grouping is a +/// roll-up. The engine merges single-subpopulation sketches across the +/// deployment's groups itself, so those need no stage. Exact keyed +/// accumulators are read at the deployment's grouping and regrouped at query +/// time, which is exact for sum, count, min and max. +fn needs_query_time_stage( + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, +) -> Result { let (fine, coarse) = (&deployment.grouping_labels, &raqe.grouping_labels); - if coarse == fine { - return Ok(QueryPlanEntry::fully_planned( - query.to_string(), - aggregation_keys, - )); + if coarse == fine || aggregation_type.merges_across_groups() { + return Ok(false); } - if aggregation_type.merges_across_groups() { - return Ok(QueryPlanEntry::fully_planned( - query.to_string(), - aggregation_keys, - )); + if matches!( + aggregation_type, + AggregationType::MultipleSum | AggregationType::MultipleMinMax + ) { + return Ok(true); } - let unsupported = |reason| MilpOutputError::RollupUnsupported { + Err(rollup_unsupported( + raqe, + deployment, + aggregation_type, + RollupBlocker::Aggregation, + )) +} + +fn rollup_unsupported( + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, + reason: RollupBlocker, +) -> MilpOutputError { + MilpOutputError::RollupUnsupported { query: raqe.id.clone(), aggregation_type, - raqe: label_names(coarse), - deployment: label_names(fine), + raqe: label_names(&raqe.grouping_labels), + deployment: label_names(&deployment.grouping_labels), reason, - }; - if !matches!( - aggregation_type, - AggregationType::MultipleSum | AggregationType::MultipleMinMax - ) { - return Err(unsupported(RollupBlocker::Aggregation)); + } +} + +/// How `query` (an occurrence of `raqe`) reads the aggregation `key`: the +/// query-time stage of a roll-up regroups `query`'s own outer aggregation, +/// unless `stage_blocker` says `query` can't run one. +fn query_plan_entry( + query: &str, + key: &str, + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, + stage_blocker: Option, +) -> Result { + let aggregation_keys = vec![( + key.to_string(), + Some(retained_windows(query, raqe, deployment)?), + )]; + if !needs_query_time_stage(raqe, deployment, aggregation_type)? { + return Ok(QueryPlanEntry::fully_planned( + query.to_string(), + aggregation_keys, + )); } if let Some(reason) = stage_blocker { - return Err(unsupported(reason)); + return Err(rollup_unsupported( + raqe, + deployment, + aggregation_type, + reason, + )); } let unregroupable = || MilpOutputError::UnregroupableQuery(query.to_string()); let aggregate = outer_aggregate(query).ok_or_else(unregroupable)?; @@ -337,11 +403,58 @@ fn query_plan_entry( } Ok(QueryPlanEntry { aggregation_keys, - planned_subquery: regrouped(aggregate, fine), + planned_subquery: regrouped(aggregate, &deployment.grouping_labels), query_time_aggregations: vec![stage], }) } +/// How the engine answers the RQE leaf `raqe` from the aggregation `key`. A +/// roll-up's query-time stage regroups by the RQE's own grouping. +fn rqe_plan_entry( + key: &str, + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, +) -> Result { + let aggregation_keys = vec![( + key.to_string(), + Some(retained_windows(&raqe.id, raqe, deployment)?), + )]; + let query_time_aggregations = if needs_query_time_stage(raqe, deployment, aggregation_type)? { + vec![rqe_roll_up_stage(raqe)] + } else { + Vec::new() + }; + Ok(RqePlanEntry { + aggregation_keys, + query_time_aggregations, + }) +} + +/// The stage that rolls an exact keyed accumulator up to `raqe`'s grouping. +fn rqe_roll_up_stage(raqe: &Raqe) -> QueryTimeAggregation { + let operator = match raqe.capability { + // A coarse count is the sum of the fine counts. + Capability::Sum | Capability::Count => QueryTimeAggregationOperator::Sum, + Capability::Min => QueryTimeAggregationOperator::Min, + Capability::Max => QueryTimeAggregationOperator::Max, + other => { + unreachable!("only exact sum, count, min and max roll up at query time, not {other:?}") + } + }; + let labels = label_names(&raqe.grouping_labels); + let mode = if labels.is_empty() { + QueryTimeGroupingMode::All + } else { + QueryTimeGroupingMode::By + }; + QueryTimeAggregation { + operator, + grouping: QueryTimeGrouping { mode, labels }, + parameter: None, + } +} + /// The leaves of the workload's binary queries. The engine runs a binary /// query's arms without query-time stages. fn binary_query_leaves(config: MilpConfig) -> HashSet { @@ -896,9 +1009,14 @@ mod tests { deployment.config.sketch = "hll".into(); deployment.config.sketch_config["params"] = json!({"lg_k": 12}); - let output = - deployments_to_planner_output(MilpConfig::Sql(&sql), &workload, &[deployment], &[0]) - .unwrap(); + let output = deployments_to_planner_output( + MilpConfig::Sql(&sql), + &workload, + &[deployment], + &[0], + &BTreeMap::new(), + ) + .unwrap(); let a = aggregation_in(&output, query, QueryLanguage::sql); assert_eq!(a.aggregation_type, AggregationType::HLL); assert_eq!(a.grouping_labels, instance_job()); diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index 325274ad..4929de92 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -1,7 +1,9 @@ //! Writes a plan solved elsewhere, sketch-bench `autosketch_vs_asap`'s plan //! file, as the planner's streaming and inference YAML, without re-solving. //! The workload comes from the eval table's `e2e` block: its PromQL or SQL -//! queries, the plan RQEs each reads, and the metrics' labels. +//! queries, the plan RQEs each reads, and the metrics' labels. A query's RQE +//! expression tree (`expr`) also gets each of its leaves an `rqe_configs` +//! entry, so the engine can answer the tree without parsing PromQL or SQL. use std::collections::{BTreeMap, HashMap}; use std::io::Read; @@ -9,6 +11,11 @@ use std::path::Path; use anyhow::{bail, ensure, Context}; use asap_types::enums::QueryLanguage; +use asap_types::query_requirements::QueryRequirements; +use asap_types::rqe_expr::{RqeCapability, RqeExpr, RqeLeaf}; +use asap_types::utils::normalize_spatial_filter; +use promql_utilities::data_model::KeyByLabelNames; +use promql_utilities::query_logics::enums::Statistic; use rqe_optimizer::{heap_capacity, AtomicCostEntry, Deployment, LabelSet, Raqe}; use serde::Deserialize; use serde_json::{json, Value}; @@ -18,9 +25,10 @@ use crate::config::input::{ }; use crate::planner_output::PlannerOutput; -use super::aqe_extractor::decompose_to_leaves; +use super::aqe_extractor::{decompose_to_leaves, sql_metric}; use super::milp::{workload_from_config, MilpConfig}; use super::milp_output::{deployments_to_planner_output, reject_unwritable_queries}; +use super::solution::OptimizerItem; #[derive(Deserialize)] struct Table { @@ -44,28 +52,88 @@ const SQL_TIME_COLUMN: &str = "ts"; /// The metric type whose metrics are SQL tables. const RECORDS_METRIC_TYPE: &str = "records"; -/// One query, written in exactly one of PromQL and SQL. +/// One query: written in at most one of PromQL and SQL, and with an RQE +/// expression tree, `expr`, or not. It needs text or a tree. #[derive(Deserialize)] struct E2eQuery { promql: Option, sql: Option, interval_secs: u64, /// Plan RQE ids: for `promql`, one per leaf in order; for `sql`, a single - /// leaf, every RQE it reads. + /// leaf, every RQE it reads. With `expr`, its leaves' ids, which this + /// may omit. + #[serde(default)] rqes: Vec, + /// Leaves name plan RQE ids. + expr: Option, } impl E2eQuery { - /// The query if it is written in SQL (`sql`) or PromQL (otherwise). - fn text(&self, sql: bool) -> anyhow::Result> { + /// The query's PromQL or SQL text, if any. + fn text(&self) -> anyhow::Result> { match (&self.promql, &self.sql) { - (Some(_), Some(_)) | (None, None) => bail!( + (Some(_), Some(_)) => bail!( "e2e query reading {:?} needs exactly one of `promql` and `sql`", self.rqes ), - (promql, sql_text) => Ok(if sql { sql_text } else { promql }.as_deref()), + (promql, sql) => Ok(promql.as_deref().or(sql.as_deref())), + } + } + + /// Whether the query is SQL: written in `sql`, or, with no text, a tree + /// whose leaves read `records` metrics (SQL tables). + fn is_sql(&self, metrics: &BTreeMap) -> anyhow::Result { + let text_sql = self.text()?.map(|_| self.sql.is_some()); + let tree_sql = match &self.expr { + None => None, + Some(expr) => { + let mut kinds = Vec::new(); + for leaf in expr.leaves() { + let metric = metrics.get(&leaf.metric).with_context(|| { + format!("RQE {:?} reads unknown metric {:?}", leaf.id, leaf.metric) + })?; + kinds.push(metric.kind == RECORDS_METRIC_TYPE); + } + kinds.dedup(); + match kinds.as_slice() { + [] => bail!("e2e query reading {:?}: its `expr` reads no RQE", self.rqes), + [sql] => Some(*sql), + _ => bail!( + "e2e query reading {:?}: its `expr` mixes records and non-records metrics", + self.rqes + ), + } + } + }; + match (text_sql, tree_sql) { + (Some(text), Some(tree)) if text != tree => bail!( + "e2e query reading {:?} is written in {} but its `expr` reads {} metrics", + self.rqes, + if text { "SQL" } else { "PromQL" }, + if tree { "records" } else { "non-records" } + ), + (Some(sql), _) | (None, Some(sql)) => Ok(sql), + (None, None) => bail!( + "e2e query reading {:?} needs `promql`, `sql` or `expr`", + self.rqes + ), } } + + /// The plan RQEs the query reads: its tree's leaves, if it has one, + /// which `rqes` must then list in order or omit. + fn rqe_ids(&self) -> anyhow::Result> { + let rqes: Vec<&str> = self.rqes.iter().map(String::as_str).collect(); + let Some(expr) = &self.expr else { + return Ok(rqes); + }; + let leaves: Vec<&str> = expr.leaves().iter().map(|l| l.id.as_str()).collect(); + ensure!( + rqes.is_empty() || rqes == leaves, + "e2e query names RQEs {rqes:?} but its `expr` reads {leaves:?}" + ); + Ok(leaves) + } } #[derive(Deserialize)] @@ -263,14 +331,26 @@ pub fn saved_plan_output( c.rqe ); } + // Queries in `language`: those with text, and those with a tree. let mut selected: Vec<(&E2eQuery, &str)> = Vec::new(); + let mut trees: Vec<(&E2eQuery, &RqeExpr)> = Vec::new(); for q in &e2e.queries { - if let Some(text) = q.text(sql)? { + if let Some(expr) = &q.expr { + expr.validate() + .with_context(|| format!("e2e query reading {:?}: invalid `expr`", q.rqes))?; + } + if q.is_sql(&e2e.metrics)? != sql { + continue; + } + if let Some(text) = q.text()? { selected.push((q, text)); } + if let Some(expr) = &q.expr { + trees.push((q, expr)); + } } ensure!( - !selected.is_empty(), + !selected.is_empty() || !trees.is_empty(), "{}: no e2e query is written in {language:?}", table_path.display() ); @@ -290,24 +370,27 @@ pub fn saved_plan_output( // Leaf query string and repeat interval -> the plan RQEs it reads. let mut rqes_of: HashMap<(String, u64), Vec<&str>> = HashMap::new(); for &(q, text) in &selected { - let rqes = q.rqes.iter().map(String::as_str); + let rqe_ids = q.rqe_ids()?; let leaves: Vec<(String, Vec<&str>)> = match config { // A SQL query is one leaf, and its RQEs a set. MilpConfig::Sql(_) => { - ensure!(!q.rqes.is_empty(), "query {text:?} names no RQEs"); - let mut rqes: Vec<&str> = rqes.collect(); + ensure!(!rqe_ids.is_empty(), "query {text:?} names no RQEs"); + let mut rqes = rqe_ids; rqes.sort_unstable(); vec![(text.to_string(), rqes)] } MilpConfig::Promql(_) => { let leaves = decompose_to_leaves(text); ensure!( - leaves.len() == q.rqes.len(), + leaves.len() == rqe_ids.len(), "query {text:?} has {} leaves but names {} RQEs", leaves.len(), - q.rqes.len() + rqe_ids.len() ); - leaves.into_iter().zip(rqes.map(|r| vec![r])).collect() + leaves + .into_iter() + .zip(rqe_ids.into_iter().map(|r| vec![r])) + .collect() } }; for (leaf, rqes) in leaves { @@ -323,27 +406,64 @@ pub fn saved_plan_output( } } - let workload = workload_from_config(config, scrape_interval_ms)?; + // Every tree leaf, by RQE id, and the repeat interval of its queries. + let mut tree_leaves: BTreeMap<&str, (&RqeLeaf, u64)> = BTreeMap::new(); + for (q, expr) in &trees { + // A tree-only query's `rqes` must agree with its tree too. + q.rqe_ids()?; + for leaf in expr.leaves() { + let interval_ms = q.interval_secs * 1000; + if let Some((other, other_interval_ms)) = + tree_leaves.insert(&leaf.id, (leaf, interval_ms)) + { + ensure!( + other == leaf && other_interval_ms == interval_ms, + "RQE {:?} is two different leaves: {other:?} every {other_interval_ms} ms \ + and {leaf:?} every {interval_ms} ms", + leaf.id + ); + } + } + } + + let mut workload = workload_from_config(config, scrape_interval_ms)?; + let text_items = workload.items.len(); + // Each tree leaf is one more item and Raqe, read directly by its RQE + // id; `leaf_rqes[i]` is item `text_items + i`'s. + let mut rqe_raqes: BTreeMap = BTreeMap::new(); + let mut leaf_rqes: Vec<&str> = Vec::new(); + for (&id, &(leaf, interval_ms)) in &tree_leaves { + let item = leaf_item(leaf, interval_ms, &e2e.metrics)?; + rqe_raqes.insert(id.to_string(), workload.push_item(item, id)?); + leaf_rqes.push(id); + } // Plan deployment index -> its position in `deployments`, in order of // the first Raqe each serves. let mut position: HashMap = HashMap::new(); let mut deployments: Vec = Vec::new(); let mut served_by = Vec::with_capacity(workload.raqes.len()); - for (raqe, &item) in workload.raqes.iter().zip(&workload.raqe_items) { - let item = &workload.items[item]; + for (raqe, &item_index) in workload.raqes.iter().zip(&workload.raqe_items) { + let item = &workload.items[item_index]; // An item's leaves (say p90 and p50 over one window), and every RQE // a SQL leaf reads, read one deployment, so their picks must name the - // same sketch. - let mut index = None; - let mut reads: Vec<(&String, &str)> = Vec::new(); - for leaf in &item.query_strings { - let rqes = rqes_of - .get(&(leaf.clone(), item.t_repeat_ms)) - .with_context(|| format!("leaf {leaf:?} has no RQE in the table's e2e block"))?; - reads.extend(rqes.iter().map(|&rqe| (leaf, rqe))); + // same sketch. Each (reader, RQE): a leaf query, or a tree leaf's id. + let mut reads: Vec<(&str, &str)> = Vec::new(); + if item_index < text_items { + for leaf in &item.query_strings { + let rqes = rqes_of + .get(&(leaf.clone(), item.t_repeat_ms)) + .with_context(|| { + format!("leaf {leaf:?} has no RQE in the table's e2e block") + })?; + reads.extend(rqes.iter().map(|&rqe| (leaf.as_str(), rqe))); + } + } else { + let id = leaf_rqes[item_index - text_items]; + reads.push((id, id)); } - for (leaf, rqe) in reads { + let mut index = None; + for &(reader, rqe) in &reads { let pick = chosen .get(rqe) .with_context(|| format!("the plan doesn't serve RQE {rqe:?}"))?; @@ -364,7 +484,7 @@ pub fn saved_plan_output( }; ensure!( can_serve(raqe, &deployment), - "RQE {rqe:?}'s {} {}s window sliding every {}s can't serve {leaf:?} \ + "RQE {rqe:?}'s {} {}s window sliding every {}s can't serve {reader:?} \ ({} ms lookback every {} ms); does --plan match --workload-table?", pick.sketch, pick.window_secs, @@ -386,11 +506,11 @@ pub fn saved_plan_output( ensure!( index.is_none_or(|index| index == at), "query {:?} is served by two deployments", - item.query_strings + reads.iter().map(|&(reader, _)| reader).collect::>() ); index = Some(at); } - let index = index.expect("an item has a query string"); + let index = index.expect("an item reads an RQE"); served_by.push(index); } Ok(deployments_to_planner_output( @@ -398,9 +518,79 @@ pub fn saved_plan_output( &workload, &deployments, &served_by, + &rqe_raqes, )?) } +/// The optimizer item of an RQE leaf, read every `interval_ms`: what a +/// PromQL or SQL leaf with the same capability, metric, lookback and grouping +/// needs. It has no query string, so it gets no `query_configs` entry. +fn leaf_item( + leaf: &RqeLeaf, + interval_ms: u64, + metrics: &BTreeMap, +) -> anyhow::Result { + let id = &leaf.id; + let m = metrics + .get(&leaf.metric) + .with_context(|| format!("RQE {id:?} reads unknown metric {:?}", leaf.metric))?; + let metric = match (m.kind == RECORDS_METRIC_TYPE, &leaf.value) { + (true, Some(value)) => { + ensure!( + m.values.contains_key(value), + "RQE {id:?} reads {value:?}, which is not a value of {:?}", + leaf.metric + ); + sql_metric(&leaf.metric, value) + } + (true, None) => bail!( + "RQE {id:?} reads records metric {:?} without a `value`", + leaf.metric + ), + (false, Some(_)) => bail!( + "RQE {id:?} has a `value`, but {:?} is not a records metric", + leaf.metric + ), + (false, None) => leaf.metric.clone(), + }; + for label in &leaf.grouping { + ensure!( + m.labels.iter().any(|l| &l.name == label), + "RQE {id:?} groups by {label:?}, which is not a label of {:?}", + leaf.metric + ); + } + ensure!(leaf.lookback_ms > 0, "RQE {id:?} looks back 0 ms"); + ensure!(interval_ms > 0, "RQE {id:?} repeats every 0 s"); + let (statistic, topk_count_events) = match leaf.capability { + RqeCapability::Sum => (Statistic::Sum, None), + RqeCapability::Quantile => (Statistic::Quantile, None), + RqeCapability::Cardinality => (Statistic::Cardinality, None), + // Value-ranked, as `topk(k, sum_over_time(...))` is. + RqeCapability::TopkByValue => (Statistic::Topk, Some(false)), + }; + // A text query of a saved plan's table has the default options too. + let options = ControllerOptions::default(); + Ok(OptimizerItem { + requirements: QueryRequirements { + metric, + statistics: vec![statistic], + data_range_ms: leaf.lookback_ms, + grouping_labels: KeyByLabelNames::new(leaf.grouping.clone()), + spatial_filter_normalized: normalize_spatial_filter(""), + topk_count_events, + topk_by_labels: None, + }, + query_strings: Vec::new(), + query_frequency_hz: 1000.0 / interval_ms as f64, + occurrences: 1, + t_repeat_ms: interval_ms, + accuracy_sla: options.accuracy_sla, + latency_sla_ms: options.latency_sla_ms, + topk_k: leaf.k, + }) +} + /// Largest gap between `secs` in ms and the nearest whole ms that is still /// float error, not a fraction of a ms. const WHOLE_MS_TOLERANCE: f64 = 1e-6; @@ -956,4 +1146,475 @@ mod tests { let err = refusal(plan, None); assert!(err.contains(r#"doesn't serve RQE "p50""#), "{err}"); } + + fn leaf(id: &str, capability: &str, lookback_ms: u64, grouping: &[&str]) -> Value { + json!({"rqe": {"id": id, "metric": "data", "capability": capability, + "lookback_ms": lookback_ms, "grouping": grouping}}) + } + + fn quantile_leaf(id: &str, phi: f64, lookback_ms: u64, grouping: &[&str]) -> Value { + let mut l = leaf(id, "quantile", lookback_ms, grouping); + l["rqe"]["phi"] = json!(phi); + l + } + + /// `TABLE` with each query's RQE tree. + fn tree_table() -> Value { + let mut table: Value = serde_json::from_str(TABLE).unwrap(); + let series = ["job", "label_0"]; + let p90 = quantile_leaf("p90", 0.9, 300_000, &series); + let p50 = quantile_leaf("p50", 0.5, 300_000, &series); + let exprs = [ + leaf("sum", "sum", 1000, &["job"]), + json!({"op": "/", "lhs": p90, "rhs": p50.clone()}), + p50, + ]; + for (q, expr) in queries(&mut table).iter_mut().zip(exprs) { + q["expr"] = expr; + } + table + } + + fn queries(table: &mut Value) -> &mut Vec { + table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + } + + fn run_table( + table: &Value, + plan: &Value, + language: QueryLanguage, + ) -> anyhow::Result { + run_in( + &table.to_string(), + plan.to_string().as_bytes(), + None, + language, + ) + } + + fn inference_of( + output: &PlannerOutput, + language: QueryLanguage, + ) -> asap_types::InferenceConfig { + output.to_inference_config(language).unwrap() + } + + fn query_aggregation_ids(inference: &asap_types::InferenceConfig, query: &str) -> Vec { + let config = inference + .query_configs + .iter() + .find(|q| q.query == query) + .unwrap_or_else(|| panic!("no query_config for {query:?}")); + config + .aggregations + .iter() + .map(|a| a.aggregation_id) + .collect() + } + + /// (aggregation id, retained windows) per aggregation the RQE reads. + fn rqe_aggregations( + inference: &asap_types::InferenceConfig, + rqe: &str, + ) -> Vec<(u64, Option)> { + inference.rqe_configs[rqe] + .aggregations + .iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect() + } + + fn stages(inference: &asap_types::InferenceConfig, rqe: &str) -> Value { + serde_json::to_value(&inference.rqe_configs[rqe].query_time_aggregations).unwrap() + } + + /// Each tree leaf reads the aggregation its text query reads. The + /// ratio's two leaves are two entries on one aggregation. + #[test] + fn writes_an_rqe_config_per_tree_leaf() { + let output = run_table(&tree_table(), &plans(shared_p50()), QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["p50", "p90", "sum"]); + let [sum_id] = query_aggregation_ids(&inference, "sum by (job) (data)")[..] else { + panic!("one aggregation"); + }; + assert_eq!(rqe_aggregations(&inference, "sum"), [(sum_id, Some(1))]); + let [dd_id] = query_aggregation_ids(&inference, "quantile_over_time(0.5, data[5m])")[..] + else { + panic!("one aggregation"); + }; + assert_ne!(dd_id, sum_id); + // 5m of a 5m window sliding every minute: the newest closed window. + for rqe in ["p90", "p50"] { + assert_eq!( + rqe_aggregations(&inference, rqe), + [(dd_id, Some(1))], + "{rqe}" + ); + assert_eq!(stages(&inference, rqe), json!([]), "{rqe}"); + } + assert_eq!(stages(&inference, "sum"), json!([])); + } + + /// Trees only add `rqe_configs`: the deployments and `query_configs` + /// are those of the same table without trees, which has no `rqe_configs`. + #[test] + fn trees_leave_the_text_queries_configs_unchanged() { + let plan = plans(shared_p50()); + let without = run_on(TABLE, plan.to_string().as_bytes(), None).unwrap(); + let with = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + + assert_eq!( + with.to_streaming_yaml_string().unwrap(), + without.to_streaming_yaml_string().unwrap() + ); + let without_yaml = without.to_inference_yaml_string().unwrap(); + assert!(!without_yaml.contains("rqe_configs"), "{without_yaml}"); + let mut with_inference: serde_yaml::Value = + serde_yaml::from_str(&with.to_inference_yaml_string().unwrap()).unwrap(); + assert!(with_inference + .as_mapping_mut() + .unwrap() + .remove("rqe_configs") + .is_some()); + assert_eq!( + serde_yaml::to_string(&with_inference).unwrap(), + without_yaml + ); + } + + #[test] + fn writes_an_exact_sum_roll_up_as_an_rqe_query_time_sum() { + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"][0]["deployment_grouping"] = json!(["job", "label_0"]); + let output = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + + assert_eq!( + stages(&inference, "sum"), + json!([{ + "operator": "sum", + "grouping": {"mode": "by", "labels": ["job"]}, + "parameter": null, + }]) + ); + assert_eq!( + rqe_aggregations(&inference, "sum")[0].0, + query_aggregation_ids(&inference, "sum by (job) (data)")[0] + ); + } + + /// The engine merges a sketch across the deployment's groups itself, so + /// the RQE reads the deployment's aggregation with no stage. + #[test] + fn writes_a_sketch_roll_up_rqe_without_a_stage() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"promql": "quantile by (job) (0.5, data)", "interval_secs": 1, + "expr": quantile_leaf("p50", 0.5, 1000, &["job"])}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let mut p50 = pick("p50", 0, "dd", json!({"alpha": 0.02}), 1); + p50["deployment_grouping"] = json!(["job", "label_0"]); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [p50]}, + ]}); + + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!(output.streaming_aggregation_count(), 1); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50"), [(1, Some(1))]); + assert_eq!(stages(&inference, "p50"), json!([])); + } + + /// A query with only a tree is planned in the language of the metrics + /// its leaves read: it gets a deployment and an `rqe_configs` entry, but + /// no `query_configs` entry. + #[test] + fn plans_a_tree_only_query() { + let mut table = tree_table(); + let tree_only = + json!({"interval_secs": 1, "expr": quantile_leaf("p50_by_job", 0.5, 1000, &["job"])}); + queries(&mut table).push(tree_only.clone()); + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"] + .as_array_mut() + .unwrap() + .push(pick("p50_by_job", 5, "dd", json!({"alpha": 0.05}), 1)); + + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!(output.streaming_aggregation_count(), 3); + assert_eq!(output.inference_query_count(), 3); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50_by_job"), [(3, Some(1))]); + let streaming = output.to_streaming_config(QueryLanguage::promql).unwrap(); + let dd = &streaming.aggregation_configs[&3]; + assert_eq!( + dd.aggregation_type, + promql_utilities::query_logics::enums::AggregationType::DDSketch + ); + assert_eq!(dd.grouping_labels.labels, ["job"]); + + // A table of tree-only queries converts too. + let table = json!({"workloads": [{"e2e": { + "queries": [tree_only], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + assert_eq!(output.streaming_aggregation_count(), 1); + assert_eq!(output.inference_query_count(), 0); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50_by_job"), [(1, Some(1))]); + let err = run_table(&table, &plan, QueryLanguage::sql) + .err() + .expect("refused") + .to_string(); + assert!(err.contains("no e2e query is written in sql"), "{err}"); + } + + fn http_leaf(id: &str, capability: &str, grouping: &[&str], value: &str) -> Value { + json!({"rqe": {"id": id, "metric": "http", "capability": capability, + "lookback_ms": 300_000, "grouping": grouping, "value": value}}) + } + + /// `MIXED_TABLE`'s p99 query with its tree, and a tree-only SQL sum. + fn mixed_tree_table() -> Value { + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + let mut p99 = http_leaf("p99", "quantile", &["region"], "latency"); + p99["rqe"]["phi"] = json!(0.99); + queries(&mut table)[2]["expr"] = p99; + queries(&mut table).push(json!({"interval_secs": 60, + "expr": http_leaf("latency_sum", "sum", &["region"], "latency")})); + table + } + + /// A SQL query can't run a query-time stage, but its RQE can: an exact + /// roll-up of a tree-only SQL sum is written. + #[test] + fn writes_sql_rqe_configs_with_an_exact_roll_up() { + let mut plan = mixed_plan(1); + let mut sum = pick("latency_sum", 4, "exact-sum", json!({}), 60); + sum["deployment_grouping"] = json!(["region", "service"]); + plan["plans"][0]["chosen"].as_array_mut().unwrap().push(sum); + + let output = run_table(&mixed_tree_table(), &plan, QueryLanguage::sql).unwrap(); + + let inference = inference_of(&output, QueryLanguage::sql); + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["latency_sum", "p99"]); + let p99_query = "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region"; + // 5m of 5m windows sliding every minute: the newest closed window. + assert_eq!( + rqe_aggregations(&inference, "p99"), + [(query_aggregation_ids(&inference, p99_query)[0], Some(1))] + ); + assert_eq!( + stages(&inference, "latency_sum"), + json!([{ + "operator": "sum", + "grouping": {"mode": "by", "labels": ["region"]}, + "parameter": null, + }]) + ); + let streaming = output.to_streaming_config(QueryLanguage::sql).unwrap(); + let (sum_id, _) = rqe_aggregations(&inference, "latency_sum")[0]; + let sum = &streaming.aggregation_configs[&sum_id]; + assert_eq!(sum.table_name.as_deref(), Some("http")); + assert_eq!(sum.value_column.as_deref(), Some("latency")); + // A keyed accumulator keeps its groups as keys. + assert_eq!(sum.aggregated_labels.labels, ["region", "service"]); + + // The PromQL run skips the SQL trees. + let output = run_table(&mixed_tree_table(), &plan, QueryLanguage::promql).unwrap(); + assert!(inference_rqe_configs_absent(&output)); + } + + fn inference_rqe_configs_absent(output: &PlannerOutput) -> bool { + !output + .to_inference_yaml_string() + .unwrap() + .contains("rqe_configs") + } + + fn tree_refusal(table: &Value) -> String { + format!( + "{:#}", + run_table(table, &plans(shared_p50()), QueryLanguage::promql) + .err() + .expect("refused") + ) + } + + #[test] + fn refuses_a_tree_disagreeing_with_its_rqes() { + let mut table = tree_table(); + queries(&mut table)[1]["rqes"] = json!(["p50", "p90"]); + let err = tree_refusal(&table); + assert!( + err.contains(r#"names RQEs ["p50", "p90"] but its `expr` reads ["p90", "p50"]"#), + "{err}" + ); + } + + /// `rqes` may be omitted: the tree names the RQEs. + #[test] + fn reads_the_rqes_of_a_tree_without_rqes() { + let mut table = tree_table(); + for q in queries(&mut table).iter_mut() { + q.as_object_mut().unwrap().remove("rqes"); + } + let output = run_table(&table, &plans(shared_p50()), QueryLanguage::promql).unwrap(); + assert_eq!(output.inference_query_count(), 3); + assert_eq!( + inference_of(&output, QueryLanguage::promql) + .rqe_configs + .len(), + 3 + ); + } + + #[test] + fn refuses_an_invalid_tree() { + let mut table = tree_table(); + queries(&mut table)[0]["expr"] = + json!({"agg": "sum", "by": ["region"], "of": leaf("sum", "sum", 1000, &["job"])}); + let err = tree_refusal(&table); + assert!(err.contains("invalid `expr`"), "{err}"); + assert!(err.contains("not a subset"), "{err}"); + } + + #[test] + fn refuses_one_rqe_as_two_different_leaves() { + let mut table = tree_table(); + queries(&mut table)[2]["expr"] = quantile_leaf("p50", 0.5, 300_000, &["job"]); + let err = tree_refusal(&table); + assert!( + err.contains(r#"RQE "p50" is two different leaves"#), + "{err}" + ); + } + + #[test] + fn refuses_a_query_without_text_or_tree_or_with_mismatched_languages() { + let mut mixed: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + let cases = [ + ( + json!({"interval_secs": 1, "rqes": ["sum"]}), + "needs `promql`, `sql` or `expr`", + ), + ( + json!({"promql": "sum by (job) (data)", "interval_secs": 1, + "expr": http_leaf("sum", "sum", &[], "latency")}), + "written in PromQL but its `expr` reads records metrics", + ), + ( + json!({"interval_secs": 1, "expr": {"op": "+", + "lhs": leaf("sum", "sum", 1000, &[]), + "rhs": http_leaf("sum2", "sum", &[], "latency")}}), + "mixes records and non-records metrics", + ), + ( + json!({"interval_secs": 1, "expr": {"scalar": 1.0}}), + "its `expr` reads no RQE", + ), + ]; + for (query, reason) in cases { + queries(&mut mixed).push(query); + let err = format!( + "{:#}", + run_table(&mixed, &mixed_plan(1), QueryLanguage::sql) + .err() + .expect("refused") + ); + assert!(err.contains(reason), "{err}"); + queries(&mut mixed).pop(); + } + } + + #[test] + fn refuses_a_leaf_the_table_cannot_describe() { + let records_leaf = |value: Option<&str>, grouping: &[&str]| { + let mut l = http_leaf("p99", "quantile", grouping, "latency"); + l["rqe"]["phi"] = json!(0.99); + match value { + Some(value) => l["rqe"]["value"] = json!(value), + None => { + l["rqe"].as_object_mut().unwrap().remove("value"); + } + } + l + }; + let mut unknown = records_leaf(Some("latency"), &["region"]); + unknown["rqe"]["metric"] = json!("nope"); + for (expr, reason) in [ + (unknown, r#"reads unknown metric "nope""#), + (records_leaf(None, &["region"]), "without a `value`"), + ( + records_leaf(Some("bytes"), &["region"]), + r#"which is not a value of "http""#, + ), + ( + records_leaf(Some("latency"), &["zone"]), + r#"groups by "zone""#, + ), + ] { + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + queries(&mut table)[2]["expr"] = expr; + let err = format!( + "{:#}", + run_table(&table, &mixed_plan(1), QueryLanguage::sql) + .err() + .expect("refused") + ); + assert!(err.contains(reason), "{err}"); + } + let mut table = tree_table(); + queries(&mut table)[0]["expr"]["rqe"]["value"] = json!("v"); + let err = tree_refusal(&table); + assert!(err.contains("is not a records metric"), "{err}"); + } + + #[test] + fn refuses_a_tree_leaf_the_plan_does_not_serve() { + let mut table = tree_table(); + queries(&mut table).push(json!({"interval_secs": 1, + "expr": quantile_leaf("unplanned", 0.5, 1000, &["job"])})); + let err = tree_refusal(&table); + assert!(err.contains(r#"doesn't serve RQE "unplanned""#), "{err}"); + } + + /// A topk deployment can't roll up, for an RQE as for a query. + #[test] + fn refuses_a_tree_only_topk_roll_up() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"interval_secs": 60, "expr": {"rqe": {"id": "top", "metric": "data", + "capability": "topk_by_value", "k": 5, "lookback_ms": 300_000, + "grouping": ["job", "label_0"]}}}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let mut top = pick( + "top", + 0, + "cms-heap-topk-fastpath-vector2d", + json!({"rows": 3, "cols": 1024, "heap": 50}), + 300, + ); + top["deployment_grouping"] = json!(["job"]); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [top]}, + ]}); + let err = run_table(&table, &plan, QueryLanguage::promql) + .err() + .expect("refused") + .to_string(); + assert!(err.contains(r#"query "top""#), "{err}"); + assert!(err.contains("roll-ups need an aggregation"), "{err}"); + } } diff --git a/asap-planner-rs/tests/saved_plan_e2e_tables.rs b/asap-planner-rs/tests/saved_plan_e2e_tables.rs new file mode 100644 index 00000000..cdcd3b6b --- /dev/null +++ b/asap-planner-rs/tests/saved_plan_e2e_tables.rs @@ -0,0 +1,153 @@ +//! `--planner saved` on a real sketch-bench eval table: the `all` workload of +//! sketch-bench's `rqe-optimizer/results/e2e-rate10` (sketch-bench c1fd812), +//! whose e2e queries carry RQE trees, and its saved plans. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; + +use asap_planner::optimizer::{saved_plan_output, PlanSelector}; +use asap_planner::PlannerOutput; +use asap_types::enums::QueryLanguage; +use serde_json::Value; + +/// The table's scrape interval: instant RQEs read one 100 ms scrape. +const SCRAPE_INTERVAL_MS: u64 = 100; + +fn data_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/test_data/saved_plan") +} + +fn table_path() -> PathBuf { + data_dir().join("synthetic-templatesall-shared1.json") +} + +fn convert(table: &Path, language: QueryLanguage) -> PlannerOutput { + let selector = PlanSelector { + method: "asap", + weights: "cpu", + sla_ms: None, + }; + saved_plan_output( + table, + &data_dir().join("synthetic-templatesall-shared1.plans.json.gz"), + &selector, + language, + SCRAPE_INTERVAL_MS, + ) + .unwrap() +} + +fn table_queries() -> Vec { + let table: Value = + serde_json::from_str(&std::fs::read_to_string(table_path()).unwrap()).unwrap(); + table["workloads"][0]["e2e"]["queries"] + .as_array() + .unwrap() + .clone() +} + +fn language_key(language: QueryLanguage) -> &'static str { + match language { + QueryLanguage::promql => "promql", + QueryLanguage::sql => "sql", + other => panic!("{other:?}"), + } +} + +fn leaf_ids(expr: &Value, out: &mut BTreeSet) { + if let Some(rqe) = expr.get("rqe") { + out.insert(rqe["id"].as_str().unwrap().to_string()); + } + for child in ["of", "lhs", "rhs"] { + if let Some(child) = expr.get(child) { + leaf_ids(child, out); + } + } +} + +/// Every tree leaf of the language's queries gets an `rqe_configs` entry +/// reading what its single-RQE query reads. +#[test] +fn writes_rqe_configs_for_every_tree_leaf() { + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let output = convert(&table_path(), language); + let inference = output.to_inference_config(language).unwrap(); + let queries: Vec = table_queries() + .into_iter() + .filter(|q| q.get(language_key(language)).is_some()) + .collect(); + + let mut expected = BTreeSet::new(); + for q in &queries { + if let Some(expr) = q.get("expr") { + leaf_ids(expr, &mut expected); + } + } + assert!(!expected.is_empty(), "{language:?}"); + let written: BTreeSet = inference.rqe_configs.keys().cloned().collect(); + assert_eq!(written, expected, "{language:?}"); + + let query_configs: HashMap<&str, _> = inference + .query_configs + .iter() + .map(|c| (c.query.as_str(), c)) + .collect(); + for q in &queries { + let [rqe] = q["rqes"].as_array().unwrap().as_slice() else { + continue; + }; + let Some(rqe) = inference.rqe_configs.get(rqe.as_str().unwrap()) else { + continue; + }; + let query = query_configs[q[language_key(language)].as_str().unwrap()]; + let ids = |refs: &[asap_types::AggregationReference]| { + refs.iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect::>() + }; + assert_eq!(ids(&rqe.aggregations), ids(&query.aggregations), "{q}"); + assert_eq!( + rqe.query_time_aggregations.len(), + query.query_time_aggregations.len(), + "{q}" + ); + } + } +} + +/// The trees add only `rqe_configs`: the table without them converts to the +/// same deployments and `query_configs`. +#[test] +fn trees_add_only_rqe_configs() { + let mut table: Value = + serde_json::from_str(&std::fs::read_to_string(table_path()).unwrap()).unwrap(); + for q in table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + { + q.as_object_mut().unwrap().remove("expr"); + } + let dir = tempfile::tempdir().unwrap(); + let stripped = dir.path().join("table.json"); + std::fs::write(&stripped, table.to_string()).unwrap(); + + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let with = convert(&table_path(), language); + let without = convert(&stripped, language); + let yaml = |s: String| serde_yaml::from_str::(&s).unwrap(); + assert_eq!( + yaml(with.to_streaming_yaml_string().unwrap()), + yaml(without.to_streaming_yaml_string().unwrap()), + "{language:?}" + ); + let mut with_inference = yaml(with.to_inference_yaml_string().unwrap()); + let without_inference = yaml(without.to_inference_yaml_string().unwrap()); + assert!(without_inference.get("rqe_configs").is_none()); + assert!(with_inference + .as_mapping_mut() + .unwrap() + .remove("rqe_configs") + .is_some()); + assert_eq!(with_inference, without_inference, "{language:?}"); + } +} diff --git a/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json new file mode 100644 index 00000000..3102ab21 --- /dev/null +++ b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json @@ -0,0 +1 @@ +{"inputs": {}, "plan": "ProjectASAP/ASAPQuery#777", "schema_version": 1, "sketch_bench_revision": "acc2db1", "workloads": [{"dataset": "synthetic/templates=all/shared=1", "e2e": {"metrics": {"data": {"labels": [{"cardinality": 10, "name": "job"}, {"cardinality": 1000, "child_of": "job", "name": "label_0"}], "samples_per_sec_per_series": 10, "type": "gauge", "values": {"dist": {"pareto": 2.0}, "value": {"keys": 10000, "zipf": 1.1}}}, "data_total": {"labels": [{"cardinality": 10, "name": "job"}, {"cardinality": 1000, "child_of": "job", "name": "label_0"}], "samples_per_sec_per_series": 10, "type": "counter", "values": {"increment": {"keys": 10000, "zipf": 1.1}}}, "flows": {"labels": [{"cardinality": 1000, "name": "dst_subnet", "skew": 1.1}, {"cardinality": 1000, "name": "dst_port", "skew": 1.2}, {"cardinality": 3, "name": "proto", "skew": 0.0}], "records_per_sec": 200000, "type": "records", "values": {"src_ip": {"burst": {"keys": 10000, "label": "dst_subnet", "share": 0.05, "value": 999}, "keys": 1000000, "zipf": 0.0}}}, "http": {"labels": [{"cardinality": 4, "name": "region", "skew": 0.5}, {"cardinality": 25, "name": "service", "skew": 1.1}, {"cardinality": 25, "child_of": "service", "name": "endpoint", "skew": 1.1}, {"cardinality": 4, "name": "status", "skew": 2.0}], "records_per_sec": 100000, "type": "records", "values": {"latency": {"anomaly": {"factor": 10, "label": "endpoint"}, "pareto": 2.0, "scaled_by": "service"}, "user_id": {"keys": 1000000, "zipf": 0.8}}}}, "queries": [{"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job/keys/1s", "lookback_ms": 100, "metric": "data"}}, "interval_secs": 1, "promql": "sum by (job) (data)", "rqes": ["synthetic/templates=all/shared=1/sum_by_job/keys/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.5}}, "interval_secs": 1, "promql": "quantile by (job) (0.5, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.75}}, "interval_secs": 1, "promql": "quantile by (job) (0.75, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.9}}, "interval_secs": 1, "promql": "quantile by (job) (0.9, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.95}}, "interval_secs": 1, "promql": "quantile by (job) (0.95, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.99}}, "interval_secs": 1, "promql": "quantile by (job) (0.99, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m", "k": 32, "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[15m]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/15m", "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[15m])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m"]}, {"interval_secs": 60, "promql": "rate(data_total[15m])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/15m"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m", "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[15m]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[15m]) / quantile_over_time(0.5, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h", "k": 32, "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[1h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/1h", "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[1h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h"]}, {"interval_secs": 60, "promql": "rate(data_total[1h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/1h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h", "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[1h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[1h]) / quantile_over_time(0.5, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h", "k": 32, "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[6h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/6h", "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[6h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h"]}, {"interval_secs": 60, "promql": "rate(data_total[6h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/6h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h", "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[6h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[6h]) / quantile_over_time(0.5, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h", "k": 32, "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[24h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/24h", "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[24h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h"]}, {"interval_secs": 60, "promql": "rate(data_total[24h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/24h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h", "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[24h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[24h]) / quantile_over_time(0.5, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h"]}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m"], "sql": "SELECT dst_subnet, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_subnet", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m"], "sql": "SELECT dst_subnet, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_subnet", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m"], "sql": "SELECT dst_port, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_port", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m"], "sql": "SELECT dst_port, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_port", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m"], "sql": "SELECT dst_subnet, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_subnet, proto", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m"], "sql": "SELECT dst_subnet, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_subnet, proto", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m"], "sql": "SELECT dst_port, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_port, proto", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m"], "sql": "SELECT dst_port, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_port, proto", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m"], "sql": "SELECT region, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m"], "sql": "SELECT region, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m"], "sql": "SELECT service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m"], "sql": "SELECT service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m"], "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m"], "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m"], "sql": "SELECT endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m"], "sql": "SELECT endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m"], "sql": "SELECT status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m"], "sql": "SELECT status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m"], "sql": "SELECT region, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m"], "sql": "SELECT region, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m"], "sql": "SELECT region, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m"], "sql": "SELECT region, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m"], "sql": "SELECT service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m"], "sql": "SELECT service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m"], "sql": "SELECT endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m"], "sql": "SELECT endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m"], "sql": "SELECT region, service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m"], "sql": "SELECT region, service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m"], "sql": "SELECT region, service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m"], "sql": "SELECT region, service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m"], "sql": "SELECT region, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m"], "sql": "SELECT region, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m"], "sql": "SELECT service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m"], "sql": "SELECT service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m"], "sql": "SELECT region, service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m"], "sql": "SELECT region, service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m"], "sql": "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m"], "sql": "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m"], "sql": "SELECT service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m"], "sql": "SELECT service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m"], "sql": "SELECT endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m"], "sql": "SELECT endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m"], "sql": "SELECT region, service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m"], "sql": "SELECT region, service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m"], "sql": "SELECT region, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m"], "sql": "SELECT service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m"], "sql": "SELECT service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m"], "sql": "SELECT region, service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m"], "sql": "SELECT service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m"], "sql": "SELECT service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "user_id", "window_secs": 900}]}, "rqes": [{"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job/keys/1s", "instant": false, "interval_secs": 1, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "sum_by_job", "range": "1s", "stream": "job/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.5", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.75", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.9", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.95", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.99", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 900, "max_N": 90000000.0, "metric": "data", "min_N": 90000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "15m", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "15m", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "rate", "range": "15m", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000000.0, "metric": "data", "min_N": 9000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "15m", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 3600, "max_N": 360000000.0, "metric": "data", "min_N": 360000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "1h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "1h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "rate", "range": "1h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000000.0, "metric": "data", "min_N": 36000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "1h", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 21600, "max_N": 2160000000.0, "metric": "data", "min_N": 2160000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "6h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "6h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "rate", "range": "6h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000000.0, "metric": "data", "min_N": 216000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "6h", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 86400, "max_N": 8640000000.0, "metric": "data", "min_N": 8640000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "24h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "24h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "rate", "range": "24h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000000.0, "metric": "data", "min_N": 864000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "24h", "stream": "job/value"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 10228202.546091888, "covered_min_N": 3005126.2445390555, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 60000.0, "max_covered_share": 0.1704700424348648, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.05008543740898426, "queries_per_instance": 1, "query_id": "t11_distinct_src_by_dst_subnet", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 30684607.638275664, "covered_min_N": 9015378.733617166, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 180000.0, "max_covered_share": 0.1704700424348648, "metric": "flows", "min_N": 180000.0, "min_covered_share": 0.05008543740898426, "queries_per_instance": 1, "query_id": "t11_distinct_src_by_dst_subnet", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 13838389.036778929, "covered_min_N": 3702883.342543191, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 60000.0, "max_covered_share": 0.2306398172796488, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.061714722375719844, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 41515167.11033678, "covered_min_N": 11108650.027629571, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 180000.0, "max_covered_share": 0.2306398172796488, "metric": "flows", "min_N": 180000.0, "min_covered_share": 0.061714722375719844, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 3409400.848697296, "covered_min_N": 3409400.848697296, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 20000.0, "max_covered_share": 0.056823347478288266, "metric": "flows", "min_N": 20000.0, "min_covered_share": 0.056823347478288266, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_subnet,proto", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 10228202.546091888, "covered_min_N": 10228202.546091888, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 60000.0, "max_covered_share": 0.056823347478288266, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.056823347478288266, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_subnet,proto", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 4612796.345592976, "covered_min_N": 4612796.345592976, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 20000.0, "max_covered_share": 0.07687993909321626, "metric": "flows", "min_N": 20000.0, "min_covered_share": 0.07687993909321626, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port,proto", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 13838389.036778927, "covered_min_N": 13838389.036778927, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 60000.0, "max_covered_share": 0.07687993909321626, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.07687993909321626, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port,proto", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 10774093.281829244, "covered_min_N": 5387046.640914622, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 32322279.84548773, "covered_min_N": 16161139.922743864, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 21, "covered_max_N": 8952441.571673146, "covered_min_N": 314412.8382575546, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 1200000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 1200000.0, "min_covered_share": 0.010480427941918488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_service", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 21, "covered_max_N": 26857324.715019435, "covered_min_N": 943238.5147726639, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 3600000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 3600000.0, "min_covered_share": 0.010480427941918488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_service", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 23, "covered_max_N": 3215148.0197777487, "covered_min_N": 316062.76837714465, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.010535425612571488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region,service", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 23, "covered_max_N": 9645444.059333246, "covered_min_N": 948188.3051314339, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.010535425612571488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region,service", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2671540.3364740512, "covered_min_N": 2671540.3364740512, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 0.08905134454913503, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 8014621.009422153, "covered_min_N": 8014621.009422153, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 0.08905134454913503, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 21073170.731707316, "covered_min_N": 2341463.414634146, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.7024390243902439, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.07804878048780486, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 63219512.195121944, "covered_min_N": 7024390.243902437, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.7024390243902439, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.07804878048780486, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 959447.493044697, "covered_min_N": 959447.493044697, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2878342.479134091, "covered_min_N": 2878342.479134091, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 5, "covered_max_N": 7568143.5735776145, "covered_min_N": 1892035.8933944036, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 16, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 1875000.0, "max_covered_share": 0.25227145245258714, "metric": "http", "min_N": 1875000.0, "min_covered_share": 0.06306786311314679, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 5, "covered_max_N": 22704430.72073284, "covered_min_N": 5676107.68018321, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 16, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 5625000.0, "max_covered_share": 0.25227145245258714, "metric": "http", "min_N": 5625000.0, "min_covered_share": 0.06306786311314679, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 6288544.323516745, "covered_min_N": 1572136.0808791863, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.20961814411722485, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.05240453602930621, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 18865632.970550235, "covered_min_N": 4716408.242637559, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.20961814411722485, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.05240453602930621, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 1876594.1875720161, "covered_min_N": 1876594.1875720161, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 5629782.562716048, "covered_min_N": 5629782.562716048, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 959447.493044697, "covered_min_N": 959447.493044697, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2878342.479134091, "covered_min_N": 2878342.479134091, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 2, "covered_max_N": 2258445.438282906, "covered_min_N": 1596962.084349668, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 400, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 75000.0, "max_covered_share": 0.07528151460943021, "metric": "http", "min_N": 75000.0, "min_covered_share": 0.053232069478322264, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 2, "covered_max_N": 6775336.314848719, "covered_min_N": 4790886.253049004, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 400, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 225000.0, "max_covered_share": 0.07528151460943021, "metric": "http", "min_N": 225000.0, "min_covered_share": 0.053232069478322264, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 673953.3609679822, "covered_min_N": 673953.3609679822, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 3000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 3000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2021860.0829039467, "covered_min_N": 2021860.0829039467, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 9000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 9000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 1876594.1875720161, "covered_min_N": 1876594.1875720161, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 5629782.562716048, "covered_min_N": 5629782.562716048, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 673953.3609679822, "covered_min_N": 673953.3609679822, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 3000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 3000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2021860.0829039467, "covered_min_N": 2021860.0829039467, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 9000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 9000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "quantile", "covered_groups": 4, "covered_max_N": 10774093.281829244, "covered_min_N": 5387046.640914622, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": null}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 4, "covered_max_N": 32322279.84548773, "covered_min_N": 16161139.922743864, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": null}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 25, "covered_max_N": 8952441.571673146, "covered_min_N": 259541.9036564592, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": null}, "lookback_secs": 300, "max_N": 1200000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 1200000.0, "min_covered_share": 0.00865139678854864, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 25, "covered_max_N": 26857324.715019435, "covered_min_N": 778625.7109693777, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": null}, "lookback_secs": 900, "max_N": 3600000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 3600000.0, "min_covered_share": 0.00865139678854864, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_endpoint", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 100, "covered_max_N": 3215148.0197777487, "covered_min_N": 46605.478008970495, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": null}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.00155351593363235, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 100, "covered_max_N": 9645444.059333246, "covered_min_N": 139816.4340269115, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": null}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.00155351593363235, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 2500, "covered_max_N": 2878342.479134091, "covered_min_N": 1209.6074483267448, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": null}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 1.3440082759186053e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service,endpoint", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 2500, "covered_max_N": 2878342.479134091, "covered_min_N": 1209.6074483267448, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": null}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 1.3440082759186053e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t17_distinct_users_by_service,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t17_distinct_users_by_service,endpoint", "range": "15m", "stream": "http/user_id"}], "schemas": {"flows": {"labels": [{"cardinality": 1000, "name": "dst_subnet", "skew": 1.1}, {"cardinality": 1000, "name": "dst_port", "skew": 1.2}, {"cardinality": 3, "name": "proto", "skew": 0.0}], "rate": 200000, "values": {"src_ip": {"burst": {"keys": 10000, "label": "dst_subnet", "share": 0.05, "value": 999}, "keys": 1000000, "zipf": 0.0}}}, "http": {"labels": [{"cardinality": 4, "name": "region", "skew": 0.5}, {"cardinality": 25, "name": "service", "skew": 1.1}, {"cardinality": 25, "child_of": "service", "name": "endpoint", "skew": 1.1}, {"cardinality": 4, "name": "status", "skew": 2.0}], "rate": 100000, "values": {"latency": {"anomaly": {"factor": 10, "label": "endpoint"}, "pareto": 2.0, "scaled_by": "service"}, "user_id": {"keys": 1000000, "zipf": 0.8}}}}}]} diff --git a/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..2d3a7bf272244300a1ae1a334b17449b4fbe8719 GIT binary patch literal 39806 zcmeFYWmg=HqIHYAyK8WF_W;2|aCe8s-QC??6A13^?iwVxySwvt_TJC7_ndq054hhL zJqF!H^`L6ioNEzB!oWy&Oj&|`IJyIz%uSpuj2N6uZ0xKJoJ<@ItgRUw%?%t(j9D0g zuK*kwEgfwEAg3KVwuC&jl}~DLof(c2*%M(T zIRm5j7iPzi*#tG|NV>$_vR+}pZOZBF?3jMP3?=TAZ^!e_`Hhg!YRA*r{KwT?8I!Ni z>+|{j$&22{``!J60`a<6$J;frt)SQAORB4_;Kuts@#RBRZh)ur&0N{kN9!Bfhe-Eh zmv6_*_1)ctYuB1j+6TIC`}^HoQ||kV!pDy9+tkOI@B7BbN!QzL*T?P9$%d_O*T=V4 zJ=ffiS6eQjx9j)5(3K6J=e?=iPVbgCA>Rz&g(1C<*CQdHXS;{1v$NDHp||JriI-R| zA@9yNr-!GLGus=5=-k7dSEG;I50AU42jWzP*ORstuH0?C+^)Bmvqe|eF|`dZ_Zy4% zQR1eHjn0>6lc&hs&I!Gimp$U9E+6lYi3aU9b&;_jR@nkC*q3MV=K6?;dtW1qf>AmkK#u9zhkk%}avc?%m4g=!vZ&n^@4!9p} zo{DeG+n-)dpCmu-J_yOkXaYLDUf%AQd^_KV)|h-h4$gA*-jVe7^|r@^Y`fl{0q@AG z?_G*zT?1VWUQc~58vs4umbbgh-}|}Gk_3-!LNT3KzK>YZ|jxEtYRRTpQjqE(tLtv1?+GMQsdkThWPf17t**P{#NyrPw zFX3?#!?rK{sL{+S>}l3cS@iVXpr6WHvd@wq9A9T6As^m`^n`{s&N#)1FI#RwS&umJ z*3fMYdAfL7nkLpBld`e!CO&Tt3%QxMQR2F6=Wpl)2P3h@i$!?g+Lre;xVL-#^1^!7 z!H13}rRVDz)F0R4JBE}yZ`KdO9I(cOeyemD>A->&=NLcCRq1`Z*Wg&a$k~WC%%SPe zkm*5R-sIn0Zp6$W*ne8fTj%*=p>!SK?8BAg7_%)8%FM5BrmZN;Yd`5lIQbh^&K_zA z7j68t)0^Q&QTJp#x||IZEgXeh>!1-kvKyV3?){~%@zWsfdU_f5YT3+J<^HI14UTdB zT$O%=_w>-KsW3pmAwtLJ3ox=1s?}Okr7?hNoa2 zpU;<2(;BIgSJ0Z)%+eZRvFYdxo&lDpN+*(+AV5Q1CSAUW>o|Ac4R`qke_U=}ufd6^o@due-t9&A*eV^{f zhF10(7mKG{^)7VZvW}0&-^;0|B1kvi4*6mH#?_1q97a2 z4aZ2%0c0i8!0S(~2{_jE<>qB4a}p2e4F`f_xp2oF=J6oE-KI>{eSG=uTI*|^z~sQ! zI9otU7!KXi9EWK-2r+vlK}Rtda`Hkb%r=i$6izSyXNZpyz}q9F)$mX?7&Mj~3_wMf zdjw^lBK7g<{_LeKb`1LCdHlsucO~`dA==2{ zakGsqyUOHKfU7P}Jkn&)q~2$04z#b{)L>!g+va0l!h`+!Fj<^vO}QU$f<7_1iLPk; z+&cj%>F%0e=4tFdEbu47ar_3|F0u;KkW;Wt&!7UR^ihl>OnlNYos0Vu<8dtIi^3ys z(&dO@kbicGYQ<^7gn=-1b1+ zMO2(L2G4kfA$y5K?yD=Cw__;3Q4uxypjVJQrobvhwL?Z}uiZ%HEs3rLhAm3Mr;ueR z6LixDRv0ygNsqiBv2$tfsY}#Mr%jZF4|`bmwdsqhANRmiJq9WMK{8j6oYAG_CSlp89c8zGduSDlIvk!!U8+RvL_k z9Iz0~#Ll~~?)8Ag9E#TrvAnpH)Ng16U3~fID>|RtUiGj-h=bofFu$eC@O{1C>Wj@_ zdYKGH29fQa9K=1Gxc)>SyX+=#OLuvqK-&KPzC#=iv~Bhx$qhcl$-sQaPS0F~QD zrWVmpJ{2FF5P{l z)oeN;q)4ok!d^6VY<1U@;0yvr1+|qL*SfbNxX;{WQO;uT#R%F){Iq zq0@3P=N;%&MjiegxX!cxL5Nx22%CJ!vG^%BDA+hiwFxwn-i$AE5%g6slh{C)1o17fb3;^2PKiy^1*0 z>#4!q;__unj3CiAS66Z%weR>Q^zYe$08zu^7ejQWQq3BMFHh+34Lq`B^ExJS@c-Gq={)S)TGKW4pOOt$`)s371>OP1;YIMkXu9D5Bv+a z?};gSt4qfwQg@~JLw~0Fq^-QJ-y-J~4Icx0#?u{wK)2OmW}2&LRq%0 zVSdM*kweMO$4;}*srBh)hz#5pA#`G=`ra{egOma7G>z0}yfpg&PGALvm+C4;$bE>8 zby;%2B|r7sl#2GjU$K+2wS{&2X4yQMB{W_YvrgS2nI$s4l5xj{>&$F^U)nmrPpr-K zN5z!EkKCUf9c>m_1>C_pqc^6l@dm%Yh{y|=PICX| zk;Y%Wg@7|3b3PKNvbh^<$KP3S!fu+{`HnAs#%bAf#AG=|!zVp1dTk)bgr1Qac1RH{ zP!x#J|FoQ+S#IkDh2B-d^b6B138qmD;xY>t3PF!g;`Y4+P&{3w^o5PmcCgS}x+D&Y zijIKt!MJDP)6F`2%(jg>DCZG$uv~e*Kg6CQSuSb*Aet0EEo!KTq>(Lp!;G4A2jp=Bdk21H3+O5P^ zt(N5Kj>m$mxIakgaD%cZU$Y!BFR$(Kh@`NN9ctug!r6Fn3E|`3&q0M&+2!5Q_F>mFRMzY=d{R0(JG1zV-Oy_Lr1lfv%tJqfNT!2CAD{OoYmqw%(xA z^SIx&E|gI)03B+g%8hsOkWw=Xo)q=cjq9^`=}bRK23bQ0YCD3)!5n0E8_3yc|CY)L>xM8BA9f^BTV=nn z80`?tcidD(>Pz33i@t`JPV=FuBA#2#4q?u+IF}bCm|}M=CXF`Bu(ERd=BVQ^fw;-C zEII^&fso8P(Uwn&J8z#*GI{!;ZK3oH#;`-D%6Y0iW5isAts8pE`XF_sQ*B2Z#86;o z(sAI(qYgo%ZHj9m^+`K!Q>^X^Zc_z0}lWCe|f z=@Yu&chP1<`a3NUy~}kIW_fC^)piuJ712jS8Yny2;-HK+c@LkoGQp-3Jf=Tz7!5GvEnHxYi$7T=-OsnM8kRm z9m&?A8F#<4F_$~S?TVJlY2YREv(QeeI?bKQ${3JUBi@L&&@!|}>LTsHnVPFp3Nl$Bi|XPffDL%%0# zBiB$1WzCB!0aP(A+MLg6j6`D=s4pU~FsD=^g1O*x21TKjN2am|_=^)!K78{yVZOp5 zY+lACBW`W=0HehFx@s@lR41-wpzr*4=xlV%H(n5}ZZW@Cs1AkYP9S-jtg6Fg8$^fV znMR+D;1`jTCPB^BcdmO&%dF^)jCb0bkB>Q~Vk@qxdkCPUan{T`5Y=tGeI>gi!OL753YSF7 z8x4$;AFS)YlavXq`MfVStxWmmK&xhspbLY*hfN03!#;tO>%J|3s@oJ!V{~!a(hVpU zeg}U;mMf+Cyrrn$nZy7%@|a6fx4}^N#{Gerq}HJztgQX#3Slw-0#OfFwhmvWs(Kd* z*DFoFetUAz{fXo-w`+P+Rf|}wF6x)}iBVg<9wDO2L<=t*<-@x$HWeOIHpEEHBScST zLTL*|2FDXt7kr}o>`EfKVLhF)GY{e$GoEx48KqvS6_eqisu+=E9{`-?>C^+7j5D%- zLQ={?f}c5pD<%+JCaLM#bWi0V@gQKr+eslSeD8Ik2&2*?Qr@VtGHVF2et2@a-M6Pd1^NPSD#TS!5my|Oh{WN~O(0-qn4+$q6MyUJL#G|sxB zx1&Q@od$XweB*gwk_%OE^?Vk3GqfN_dcHmSG+72Su69?%jvSct6l~^I^Zb)3OY80$ zc6b?9z2buYe(H$ENc2rc zBu?u*v-{kSj*vH~f*^y>)>z@ieD^=w)ab7q10gbyMKu%KF87O64^}(}ojwS%r7VoT zg=u8Z3$5%b@6RZ(5{sK%ys_P}hgRYRu+u^`< z&|-b%!G9YKGHp?pu%$D9UOQdmEHM)in#&!DQr?t!KZK2Q>#LUQujAvI%+;52-BYXr zRYw#3QU#yy+AIz-24!VKHnlp?z)%BCWV2T?0aZ~+)Nu-h>1G8JSVh<;D)uESTwO;X zU;vrM_U&u&5&Zbo06L<(v|v%2u`uArM3C!O?NmkI#RD_K%hul_7G8LA{n(? z(47<^(a1;}g{%&ustm7%2c%m&{R1C?@`ClNzQ_-CHsl^HE~mOpU#lU&7eL%7%NY^V z?`DzN4qVoN#I~E@5dn=>j*$+o=p8q$#k;t1*kWcb(`sPM5g74nR42jiQ+G766w8AL z1VUYIiozHlP%~op?lCXi?I99+Y>3jI9B*r`Wdz^m8;|#blIM|(nV|+TTl^cAOcM&n#dw;_v%9Sh5Scdz zvD-wfZIk()hPjuIj@BU?et8Z%Wv?2=zy5NdD+IaBD5E;t?F*1@pNx}8XgV@{Pa97M z2e4yYxB4#w5H{2TV6=s-_;B@3{l_`!?pj48wwS-YGoqc`ZAC~h`qvKgXSTQaZhAQ! zMS4y+*DNZo!mA^HiGs&Jdbi(eB}paIbG$pe!@krmrx1I+RE}P!KUArd?c@~fSYz#P zARDm$c#AKc$%JklO-7B_4=~DgWQ2{4t-@`07tYg82ZtM3vo_z>2V1^CR*qhi{W^Nw zf-UP|4~`mzdA4PLm)$3Kg3+$BcS}Hk-q#{uy&2N2w>a4kG7=hhdRssJ8;iDS3K!qx zUJRqX9pdyj7}+4KH#L1!TOchybQHH^xFgyhF>YP6e*Nin8^gL1p(loQLKLF7bO)QEU zz!_P+`&wMpUFe*ZEA#I$!9q1dZ1g@2tvsA3T^&NjyCsiRgGZP>1y`o7=1t>7d;F zG%`|r6ZjQQIfy3R2Mw+(*;Pr1DWYZ;)~NI$m|J7AfaA3V`${;We>OY3u<$vNT&+gt zcEN|Pa5{x2tWOLt-@4q)Tg16r4fSX283Z_Z?A+$rxa=AeXN}aoF5X6nkXGKsNK1Ty z`Bgc*)HeEe{)cRAt-=D2t7S4SItVcM@m9X99WIAr*Sw5A@T}Xtgg9OwP%{UbI+k2A zRMqbD@^rpZOnvQzemqa!?)aV$|8Y4#HS$(M$+H*>_q=a@VgAs7Sfpdk{0S*UXpT9X zcL-oh938C`IE65X0D0!Fi(%+=k%qZKQe!fhzn(-ra8oHrO7axupZpXu=c-=CqRum1 zN7_*a?b$j=kG?4*Y>WqXwDze28MQw>ZC&o#`2#fz{-{H2-R74CGgC{ym|o&uAX9}} z;z@pbIoo|2`DW6`n)x5jX0VkCk_vhmoZoC4S!+OVcfDC!kJN=%%UwJX{zCf1lU@pi zFofgqJ+2rrSaethx{Y->^1eGYHRAL^LhUA&Uyd>Lz;kdUZIflv-(APQk+zjY#J5S%N&rd*;UH~KdV)nHlqQaG0Sl)A!~KHY&Fsg+;v8C(`&ew&`de8 zAd8QMNtZ?lnb=DxU~GnY3+o>kKdCOixWt1d->*mZOM4O*I&d~}XZyWFsf%$na(4*p zI?qpoX;+nnKI2UGaF41;j`w^ro`LQ=)SP5w&@#ivd8VmECw^$I-dmmB=756sFn@Pk zlU)KAX~lLdIjPXm_GHk1OYfW9JoS`b;@&H+)2Jn{KvWXtjl7AD)9Rf_r)k+#_y>uy z^9HdOyKn2KMKuGN11(q@iu22~z9b^`%OAoFgPTB4h=XQBKj0pjJ-=5z$9z35!Z(ItN)Wh}->+3Hc zZ_n&p>E63O1>pn=4ihZdZhX{!VILSdmMzz1;gL{agthV8pclHiGlHP+!ej7G7V-+) zxjvgaDS39d^?gUUIPYBb$qFQwu@Y_vdA9j-x@wSlfK3_`H1*qWR?Lc6i?h%J2j4wTU2>o!Q&cnHCvNp2X zK@b7kO84Ia4|JL~iRJ1AkS_qH`BmWVSjMz+jpv|d-r|1lixEt|0%#GIX)}UA#5>x9vf*k{srO;Cqk!{HglO@o1-{6(4Ad`ewjEPxk*7z+<3k!N8 zkoVABUfMJ9fRG*eFfUuy7%>zYI=lNAV3MbRGG3Uo&Mjb;Noc$V-SM1}cU_6A^q!%N zk<3yS3uH{(e;8BhAI8M?FUGY0y<%$X`ya-X`4?l-{ll1Ga_^7SP;>PuMrRZ^Av;~k z^Md{cnF2(%i6`l7?B>w3f*3**xvOlP)16x$F}0MC)jj-xSXbt8Vb zm#q2ju%Gq5Es61ROUxjnwT?VXm@!{PK8x!3ONQ(7eFm9IH|V}>F#3EEd+ZYNl~3Hr z=vtS5xqArZx5JMucTSh7(|UpB1S&0Pcu)D2XMcR*EELJHU2vj9M%e9ut0O;g&Oylh zr)-HR9W@TS-09U9Jh45sn(J+0Y0F;u%@Uo2H<*h8|pphYzOY|QL>=s6V*@2%Q6 z*WW#|_F(5Qv`S+>;*1|zl*s5%ZsTU{t0RM_#U}A<@N@jDw3iRn8N-t&wl*KEud4<4 z{x*E;UkGZ*MN}nv$R2%Z&EuS+b~P>s>_uz(yCwf<0Am_U`M@;wm{a_UW@OJ*qA7Li zs*KWA^r{qFlGfw~c6;fveuBd5@1vsyNDli6@U|G^pCbwkT&!@y7giTmVO@&4-W5SW zUd5p&WhFH1v!-IV{YLWQGU}!_*v7gpY0$m`&nt+_5?kcbG+Lmn`od`u{|MSFC?ubM zVz$@emuhf^%9{;~k&`2*RjmLUSb%Wrc)Sg%Gevy(eXNlB#(H_SzlY#TQp9~1)}8e2 zJLfQq%4)vhz>O7wQm?%cts=LZCDeVyBclS%KI!P{QW1ctV3h`-Wxo{CC7!UB|mJ++~ssX$S3upQyr#xB9ADPLSi{D6~pfS{S(Ibvjbo4Ugi0BDa$c zG|lf?wns7Jsmdru>0)8zBmla_=PJ2|dqH=ex4@2?-)C~F>c%`FbacNwnu6rh7uq<+J9>Jh``Pw9&JATEtk$l8qU7F z>7?O2U5yR5!V>CnGyUwt(TgNtywCwWoNi-K8%?pMzjL9b36_o73 z+zDxww$Fy!wy+Uv-mabS07;wxFxDc5IJDw;{-iKU;9e~jZ9qvmBhgB2taGL z1&u6Zqd_OV+IKlMmaMy&5bYJW1?GZgEnDc2<^0IEpITw&9prXR{wfyy7NM{!yNgV! z`&uF!`3-~mCN_176=k7S_-fk*y=js&+~_N<9w=iaV2j9G(FmrpUHCBmh=s@@Gnk1@ zD9ah8CvZkb%Ov_uHU2Db)t|QQyDm#-_w~V2&(*wX84#(?xD7IWL}&;YsPJ|JGg*ul zqkq~4f~+s76DGFxr-zSt81Mg9FzQ`5a%Dd`)`v<$oj;r3Yn|xerE94X{dDqdfIm3f zAkqyG3yo8ZaK1{AV6GnS+trn}C8S(009WG^+wD_RQ|SH-9>FJn=+bCThfq;T+yUfb zp&Iq5bTzGB7-QXJu{&WMXk{mR(RPrA$cf(*No`SnNJJjKmLDHmEcXu6A}1Cz&=$99 zZs%P!5K+l!L8>guES$sj6H%Gf1(cQ*qqh)EoIpQIFcx%`d@tta_la1Wqzx)v?*rZF zEk!blSjqdi`LPr%Lyd&9`}p%G089V4bE$(s6_L598PU)c{uPBW4?0}j_4SuB7jkE! ztd0-WblNdeq8n2;l1d>GCX}j@bNt#cS!_b+91jShmA0BH_2(Pnl6eeDq;36k9ULUwk5Iw+n++0-vvyohHP`u2rFwX}U?9KD% zLksle&Kqw_jD+fvg({AfKLSS1gr*0=2W8kq9Owl_E`L;8GK!nYbq-(j8t2qM>;~bS zMX2h?iwQ-A%vOYrJRC3ofi!VLGu11?vVxuK{$Uej_BR&aGH8&awxf48yV3S4+BM?m zpLSdHeTmo5&49%X4jS?D{j5!tR815VN@9u~;ZvJeJ*&v*y}lf=P+?&hw(FN=w;@>ozQpGvHSr_^WrVc| zw;q9+eX@J}2&6&98V&lK9202(1RQ&fW7LfcgV?2enXSB+Gp7}pL(jc;-0XP~N(XP5 zcSsyBVqm(RZX_<#my>&3<^iF^CR2Vljz{aZDMt4XadWczn{Y6;E?s1+aHgu@eF>~* z6M4F4Wq7$)r#2J0T76zqwUuvp3Gs^TNj~DT=Q)eo{nHg8lZb@|UH;StIH;N8+ebOx z$Lp9fud5{f`*U#%yNvBTGxJX_xh!h}l4jqQ(=fo)%$%oE_TbhWVVxR`wZPhTI}t$^ z5VWc+9c1Ek7I5I` zw`%>Q=4kRDCf+ zdpY@4!9vd-)SsR=KTody8h5=AyAf?}axurGT7YOB;8^PxnsA;Ee8aNH%=PPF{@=}y zIxbh?-xw)xC93vEG4ylWQ9?#)SQugAH3Ak@qE`tJ^edx4kSL$P`akkfn`opx-R?sk^1tz@!2oPNRT)lkD;Tt@1!rTM|377_)K z+J3g_#(UrC49-P0+v;H>vypSEu%pXGBCoQ4OVmZ`CiA`yy*SzP)V(mTRHOeanof=5 zc@Y?14WOMIZmMLgy|5Km!?D-V+eT7z`p&XAU#5F~RMn`$L{(MpLK4WrTse$bKERns zU_vz7V`2lP%V+Bc%GJ4qXy{qCIwCnZBd_qlSn_>c?8nJ2QRHrMiWV-AE@}%{P!VG( zA3BGyC$!69-wYW~64({ymnJI8#XnF};-Wur$t+>{}XI{H0`BHfkQ;h+h~ zq`!|W^6MCs0l93Pqo<+NZtkQ@^YFyID8Wo@X)PYi&_ffPF59D(*PkIMx#>i6z+1L|xGUIR5W;2mby4Pq zoDH$tS~lx_6=nHbJPX>Ab#z7BcKgx7hiJh+r>)gD_hWa2gUfHo9G(#TE3@(_`-N$& zv9j5lG-lksFr9&=jjYgQ0L}$HoSg-D2Qqz& zWYNv|VqEht_(XwDr+sZY_zu)ZO=WRgD8>KTEnc~nsVr5UO02+=!XZzFyAe0XrB3?w zPSQ+!MEWF|X{P^EE{;i5f0qL6bLeH3&5oDhAbU!#RN4qz<0!6HG(m`+O%J^PVhHM6 zM{QnlD`(HC?+dcIKEbBDsmQx1(lV;9ZT#)FG~c%vO?Sm?FkWb$1skp-5N4w6e-e5r z0bu*Fff&>s@`wLKjA+4x%FR0m*ErdrRIk|erunF&%_yqlA*!WvLW{dejsyB&$p81a z<|ELOj{6^R?e=8WWlLEHS8+{O=lo}TCli+{jF*m(b)R}GC(M#f7^-wlEG!!*+oavl4-cbrB0ckh^XN2L0QmGPH%tSx7n#ttM9 z;+UTZP+Y@q!~eSQ_Y8*)cLnAVr|$H}NZxGnk0|fE&;EOa8&v=A2p9BMgfsZ>2nSmJ z_Xy|pw+Q$6SAcB=DTQ7n(IE&JnIKe0<|BXd zkg733liw7Sfdehzx2482bU@_aI*oiV&}~M$uBRrU7PuXs>DOy#M>#z4qewI5JV9KVlj zK$4)qn6Kto(4<(6ZBRvgFsjgMHiJXwfyKmf<49OSsTo@IQ(Nwh4k z5Wx99rtgA&`;e=O|z!hNw$$dRdOr=^D%mA?P`ro9=f-zD9?go{G@s<})7<|~Qo zmA=AanhBWKbhE?lexH5fc5@ZN(1h4$J4P&5XH)jHRPF8jv61VZ)P=JDuKF_gPhPHh zF_5FS-}E{vMGd4+KaIh3fOfU8a^t3CLQNqKcxNUd8u=$en*nVq+Z6myUMdIXlonRQ zmS0~kArJ!ixgs?7#n+_bA&0cRRdEti@;GDRx`V9ayhyZ1L)fYDxWpzCw~Yc_496s8 zRqm>qB2NCLL_AIFx&jkV}aMc~92 z2Vc>P1*_+zQZimYH?D=qfqwD6d&6&Y58S+7F3oy*lj_=c8^GYq<{Ole3$aS8-Fh9K4191pP-ylSjhepPkV z{r!*&YeF=km(m8AnQ|HbsL^_s{IfrjbEwQ7%Iy3C21CyY4+^$Zqa%5YikIM+a*n_Tz@uv08k3GLC|CTfT`d0ms z%Y@X)`$5mT*XhMrP&?tD3kz8vb%+Z_-iN7?` zbbw^WTl=y*7hX4;0iVzHk%P#*S!ukD*oa-2H1(1*|9%O)K}##Ojo}c%C7(77mID{y;XEzT4%bl~(?2 zxbyx=^(2a&Lkdrv38~>obkStA#=3ZI2i9W?*skNU8K!-gFHnZym;+@<;J(4XuEp(t zuEmnb6M-J)*aEQ2k9C3RQ{a^qVMOTw{! zNNx8*GLVBd$Kig?Cai29`>cm}t37k&?p;5c)+bdKq0tWfVL(nT)xIrTx9vKi4o;}` z$SlJ4!UeCGda^uD9Zg-QaAJ`TzU9a}CYY`-Ua5k3$M{75V-xb-sc@kqhM<3A4Gt>Y zm%FOIM@|6+vQc|y{bpJ9V!n+9Am?iT!tdOw_zMu|p7~yIpH*AfZ_Qc{(JpJQ3-&z~ z<30|F)$h53ANkN6o=XC3}@!0ur=GTMgDONG|_cuMle{l zUSIn`_UBR|!;@xGgyV7yxbwr&v>;H2kh030wwC(yQ6{fZUC@5r+!1_A7HeWr^VT|r zI!6jli>M_2-g zFK(7G)8VsmKfmbm8g0loma~;2@ongZ=U48%irnM}F!PWG$zM*CUjZJRHa&vD@DbZn zQ$HiIZv}lpQW;~vjn`SophG(b5kZQ_Q|$)drY@5V3EPKCV@y@q%fb#-{QTUq`4e}E zZmo?!FD;bX(MYV-J&bt2jRcV{2^hvq%owd1-1{9;*RW<*LDd42q$oO7o#6+a!*Zc)Nu5qnz))A>(B zZu(C{j%D#rL#}20XP#33pN5>pcf^qsGYCphk&AAYAMO0DwowF&0k2CqPt#&{?2Q}; z$6R3}p~!2u5JX4Ub>wXz5;+6SloyRd&i#oM(nEWygC@fZd+fJ;=G1EuY3<^3x_@vQ ze@w_4Q~}3M*b+S&SJ!pHV{E!&yFCHHQc~UFMEAne?o~uXXFpfluLh{u&kM^fw)J(! za$rMtYusW$rZ~;3k0?(QZC3!C#$vIJ$Yo)VU-@aAMq|C-eMHS@V z(vkI0lZ5~{WY_C0s{V}{#(1-8NN*oN!S=InoDJ)^IRZP$X99&PMRz9ezbT7ZV|@~y zzVdorJ%Eay+)<0=zcUX6qAuh7yJ)( zHpPF`+3*rsEA%j^!+;;dRuP_LsS{->H`>66+cpA^O`)YXtadlTl$31a?Y>Ic<{YeL z8>#CO2-nUL)U;9#U!P?j{`d>$#-Kp_-2eeZQ^Wz;VBS-O*9Cc(@;ajI^51zj@f3X``L5e9h zx>yN9T;+ZNl>d7Zj6R};k-jF+urf08RD@;mJB8_hHg=WANNCrxYDZR9E!3_uj@mBN zf?=jq3@h^9Ca(odCuTbWz^ewo|6a8vQ6Y-%(8IM={=B-R=;Tc*Gs#(pC0f0#{=~zw zjm1k37RTOi$7o z(_07SJItSA7F<_n928vK>X^jUqLu>_PBF+S9IdCgKK4{UND;~@9;@eoekzj}3|wy| z%J^_ZZ)Hd?)E8U&Qs&bM=2PlA6#^GLq?d_5-caa}9)*v$YlJ6E6Vd=wR~Q|dt` zN!S{K3l;}|%#S;PB5vwJh>OCIulUO;yg{`@`Vt=|1dvQ6qa;WCQ2_M!y08@vHBIaq z`!cj)HRGiHsBt&*E*)L0TYmaLDCG=iW5psApW0F zs`GD9it-PXa{n(VrSNwsh3vXL+W4PP%JC1BBJZXo4*h?JQkDM=r9%D&rR4vDQUU)9 zl+rhtTKWr084G6YuQxD$xg=d8>8y8NA4p#YDP3q*Z`|5YH?{t<|5dSct(m}KuC z7t$Ro=x-%9trRgka6gM5X-VLG9-MB7;Ess!h9-|DGa57J-`l8AHR!6vx1kE6=I5r` z_TCiL)8WqSS-}L{*yr@Y-3oStRtvkt)6N@uq;zLO^uS=&~uep&GItS7m=+5KC-< zJtd{f9Z8BJWF>Jge$Paj4;Km;z63EsAswYHDG{g2BIe%}-wv$Nvw_rSNrNMHZJlKT zoGn6@wrKJw_N@AyAe(2%?{@WB=5fe?!6+TBADxI`ZdxCAy8u&TO3IV?bTgr7SkKtC ztty$7U?pdh&(5CCEydT*ovZ4k*UoR%h<|sKEE3dDrg+;<)&mIjpTwerF{I9Mei#0i zPu$V@51%O8@-LqltGLBWnrIVch^xM9Pf$ZVT5fW7t+Rm}`nM*$O(M3Zfn|NzWNc$) z&JPBlmt%HcE?})Fqd>ybJF%~=CoH6Nk$724D__G56Of-&&2fHPWmqh9e-qvalexui zxWxz^Qm9$c2sU5v)ZfywTQHEd$G`PmhpFon)fNzbz^52`R2gWeA}|3HK_UubM2@n=d$j;+8kULZ-h^d zw#|>`1uUnVn3>t!T;hKL7aTKK`&T}q)bkC6+uRD^E$$=x#U#KNOS{7@1KsYF$nPETA^5wR*&S0E0!gj*b}k}#T!AHBg-*f-#Q)rgNq%OHbpf50?S zET_?G63Im96UROvV6uS%mf(T=@aG^ikbDA8hQ}qCYf6cqgiQq-%Ab+*S zyCMOLOc(*0Tn_)hD<|zPKSlocare#G>%y%es#~wo>96Gk9GfoL6Mf)PP}T}K`U8H< zcz3-jDjRc!)}z^HE)k~?UH!r0;b+(;df<4@d3q)*>7Ft;Z6TWAA6Xt`D8qknZ|y5I zQY*wl>&R2|Two*B+>+)T@{(bQ6G&`t z-hnt})Wz@#(W~A6vs?;PAzx7uqK#0zO_P{1h@75x-!Nc3zPY`TcFo6kwu(oWC0Ont z=JH$O7TynQw@%#L--5rc^+6<+gYb1XPFFI^{2P#y-$SL=vIm|d;7)2UwC}%1XZmGF zvrT92NpYtsN4$1c%su4XgOjoJg}P@$K&&CaHjhAEpn@$uNx46LwUo_!&U4~d+%9JA ze~A5+MXf$4~Wv6v<=xWd2_km1A)g%=9jBL^{k4ACDPI z6v)EqC)bH5qsP6Wb(Rt#Oh@s1wV+-QYHg?E}NL`Sa=S`e8d7NtaLN%hkW;$rjE}$E&|m z+Vy@rJb{4sFT&PafSXW8@(7BcG12&kbD`_X=5s$jd#m>q`>jQ96AcoA4;t)J^Bx`@ zex+^tr`mRt2oFF$-W|In`|-63`-Qu?f#V#L1qsCl>$8J=DccH{X-?sS-Ou*HAf`Hw zhKlyMIF>S@AJ|@9kwWT@zih$ninU4uWdt|5uLdo8{iY5rN#yU7^0A&EY7{J>myq?J zib@G_$hS&p@=en6>epbkA0p!qBV(pUYd&M#i(7+Tb8G=HYuZMs+`*KtUO&dFz|DD=xAr9KyUg!Se7|4>i447|gAIaiHzy zVXVW<;Clw(uT%Yh%4G;cLdSXf?%rhExOO;_$nRT;><2 z{=shR_Fq@gMAt$7!~S{I#Q%49R2D*jireVQ zRB)xV1<~UD;8})JtrXy@@hA3S12OCY>QL!58-T_`?kNczL<>@G4Kq~=6*XT&OjqyefI7fot@ihT9akS&`YpMA(TLg9ty0q-xQ=sDrSAsY z&akA@d&WYMbOOti7o9F8*babIrqB1uYLNc*UNeuR^viAim2S^i&0TofB#f;5R0-2k zsfMV^#RH$qWnkAK+VJeYimwN^jm3_BY<^Kb`c-g8{O^y1dw5i5%9=Y5T(NZ9EgUJd z7=m2TUi+SDQdrmMCbrgsJZLIs$+eZyr=Nyg!)VTHDB44kP^I4GLaMV$%Rg8>K`7?{V*yF|yC#Q(A_w7Iz=9t)*GPxOjh&&UDkiat z#kigR&@<2fYcePYF_FFM9>F}t9(*fT>J7rh37(TBOjMtSmxl`62r!cO@NwS6=c|Hy zsO7jSh}tsOx0+mVK#wu3WC;QjmFDj$14$ELqOv$-(()ffMdp7;RBrx*sLUNr)VvZE z#Q!2HPJf8X(BDL5@()q@rJ8>-TWO=V}+*vJXP%MV z3x+lii+k6g@_NG18O(RkcOr!sK46KgTWjtsX86`Fn#);8OG?ftKjI|lV`GRn6Hwc& zGyrQh-zT$1gI%;s=kLN9OjNIXLBmWtw^!||VDaHDcZPoMk^RLwNi{=CWW<{>r}KVl z$!rJv33)5>{C%3g)?wDrB()wdq!kUJ5oenlx3NBxGOR$M*7>p{AGDo|@B&1nPu<~; zYO1G>L1iUN$^QYX{I4rP79jtsVjhf2m5Xl`S3+WCR6?c9@*wlNd+=6?ZQ>2o?qwlPb6pfJ(FP2r&hwWYVCMZW` z|0O56%wl^pc;m1AG~@|Ct#qUz!B=0pJ$s$csaOD?lDO&4+YAY6)gpkme>q6+|Lq`U z;CSkY6xl1ExY(R<;X%Uc;X>(6af(Rj;l}7HA$Eg>qu3wxzuQO^F~-3*5stqP83s0K zYgP<-P4k{Et0LJF&ZoP`Ugq2gNNLd}F5|n%K4!h2=-8n;Wa$YwVLB*bufw%Z0ZUl{ zODd-2(kGmhcUg9N=VyM?nzXLYzDZ8#jfj0v9r@I#$|nnU#Q96`wwSEksgMqoa-?~i zezv0nu8IXqxI6kWA8Wx9cI4ozgpKP3maxIO1xzg}BRh-we@obh<)@xuCVh}IOTBwP zw9CfhlGSyp>ZN8TVT7AJjY3}FTtS9bXmF%1hA+_<+dmyw zXwc3pXoH;TNp8n(JrED=po{8*aka2k@#)0-2PM{=rEOcs+4ua4cGDJiV0NC2ShgKh z^}Dly6r24BczJuZV8J(zg9_KV2O9#|{NMc*gi%-HEH>sHy7Pk#hrn941w6Pga7H-* z*0S9QX2IT1Q)sEqNRpK;MBL0 zw`SoyNi-efqz?J5+oC9O*nI}&eh6m z^p<0qu*9(6Wn*0m+mo^61ccUYw=Q zMtT?9rL!8o^r~l9Bm7Ix4tv$JAvOP7&sJ&vqh~|b{+FJO)&7s3z4brzY_zO@^z6&v zS3SG1sK%L~irEG=`XV-ceF#I6Kyk&PkNY|BBd?Ys=kOaIqoFFWfCC5FQTgKO$iu;o zm2^8iUY|D<-1+_N0$6<7SJuW}*w)#$rt0CNXIp{((B``o`j=CLN0IE!f#Sg5WC*8~ zSTH0ON$D1SqOxb;$Wq*lie)zwP@Y3}by-zTqhAO)NKE4_0BFD|(FVrpf@w}a<4XAV zul70dBixd;$c$IKu}f$si*>FKRp-(DU|1rb+B6c?`6n|V1^AE5K*r)fnSodpEPuG~VIb{5IB-V&b_e!`wDu@AFZ0L2wnt(a^34~v1!C$;U4P71y>D2p8E`H;8~3>+D=o?+0P8}eV5AvYudgG$1T3R zi`s~5mNAYMFZ=|Nk?s?epa9PK_Z{@9jS5|CNdGj4G@Jyg&@#DxYCl5g*ki)|UDBfq zdDU3Jztq2KENGo=mk~McV2#C~AV_{M-ZVLyXK`yqP*>QJ^Hr(bf@s7w{*Ww|LXvcwl(6c#|j^ zK<)R5WAWh55voGH0&L0{!c~AB6=*APEU#CqJX*76k!h_V?{gMIEdmtkX=X;##!Wz@ z?RJSnj3-VtOol6W7tWon~M80XYd^Wq9I*#Fv{+(18y zUYh{2WA+NLL`%Oh6FX!fzZrTw7#-j1v7m4;Pe~NS=Jl|DJy5-Fi2tTa{zfN*p+5x) z1xBfdbQpW=7dyib#Rx{C18OI71?}u9>&N%*($6TEFB3Jrxz3ixj=58e++al)>u_}9 ze6S;4xvRM&({Jx?PfQRNEB4W?)^K<#HtKM|;X#-njgaiWnsSc6u72z~!j1)r4LfzE zCVXucq=l6^9G%ZZl=yJ`LkH-d&$@jH)%Tf+TC?>e#!}YM5pg52TThx*Zh<}lZEA8| z7MX2A`Csw4HTdMaNAj#FR~lxPNeHZ~ORYBBUuMBox|>V+F=G+CpWRJ1!5bAVFAQCS z+ER^&D&S7cv#MGuX~x?zG}u{nZTaYW%Hh?{?zaD9G7d+7PJNTLF3XLQxw#Hy= zu~PESGUUr=3N9Miw~;qh8BjBL4}3Xt^rH2P@)5f9YPI_bZD5k(nh6p!WFR4Cerjndn6dLYx3XOHwFnSTW$+a;_TMGaF3I`oq_=T z_8GD3wdWa`cjbQG)=@JSc`Ng81g$1*=h2*x-{cbJ22fA&;(x=f-GzZ?h3-iu9-a@` zbJ0;s4AyyYWl9N;X(Z1zYGYrU_&MKRk}NFf<(lB9D|wW62^#zUcyVxnwz)=wg7v~k z{>%62c76TI?O^+H^?Uo28;P*@6X(mRi0x*#&;9M#?D5@P^87(t?4r4}|R4Uy#N zt|)kueCjE8t*8N3R7k5xRV}t0(USKWc5)f`g-Zj}P~4u(QtZ2Zz?a+hLxUCow&6ta zA2Y??W|bbP|1F%!QYLA!!U)o$$NNy5e_d~NF#dhLwU~88bKkcz%w^yaF7;5uxNkRG zU=8vC6YHO5?}kJu+7JpkZlE~Y~$@IvT{kGY+7?W4;w2tO5BXrGS7DIJeHh`zOD~la6e0&@|j3QXi5uL9Gi$;W0G`! zsR?tA4-nB4a8A07nbhCGy$xxDio@)64Ts3(==a~Y`X0VPvT%wG=^AN0v^U}2pyYuw z3lX)>k#88+v}XlLKWQO=I6)k(^aiHx&*;Pu_!(slL-g!;XXise9AHl!b$tl=-lmrD z<0hkneA{;rr)|8Woh0&>aY+AOF5^%MBZo@LKs~oGl|`T%3&~tDaqs zW8=_m7JMOe>upTBK-}@clQi{wftLcidd)I37 z^-QUkk+fc?wBD}GviYCH{+iO(gQv7A|4eCF!Bg6dKU3P)z{0)j*D0;f6iGUGN(=G# zlvYJblB z$;zS}Dl1IhP|y%QROY4o;xZKH!;ie#itf$sg1!E0)*PQHr4qedE`2tPUKEPLn{x6| zR{Bs|i7%|N|86zWd~>)$fX$D{G0JYX#1EVTfJ;r4;8GKMxdvz~-+N0n37>!mlBp0O zsrMG7v_Yx-kX?fxXO(03wn(+=HUwM!ri?oC=dtM+z5D3RSfk{>b4)Nr;s4th(}+D? zL>?g}`kZ$sh1>RoOygwpeY9G^V%mdQR`#giqs)QR@kyuOkT6#1k{N`$_nS7D1^!J= zSMG-)1Py^@yrP-TdNOFM0Qx12MbD|!F?>((xD8JEBuNN2uWlX`X=UTXhJ$Z`jZF0vwJ0=!WLq}g{%1J}v6UIGcZ4|ODY4o2m+c2OuX8X~ zQC!Q<2N~A9PR`V(Z}+qHgTPb;z70-!)@2(WPC_S*rl>%5(S?tafbOL2Ylo--aB!eA zAE&afiiPUm(=Y{y-kAFHp_SAwq2;`xPTTj{-^L3oKz*1<^q{tu;dfx`M9WZZ(dF`6 zn)hS5&B7Q0gO$Hhh=YW!K2uZ19__U%CA#Lb-k2FelOcsMOLr= z*7cz;ET8iPu{XLx@WfKk+W%&;Pe+pQ2T zhkV0oU(owON&6)K+FI0b3Qh8dUvSQf*lr0Dx>q1 zX?g-Lxjd7uFM)VtL}Emce0nxO03#Oa@DM7s`fA@^Gdbt8zMZwJR}zi@zC7Jq#?>32 zyp?)>2Ez9IU|V#AVj~m}bQJM!%u!lQSE5i~Y?Rf(Z40yMc?ypt1By_h@M}l&==kab z(?P>SRT}&ZHK+d?zAIzC4&QH`X{t^bls@PO)|2n?#WUDu-aFrK6ioBMTVHuB`O06H zq&~q?VRPhyGmYYSlbmieWU*%85qvzDwls1H+Q{W;n&us%o6_DMI!36kheUU3Si)L>y?m_W)rGYi;8 z+8d@H^mBI61J}|qPV(?)HTzoVwd7AL8SvcQ0X%o#Zpsz_FRO%JSMYXotzF@zOBwUC zAZkwAs#aHnd9t^T(@z0Qe76q5DtRGUSM%A@5=iT#r4?RBw<#x&LmX%|m)cNar^^Q% z)8zshEtn;XLpQsO;g^$z5NX`gPt%EBANZAgP$J|yS(kUM%qAP_rU?_OB|hJnqT>|X z;Txdiiv#I*XzA~Wqd6m^nTjs4{hPHyo=0sPa?lGewz9+T1CTt9ky%f&p(=Tfr94V9 zMzD%@tR;BXCmXCk05bPnb*~3o=}$W%F>2j7&sIVQ{?rns87F~k2|Yn*KR ztJWC)U$w^krGILTlsx}kYxH0Gr`Cu<`={1O98$bs?_%L(AM7@8!X3{#Yd;%{)pU%T z+&AM`C@S27&*8eB{>Tl#WrMPmw3?BSkzOHXn)}R2eIn%gy_?wvS8*Zb&-^`i6!he1 z6fzCG?dOgzwhHoJhr=(}%d!ZctHt4%#zf(It4i5wVjM?~mgGg(>5}dqg{pmuoPg%8 zBLsCHcaLyyo7O-$!O$qcfNA&)kgu&Nu=B7H^%J zwc`$Mp?xS=X7uJ~Rf*n-4BDFE^l6?KgxGCshOrIWOuu+gOguvQunk|Ts;)DVF8I{8 zLz<$9N3N2(^7Nv!{Cg2Uog{BkNE1_b#u{-Cb)wAmL)jpn>oe|h4C5882X7Aabj{Fy zh=w{)V`zBDgKTlZDwhPXuM)_IoXpxb<4M@J^7uQ}ANP&MxusNXh`vq#F#Y+mxd2j) z;-LD=pHNTrB}HE*jg^^Lqj&^+c$`h4ZIa=DKd!Glc3Yn_5-axMrx%Q=a}ncfUXTSC=E%q504AE<%Ujh ztGrj9BVr^Az1s+i@WBDyJAY(~dKUbogfT$CXYbpc8KsMbv61YKUzR=j5TLP|N`GC1 zML!(yd3=X%=I);_YoDMUr=N+&wM}n1d${`XoPZtjbf14fIItK|-jH zCYd1E+dyhJZPvaA(o+Om;%HggZw6K^jj}Bywv_~DVSr}VDQs#-Y(WXDJ;91xv}F}% zr`bwEk3&*05uD6mYPaw8ld7eAyEh+)HI^GGcj?2*Z zwOZLJcDWFuofguF68gEaUI+9vy5QqnPYdRUxw0p45xo<v@HH_F%XNsSSpE5J0B2 zo0Xij?9aRmTYo8291N7HFluS$*FxW!kHK(Hxpghd^FQDo5DfQbU*X>L|A2c9hBzVr zz`e#nOqbZnlD8MH>XfACt2z}@|F=4&{HjhRkN&qhmA>{x{*1v0+`tBR?}l}ViTsZg;Wo&5-S?1Th2V;C80VDAvgo&EpSU@fvus1pTlIc#e3~LuvYF z-(R&B{1k>rQ$oyseaOW0uTZ zZZCCS#uj_va`F1FZ<_q+u!h6&&TZ1d#%{ob67j-dvkXG3RE8J??jF)uHl}OPS91aC zNW2UHTIBGu{rc6sY50+%s$W>dOU#Pll&9r@@)kpu0x6EJgY6oep01pLNTwds=9Qyk zt527iSw)PJP-jVocMTE@bNYS>)*#+YY45;76Uyz$!34_;u{AzD!pwti)dytk+$Ll$ z+;lBO7u%Bt3jsS1Hg8)p$GJXXV<`JAlpH(Gnqw%{`MnK`oRRDr!wbB3+(hHE&ik}9 z&W8pVL1Y8w&;7AQKlnbHRC_rMFAFES0uDzl?;S9B)&ziy?VGn7txf|fHub+4B(e(w z-#0h`Sru-b%s)4*Z{Fijum<|FrafslaeLvx^_w*h^4xPQ?K(~j%4K}?oea)GqmF95 z(`R>W)ppt^U?oC5(8znL633MXVwRXa;9DmXr7zAik_9b1D6TWLt#hdUb}H%Fn}bJo zeKyY4NmFBg=M;H`0o(EAGkFYzL!&$<7|~KYu?;evkE*X=HOy<9gE>MU z+87TxD4t`XP%8I({P|A3^ktsgsV8dtWMUjz{2&0lNGa?sG7BYubtQnnc-y<;36;^F z&@q?RO}XpdrFTeg4C<8L;3UY`5E+l{HeT;03riG0yBd03t5e%^iF z?0Wv47Z&7mX>sI%3v1oh1v>t<+2t+LIgEpWP*^`6aqP(@^6U31;`o;WP{yE#c3Z*E zY}erP#q-CzGE&8>w#JF8Ql(>dpO-6co#0yq|NYXRKOd&<{TLfReo@J9?}fIVV`=AH zJJ|1nFW3d4L7rOS{8Xm$JGg_(o_tU^6ukX7xrrv0fsDoxjkRqMT#p>baiSLX<`f0g z4}X%o)=TH6{RTMk?w3>3+>iGFZ6EF)v;fxjLV}AOLlDJdsMqC$D$n|_XVzqd+ApH+ z?2P&7x#&o|!2t0!78?jwVWv*-gOD&N4QGn)<;gobcj#;21tP~FYq8!NBIH)*rQy!Q zlOWP}3igfl0+5yHR`h@#Ze!E;1@FZ<$p%Ou`6s@bm~0ip0;Mo-I(ZMv(j0&LqK_JD zu!<-Jf8V_}b{7;XXUjF3Ht?sT@0*K^ z1apK?`##W=3t@$iJ-*HWU8BRIuNb&diaa69!C5T71uSpE;1< zOhm%r9NO8g9^4hr&M-0EXAYz7oJ$O)%y{yf`K(?zx!`<=LusA!`u>9o^-!Ux=SN1& zVwxdxgEqdQXA4hIN?Kz1g=)I5!6mVffB6ZREwyp{d)>J| z<+)U*(xIwpy9H#Z;Spxf&juxA{8|`Sqz^I-`+%zQ+GeZT%(eBlr}i#Sbw};03uo7* ztC!#I2sD|vvY2uLNQahzYd2iIxg|kP0-cGo69-v|0o&>iVt^hk`lWK(EfZ5^jV@iO z+u!nT%9Sg{%&R6OLjoiZ+Kd1e&W@9@cgv-&!Ax*BHlJ#NOHC|m@M3}4Wh+wUQ?0F!-?x6q?5!-POv~4twQ+y!legi+kCEUZU7TIf`V$#gAOMBd zA2+$xNABCwx0D77IDs(Wysx_2M6w7$_5J=zTR?TaYt-x8w0pgm=jUhO%f4(iF{xPQ zaAh2ZV*579BnW$v=wnU#0i4yoiboD&#@ql&j`$57eYL1V-W7iv_o$UX2#|+fxAg20 z?>&zR@D0MwCmj;#qtc^CQqjyzkUB!&zHC{s80xt<1uPN{N<)iHsq#6SlC&ob$s@;X^>aBwt* zqx*64H4V;KlO{}I{bR@CU?k_$b2n{ClZYdX#op%_290swz<1*dSZBOssaommtnr3FR4u@o)-{j_&APOmaNnM=Qws^vVgglaF z>$cL97E7x`E4B{|T6m)^=ok&}qx;z@KlaPZm{%rh!K3n$1CkhZTXy6R@pdj&Lq)Q8F79_OhR<2+OjyUv+33`} z*7GpW+4>Y2aT#yQ=lX(HLEHkBO1-yTKTJL%y4xX~$7y<5Y1s2uF{s(QiXYToG*xfr zP#>V1TULR3W9FQ6CUA)Ac>3m?eA=m(+KyYRj#}Z;3qhke$aBs*deH~w->N|PGxlv& zim$hv~ zX{>>XNEng+w<;@&gR-_N@S76(wcxhEa7+)C$g8@MO_pE0yKE*F_+1EGxF z>M{^dgq&D3GBHo!lKbcN)$<}C?IUXd`lzZQI)2+utR_n+k;|9!20U{H(0k1|IbtX4 zVkg1NN>r}7m{05Xc6lUSDyi66G50DO!x2lI%ekhRpCfYu3&;Y49Uq|d?~Mmt-%Iv9 zHekaSR1A7QBe#{~WSLvm+F|#t91jXhqnF@4(Ted*EWQ3b8j(mGl@fX>WtnuDz}DvS z4LQT^Jz74AMVXW;#}eC=O@=J|>xZ zoawME-jD-E6&B#l`6$}>B+;!>|i1TnI|K_Ykb_dwUt;BWfI4FDT=%EQ_j!?It3vINoKJhlSBlW zBnj1g>|~gHk`$iKTz6y}^kL2!FcbMUhZhZAWj*2AAN-OJ{BpCv$tyhVAY-jKExNv; zqSbd{o==@ll;qwZZBLvmY3iGz$2-L@2Ek`3s!bd>cZ#l0TNyY+g1`%dH+TC`HwOr* ztF)@;b2sAe2H>s~{jJ^{8M%&7#8-Lg86Bvx5cvH*z*nh;?^J!%N-?YfKb2<*PZVK@ zms6rNWM^qGT#NOtVo2`I#sNJMw_;JJLwpZ(f;4;rgUKMx3Q+6Noi-F7`{O*cP3*QPR{WdgL`xMoQGO?DBN!Cl?N5NG^ z=XA7n7`c4C$J7Xi!B&H=(9S)-H_ z;OQ9mkDZkQ{d*5d)~5;1!*LJn`sq2;VmXGybuPyb!E66 z22S6CYFm)a3Vl}9syfT@Ub*R9XoXP-;ecXOeWV1Nu}}o>S41lb0iR~2p-WW#@{D4i zJ5!cS#jK+%Ptd_!!<&iZh4Hx!g=&}j2!V)x3#tbN+21(;PjAWn)xE^W#go=}xDp!2CG>r)86v39=;9VXcF0 zZ?yN{xvDC@xA7Uxuo0%+YVhb?2ilHmMe@;SvW6OTP8pu|yT|#fyJ?VsL8O%7X_4uu z6zYnuo5pTvKfmZyy*f(_Jf9>a10MNu%EE}QO-vfZj(0LA`Ecbriqf5BjU;28lbD=- zu*;dBD@y85bFs(sSq`kesH-1b1$x8fz(dE;mGa)fvg$5{6+~uvzgs=n0{PH;D=^2j zPu=e1tH|j#*oXbD_>mVz_&LH0KmmBLYOnBV!UU=ixO8w6elSGH(hM6)&~#$*DsNUb zeFhrPV(kclDsb4cL)_KHtn!{$82vBloa$;Iq~cpl4vW~57tv)eAv?Am@T8{dA`Y*% zzogD(+N7bQjMai~7|1+9H*;v11tJ>~fw6}1M9lmd`kgL-e)ngU@2mhPrX&!`7%Tz5 z9?Q-x0kMoV@#zQJBm=wJ+Yj#A)tx!URYO;JOKIwZP_#rADp1W6~t5GT^oy1Z$6ALGm4-9Za z4L|_^yP;$Os!3C<>qkcYrd&L^^h%mRa#K)5ZTNDVR5~hBqX|`$ihuUz=&bM(+xVhT zUjqk%;(qc+Ijf17g=7$eijv>;T|aPYvzu~wEK9jt(1It;uFVq3?}3XaHz}bZjVzLk znMv)Jv;KWMULvz3Z#y2qi;8G1BSxsAvRWs4aG&q0r3JUwH3u|pD?MN5z(a!0>aw;F zC6vb&1$jT*;&7XqRBMV%ZDv2SvlE@SH!h%Pzj<78yxxYxnp@FO+Rhb|l8eQM#BV-js3 zCov`!%-GseHL?}jR9e*l+V5sG#6KLT@)}h0%5hrK+mRy$TWmc{IX*a-ws!ZAzepN2 znqkF;Lg_8rK$WpZI_<*!yjmfDDmq=bVqz(MAfAinSR5*~)t!?OUV??f9d`sq7)Bq> z>F-!Rg?j21UAgS{t^=m*hyC-c8(U<88H_~oz5*lQ`VD!Ml_iJO4?i#_P zlKzHQ5SC^resbb&-$?T^wHeG%X!~UDd~)BUe?_uplj?NndFQl3GDc zVK=n1EkCxg*X)SsRcW}oKwt`!#h2o0{`{#KpJ5CuX>v6z&W>LVF(UQc$v_4R*WL5y zo}h^4SlGJFbTs%$P&kq&s$Q88#C=OytsS{N?WS{zZ`J1c})!YG>|SLxsGRNF2+i5{qN;OH|-w zF$TIx*%ey%OVbs*#a^Oie|l+#WE)dAKVhTHJ&JQjg^c7f0PZkF7c*EZ#$; z=56mDifq2rev(ZP4mg8@Zjrc@Yq!xoi$v$|`VBwbr5J$5b!BF2)nk#h*#IUM%fFsU zlSeLC`*|_mN%D?e@b=rfxKFRw2uJpu$qEfd6wN%U*Od z&MdVpe0Tz1gAHXoE{rvL-U*Vuoqp3C+M>4%q3=EV`Veaj%AJK(GJ)sbRirAYfSMw+ zRON7CA+}Z|J&;>xS-#Gr!wL+5y2`XfK-BssJaUz}3fC>+5+rZa)!t6Lv4Wa*q<@z4`M*)e0+d+A9h@$3{W#9gdKjUWIPp*V0`Xd5%5fRrS2`u z{-`@Sn*%kbh~+Zu$8worRG3Q-tDE5CRwsv!`DJrGDk|}}y=FtgI7f4E^=?ytTI zY4UIURHLraGd7-IYdze|RM&_No90DO=2~X4Ev(ueN4F&e0mP5?u5=RSEG#c} zr9eBZxlwO#DyQGF?nA*zZ4GavWn4ZGp|PGLAi!96guv%nbyIOEFQ7-T;?|C&7IU|3 z!CB$g7;|2+tZvzdG&(OKkSr2Oe>2_*-0DkbSlu7!S+4w)fjVVe}XVZ(1SI6DEs0y-lhi$0X* zJTC=~Qg+prOK#`Z=tk?K!UYVbPhL#dWcM*|M|G0bKFhu0Aubn*A z9Coe%blCAzRW+k2&RDjs4NtkoS&5fdTQf)VJA)pY^YW<9`bxJ=T(83(cItorai{u=! z=mne=q*1GQ%eWEi5Uw}o)PV?Zt5jWlkn^(&K|wGMmQ`_(H>!?4@uUiax8*R~j>QxA zSA6=|=(zCVI+Mg-!VZQl#D|q85|m#5Rqpa={yjJ7|1*$2lnuV$; znyS9Del?)&7aJEYJw{_g9gDwZC2B(NT!5|xP@Q$1ha8h|2q2{LV*bp2Gu!S_Ikt^3 zXX#U}6Iv%XXp{zTPhGNzfu*~idbjL!upQ08_nV6Afc`*L zC!N~BdN204kY=Cfvt+ig_h9CNW}E#DWZAYA--(g?4YaDzy9kc-N-S{oIP0ia>8IC$ z9&I_L=3I4@B12W(ba3pl+d!7~cIJ`-F2H7fNpces0rafvlT`_fHIKXX)~0;EzGbrnKsE`QeN%9&T=&K18DK?ajT?AIXP~0!r zLpA3$%G~s;VK-%z-nTtR_c>%%1G1`n5#{e)sP1pcZ?f({-xdh%AFHh%s<;?~76k%hNwx>-r=$~hJ_0Hbt^W+niAhbWqBm2*VtV21abDumSCjpJ$mz!i^+SN+0(W#{H2 zc+xE%ZCw^TKHHsIIDlRhhiNHwC|)%}5nY9#FR>nQt@vc65c;SHeo)ibvth6o{y6#i z@_f2gXh7~-{%lrmNKM0G@4mfvHMZFReVn`ns%Ry zaYWQyE4*6hCr3!nYnU;cW+DWmx3tyJrc)Fd|8wN?XcR4lfSL> zUroDE^r#>6<3T3}xRPnPenPRMqT%K~L~LturCst-G)Q}?`QjApaLT<#WP^}$5suHO< zj5_BmG1tEC4vJB6t?V7>&V)I72Xn=Qr}IV0dBH|BC;IcP>r z+9HN*%q1J;tIksD2}j)iLATz^ZIhw^JS|UizWQ8mA--7Khhj*xsySGQSC{*N=u=2z zVNTa7&93{KnuYOMVXt~zg;E7)k4~3si75VX+;--cJ}Wd;C$Uy zAGXHJXYe7ZdpDSD!JFZ4WDTr55UT0gpHLQyZ_VYkH1pJ@bT^H!m~O;lt;91BilPk2#bJbu4o{M z3vweD@MG&njdp&)^)0z>^-RL};*4T2JN_Lv0DP!Bv6!^&Q123V63tfG3O>~}2A}GZ zpgOq1H@cJ_5f!`AmVIeD!^xfrK02ve);NPEwZ(7>qBQG@+mjEZj@jZAE7Y-Vv77ML z;O2KKJ^Ix)$mLfXZoSs(9m(;{nX=xcA!*y7{x;Zs4a|SuvVs8(ESQb_a|a>)>3`fo zfFY0~!e58FST}IBx4oH=P|hgLE@`Da%p?t+c!L|O4r`0r3Y(eThWheeo%K`{rr1A} zEkWiqgHV=W8WX$%KHn__1DdF>m0rRBbxRRMEN-(-ArC&{{iI0O*s-{_q|n-hd3(cS zC*uA5>scD^>JlT!#I~BF3O9|8rX_@8v22W*zFn;a_+lM=h8KPJ)V7BG@-g3A0dw-;eF0> zi>3r@+D8W-b;%$fCaT=b>eD;`{4LJdPnx7?V;<88uY3hJ9^c;g!r_l{JC`J^XA zxrqq6ekcZ;*s)KZh}n5C|C%V~WX5XN+f%sdg~(sN`geKq4Tuwymcm)^&+zqb6I z^7WySckt%;yHTT>j0(e}TgUQ;gFvZf+~ObHGM@y-f7x-Wd2w%>(_W;kQ^z9~^YWbe zvs5$Hi9G7Q+BKP*Y!kdjHr|$pT*Z4qI;8f!jBYu!L*c?G)KyJ(Uun4Qd?U{Er<6xR zCc*D=$XDITFGfXBIOrl$l+UM0e-)P>rqFS6IjeaW6x zZZ4%Ldn_kR0yz}D=oU=?^#jmVf%dDj9V#C{V%8RO4xg>ls1G7hlxM$ikP-n3m)x+vOV^xplneq<=kpE9%cm59b+Q$K0wxT4mR`$xii=;*xN%lBM z_H{6fEo2$HAtYOtCR8UT$#g7ZYLJStWXUm>QD#zhlO}2G%=6XqNayrC|G<6SKfSNd z{k~t<^+}6S>O8VT1%wz8jdKKPVtu^r_j^WNU=@~{MpdWBHnE=R5l-iB8&ilmIa z8YahBXf&1~Q4Y1st2$CZ-XH?RPo@C{c>sbm^XYcIGn_A$l&z7k=>W)@Lu)}bga>}2 z^G5#0Y(dMj$k@ml0Kv&Mljc~#^0vfA#JX^}q1PrBweslRE zID|a`%WucR>ioSMALC8*&$|`aCsuz8x0%B0!8by&yWk>>HNe02WH|E^n$Yw_S)T8A zL<0iymavIvPL|IVIvYV1T4JOHFT`|Vzog!e*C{(e)GHG*a5&%1-v8W-2Qejg>Os6R z_&rHha~^&e!mWm38I-tKK|O!Ww83aTv>CW=98mJe?JJgSW?U}lAA?M`oi;$|I0XP> zu>|Zax0uuk%5F%0T?EV2Zn5eCAX1B~<}LO4y7?exqcH ziZWl<-`OsihfgWH4rtj?Y>Z)@{9iQ8r+5fl1FY1SytLbIg)R*4o% zZlTOf2HoYoYP;?!Yxa7V)sOe`L=WpX*75TnS$|jMDNz6kWOFoC2q^Ros%Y-UVVe49 zJJP8pmUX6uu-5X|P&Z7UPc4i!P5|Oa=pAH^xVg6*|HZ&S1LnTu-Xv*v^s@QDx+!m|hlw5q`JB}$~HuC?K;p0J4vutmw& z+Va^@!od@AL0r&)i@>mELM%O|CXO3tP=PH_@6KETTk;L~JmxV@_|z95TXW;DTrW&N zIiQ27g>0;mrWdQknIG}!d7`p)ZO7Hv>WDc_@FbtilvfyGyuUr^)_{7baC8|8S%Fmc zp482R8%92n6apG+p5burjm%!2ZoJ0j>!JWT1n(i|UO8jY65g9|5T?C&6Jp|C>Obl` z6(|TD>N#63ap^vv?F9JaLHV2xsfq_VCk;~P3i%AmW#sdy+~gc};s?o%`mm_WvG(TG z@B1>Z9f+R}%a~*LmY+PpO*V>vm$PZr7fdYcJ&ezq$JPDSvh&Df2w@3yjY?QTWEl`hzINC%g|hWo@+01yi~jE_$LK6 z6}~a0Vy|uC_U`VrSh9a> z(+ zN|NhFwdIK`v4)n`?RB1B5Rl5crz5sBRcC_fys)dcWbDe#wdfPQj!zj$U{&M0OIcI^ z$~=W?`3`EV*aN-+fHKCLQ09x>CX^Z9gfi*hp^V}dl(E(4BxMrPGm3E%#V6Rjfu^+# z9p7zIN%K80_(65C26-jK>Aam^v|X!@{dH%vCNo50KHQ8T;&LcgjN*LUM=ZR2Ctyd0 z;iJ0=+8!a?ash`wX`0>(-EW)Mw7+y<#R|L2eKCo7KetMH+q;sW?qWN+H{M*y-80pf z_+@)PXfU7%E3MGu`RjX5<(Feu|4xTuSgkq4GY%xHD4dn_mTGYpbLnFW7&brL1Fp1{ zcQ70Mu#=a8V}GS>I#kd_s%|B>#+E`+}03Yg*(Zln=VWjgNyDmf~(I zUMYX&Yu_C<>WQoHQ`_<|uK^DO`@bGW-RQfAp?>u+H@|rp{Fa9yZhDw@NaJNj^1)&m zwM_R^zS@_644?L(NVLjZGoyI|4K3HI=X1NvIYOK;7|wpFDzH3{yIUF$#wm@-BJCdH z{WHAOPy}v6VAXPz1+$htRiR;PpwQ$_8oXJqpiSGitW0Ae4vSzxP2yZwAR6w%yZu&INM5dcV@qdn!o$by2m2$Ri$zGB zv>(Kl$5kNFloT{HM!F7d_gu+$kT&j_UiuMN#LI!A;$+Cj;e-7nwPX}T{# zA)@M&L)cV_$MYZRf_yu)iw<3%yT5Z^UQTs!FcL>;NX8qJf`i+30B5MX6yYc{2ypMYzK{8{@wEZJZgEh7;uou^lvL+{TvJgycIEPg?0A_RwA$9vU- zKwhJl9W-v~)fW#?LTb*R`&cf<0@H?OPbF1I`KX%C1+NMwj{f0@wQ`K-&Hqd>_)Ni1 zY0w5O%QZ%LoCdj_Bykq8_7}=_Eg)J!DCSi?S*g<@ipa9(WYxvE66qJP)0#%SA+!Pk z2GpK9mIdI(pEAsA4yC-znX=t)|y+*kI>^hN`D~mfpMG0*6LMc7ftVZ-S z&XZ$J9Rp)SP*}oOC*q^kF+oqOLefCw$to&Zf%xq~H+BYp@?ea~_ z%=|{nn7RB`%Y<)g8RQQwGxJT$to{!z!}f1l=FFy+IY^-asj{?Ibgn_33)i&JTmVv$~BK2<9GJF52Wxi}_nX5l(84J#m4nWJO|3k}UZ`U&A+qI0Y={7B6`!8B1 z>Srxexn0Y=+|n{RKWUlnZCa*;#^t&sjQXKv^tQB2+gB}f=ew3+ (InferenceConfig, Arc) { query_configs: vec![QueryConfig::new("sum(native_metric)".to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; (inference_config, streaming_config) } diff --git a/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs b/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs index 5035f1ba..f5b58515 100644 --- a/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs +++ b/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs @@ -90,6 +90,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((_, result)) = engine .handle_query_promql(query, 1_000.0) @@ -317,6 +318,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); assert!(matches!( @@ -374,6 +376,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); for result in [ @@ -414,6 +417,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); assert!(matches!( @@ -455,6 +459,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); // Prometheus omits an absent grouping label; this native representation diff --git a/asap-query-engine/src/tests/range_query_arithmetic_tests.rs b/asap-query-engine/src/tests/range_query_arithmetic_tests.rs index a18c6497..0fe3337e 100644 --- a/asap-query-engine/src/tests/range_query_arithmetic_tests.rs +++ b/asap-query-engine/src/tests/range_query_arithmetic_tests.rs @@ -108,6 +108,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -408,6 +409,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -531,6 +533,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/rollup_tests.rs b/asap-query-engine/src/tests/rollup_tests.rs index 62d3cf11..4b96eeaf 100644 --- a/asap-query-engine/src/tests/rollup_tests.rs +++ b/asap-query-engine/src/tests/rollup_tests.rs @@ -322,6 +322,7 @@ mod tests { query_configs: vec![QueryConfig::new(sql.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -417,6 +418,7 @@ mod tests { query_configs: vec![QueryConfig::new(sql.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs index 8204cefc..da95ed5d 100644 --- a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs +++ b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs @@ -53,6 +53,7 @@ mod tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Streaming config diff --git a/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs b/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs index 510ba17c..6815e7c6 100644 --- a/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs +++ b/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs @@ -254,6 +254,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -466,6 +467,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/structural_matching_tests.rs b/asap-query-engine/src/tests/structural_matching_tests.rs index 10ca99eb..dcf10629 100644 --- a/asap-query-engine/src/tests/structural_matching_tests.rs +++ b/asap-query-engine/src/tests/structural_matching_tests.rs @@ -82,6 +82,7 @@ mod tests { )), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((labels, result)) = engine diff --git a/asap-query-engine/src/tests/test_utilities/config_builders.rs b/asap-query-engine/src/tests/test_utilities/config_builders.rs index d432c204..7b358a21 100644 --- a/asap-query-engine/src/tests/test_utilities/config_builders.rs +++ b/asap-query-engine/src/tests/test_utilities/config_builders.rs @@ -206,6 +206,7 @@ impl TestConfigBuilder { schema: SchemaConfig::PromQL(promql_schema), query_configs: self.query_configs, cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create StreamingConfig @@ -231,6 +232,7 @@ impl TestConfigBuilder { schema: SchemaConfig::PromQL(promql_schema), query_configs: self.query_configs.clone(), cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create SQL InferenceConfig @@ -247,6 +249,7 @@ impl TestConfigBuilder { schema: SchemaConfig::SQL(sql_schema), query_configs: self.query_configs, cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create StreamingConfig diff --git a/asap-query-engine/src/tests/test_utilities/engine_factories.rs b/asap-query-engine/src/tests/test_utilities/engine_factories.rs index 6433dc03..85f9bb13 100644 --- a/asap-query-engine/src/tests/test_utilities/engine_factories.rs +++ b/asap-query-engine/src/tests/test_utilities/engine_factories.rs @@ -128,6 +128,7 @@ pub fn create_engine_single_pop_with_aggregated( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -258,6 +259,7 @@ pub fn create_engine_dual_input( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -374,6 +376,7 @@ pub fn create_engine_two_metrics( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config_a, query_config_b], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -482,6 +485,7 @@ pub fn create_engine_three_metrics( .add_aggregation(AggregationReference::new(3, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -556,6 +560,7 @@ pub fn create_engine_multi_timestamp( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -689,6 +694,7 @@ pub fn create_engine_multi_timestamp_with_window( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/window_semantics_consistency_tests.rs b/asap-query-engine/src/tests/window_semantics_consistency_tests.rs index 8df32eb3..03bdd871 100644 --- a/asap-query-engine/src/tests/window_semantics_consistency_tests.rs +++ b/asap-query-engine/src/tests/window_semantics_consistency_tests.rs @@ -391,6 +391,7 @@ mod tests { query_configs: vec![QueryConfig::new(query.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -492,6 +493,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -698,6 +700,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -814,6 +817,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -922,6 +926,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/tests/e2e_precompute_equivalence.rs b/asap-query-engine/tests/e2e_precompute_equivalence.rs index 7ca83075..35979cd8 100644 --- a/asap-query-engine/tests/e2e_precompute_equivalence.rs +++ b/asap-query-engine/tests/e2e_precompute_equivalence.rs @@ -334,6 +334,7 @@ impl NativeDagScenario<'_> { )), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let query_engine = SimpleEngine::new( store, @@ -929,6 +930,7 @@ async fn e2e_nested_aggregation_operator_matrix_executes_instant_and_range() { ) .add_aggregation(AggregationReference::new(18, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((_, instant)) = engine .handle_query_promql(query.clone(), 1.0) @@ -1320,6 +1322,7 @@ async fn e2e_sliding_precompute_outputs_compose_a_wider_query() { query_configs: vec![QueryConfig::new(query.to_string()) .add_aggregation(AggregationReference::new(agg_id, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let query_engine = SimpleEngine::new( store, From 2138a0250dd7e7e25c22427b4ddef401d14e162b Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 19:08:04 -0400 Subject: [PATCH 03/12] feat(query-engine): answer RQE expression trees on POST /api/v1/rqe_query `SimpleEngine::handle_rqe_query` evaluates a validated RQE tree at a query time without parsing PromQL/SQL. A leaf reads the aggregations its `rqe_configs` entry names, over the lookback ending at the time aligned as the PromQL instant path aligns it, through the shared context tail and executor; a roll-up config reads at the aggregation's grouping and its query-time stages regroup to the leaf's. `agg` nodes regroup with the query-time aggregation pipeline, `binary` nodes reuse PromQL's vector and scalar combiners. Output labels are the root's labels, never `__name__`. The route answers Prometheus instant-vector JSON whatever the adapter's language and never forwards: invalid body/tree, unknown RQE id or bad time is a 400 `bad_data`, an execution failure a 500, an empty window an empty result. Top-k leaves rank globally (a leaf has no bucketing labels). The SQL context tail moves to `build_execution_context_tail` (now returning the planning error) and `get_aggregation_id_info` takes aggregation refs, so both the SQL and RQE builders share them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../src/drivers/query/servers/http.rs | 72 +- asap-query-engine/src/engines/mod.rs | 2 +- .../src/engines/simple_engine/elastic.rs | 2 +- .../src/engines/simple_engine/mod.rs | 53 +- .../src/engines/simple_engine/promql.rs | 10 +- .../src/engines/simple_engine/rqe.rs | 318 +++++++ .../src/engines/simple_engine/sql.rs | 55 +- asap-query-engine/src/tests/mod.rs | 1 + .../src/tests/rqe_query_tests.rs | 830 ++++++++++++++++++ 9 files changed, 1278 insertions(+), 65 deletions(-) create mode 100644 asap-query-engine/src/engines/simple_engine/rqe.rs create mode 100644 asap-query-engine/src/tests/rqe_query_tests.rs diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs index 94ce7fd7..16f21c0c 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/asap-query-engine/src/drivers/query/servers/http.rs @@ -14,8 +14,15 @@ use std::time::Instant; use tokio::net::TcpListener; use tracing::{debug, info, warn}; -use crate::drivers::query::adapters::{create_http_adapter, AdapterConfig, HttpProtocolAdapter}; -use crate::engines::{QueryExecutionError, SimpleEngine}; +use crate::drivers::query::adapters::{ + create_http_adapter, AdapterConfig, HttpProtocolAdapter, PrometheusResponse, +}; +use crate::engines::{QueryExecutionError, RqeQueryError, SimpleEngine}; +use crate::utils::http::convert_query_result_to_prometheus; +use asap_types::rqe_expr::RqeExpr; + +/// Answers an RQE expression tree, whatever the adapter's query language. +pub const RQE_QUERY_ENDPOINT: &str = "/api/v1/rqe_query"; use crate::query_tracker::QueryTracker; use crate::stores::Store; @@ -106,6 +113,7 @@ impl HttpServer { .route(range_query_endpoint, post(handle_range_query_post)) .route(runtime_info_path, get(handle_runtime_info)) .route(runtime_info_path, post(handle_runtime_info)) + .route(RQE_QUERY_ENDPOINT, post(handle_rqe_query)) .route("/metrics", get(handle_metrics)) .with_state(app_state); @@ -143,6 +151,7 @@ impl HttpServer { .route(range_query_endpoint, get(handle_range_query)) .route(range_query_endpoint, post(handle_range_query_post)) .route(runtime_info_path, get(handle_runtime_info)) + .route(RQE_QUERY_ENDPOINT, post(handle_rqe_query)) .with_state(app_state); let listener = TcpListener::bind("127.0.0.1:0").await?; @@ -445,6 +454,65 @@ async fn handle_instant_query_post( result } +/// Body of an RQE query: the tree and its evaluation time in unix seconds. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct RqeQueryRequest { + expr: RqeExpr, + time: f64, +} + +/// A Prometheus-format error response with an explicit HTTP status. +fn prometheus_error(status: StatusCode, error_type: &str, error: &str) -> Response { + (status, Json(PrometheusResponse::error(error_type, error))).into_response() +} + +/// `POST /api/v1/rqe_query`: answers with Prometheus instant-vector JSON. Never +/// forwards to a fallback backend: an invalid request is a 400, a failed +/// execution a 500, and a window without data yet an empty result. +async fn handle_rqe_query(State(state): State, body: Bytes) -> Response { + let start_time = Instant::now(); + let request: RqeQueryRequest = match serde_json::from_slice(&body) { + Ok(request) => request, + Err(error) => { + return prometheus_error( + StatusCode::BAD_REQUEST, + "bad_data", + &format!("invalid RQE query body: {error}"), + ); + } + }; + match state + .query_engine + .handle_rqe_query(&request.expr, request.time) + { + Ok((labels, result)) => { + info!( + "rqe_query time={} destination=asap total_latency_ms={:.2}", + request.time, + start_time.elapsed().as_secs_f64() * 1000.0 + ); + match convert_query_result_to_prometheus(&result, &labels) { + Ok(data) => Json(PrometheusResponse::success(data)).into_response(), + Err(error) => { + prometheus_error(StatusCode::INTERNAL_SERVER_ERROR, "internal", error) + } + } + } + Err(RqeQueryError::BadRequest(error)) => { + prometheus_error(StatusCode::BAD_REQUEST, "bad_data", &error) + } + Err(RqeQueryError::Execution(error)) => { + warn!(error = %error, "RQE query execution failed"); + prometheus_error( + StatusCode::INTERNAL_SERVER_ERROR, + "internal", + &error.to_string(), + ) + } + } +} + async fn handle_runtime_info( State(state): State, headers: axum::http::HeaderMap, diff --git a/asap-query-engine/src/engines/mod.rs b/asap-query-engine/src/engines/mod.rs index 5d1c8033..15efd232 100644 --- a/asap-query-engine/src/engines/mod.rs +++ b/asap-query-engine/src/engines/mod.rs @@ -9,5 +9,5 @@ pub mod window_merger; pub use query_result::{InstantVector, QueryResult, RangeVector, RangeVectorElement, Sample}; #[cfg(feature = "native_query_legacy_test_support")] pub use simple_engine::NativeRangeExecutionMode; -pub use simple_engine::{QueryExecutionError, SimpleEngine}; +pub use simple_engine::{QueryExecutionError, RqeQueryError, SimpleEngine}; pub use window_merger::{create_window_merger, NaiveMerger, WindowMerger}; diff --git a/asap-query-engine/src/engines/simple_engine/elastic.rs b/asap-query-engine/src/engines/simple_engine/elastic.rs index 6c974a65..c3d5f461 100644 --- a/asap-query-engine/src/engines/simple_engine/elastic.rs +++ b/asap-query-engine/src/engines/simple_engine/elastic.rs @@ -44,7 +44,7 @@ impl SimpleEngine { // TODO: Figure out how to handle query configuration for ElasticSearch queries. let query_config = self.find_query_config(&query)?; let agg_info = self - .get_aggregation_id_info(&query_config) + .get_aggregation_id_info(&query_config.aggregations) .map_err(|e| { warn!("{}", e); e diff --git a/asap-query-engine/src/engines/simple_engine/mod.rs b/asap-query-engine/src/engines/simple_engine/mod.rs index e2f8990f..5f237069 100644 --- a/asap-query-engine/src/engines/simple_engine/mod.rs +++ b/asap-query-engine/src/engines/simple_engine/mod.rs @@ -1,12 +1,14 @@ mod elastic; mod promql; +mod rqe; mod sql; use promql::METRIC_NAME_LABEL; +pub use rqe::RqeQueryError; use crate::data_model::{ - AggregationIdInfo, InferenceConfig, KeyByLabelValues, QueryBounds, QueryConfig, QueryLanguage, - StreamingConfig, + AggregationIdInfo, AggregationReference, InferenceConfig, KeyByLabelValues, QueryBounds, + QueryConfig, QueryLanguage, StreamingConfig, }; use crate::engines::query_plan::{ NodeId, PlanOptions, QueryPlan, QueryPlanExecutionError, QueryPlanNode, QueryPlanRuntime, @@ -691,6 +693,47 @@ impl SimpleEngine { )) } + /// Builds a `QueryExecutionContext` from already-resolved plain values: + /// plans the store reads and derives the grouping/aggregated labels from + /// the aggregation configs. Shared by the SQL and RQE context builders. + #[allow(clippy::too_many_arguments)] + fn build_execution_context_tail( + &self, + metric: &str, + timestamps: &QueryTimestamps, + metadata: QueryMetadata, + agg_info: AggregationIdInfo, + spatial_filter: String, + query_time: u64, + ) -> Result { + let (query_plan, do_merge, value_window_type) = + self.create_store_query_plan(metric, timestamps, &agg_info)?; + + let sc = self.streaming_config.read().unwrap().clone(); + let grouping_labels = sc + .get_aggregation_config(agg_info.aggregation_id_for_value) + .map(|config| config.grouping_labels.clone()) + .unwrap_or_else(|| metadata.query_output_labels.clone()); + + let aggregated_labels = sc + .get_aggregation_config(agg_info.aggregation_id_for_key) + .map(|config| config.aggregated_labels.clone()) + .unwrap_or_else(KeyByLabelNames::empty); + + Ok(QueryExecutionContext { + metric: metric.to_string(), + metadata, + store_plan: query_plan, + agg_info, + value_window_type, + do_merge, + spatial_filter, + query_time, + grouping_labels, + aggregated_labels, + }) + } + /// The bucket-map grid width for scanning an aggregation's stored /// buckets: `slide_interval_ms`, not `window_size_ms`. /// `precompute_engine/window_manager.rs` persists buckets on the @@ -1771,12 +1814,12 @@ impl SimpleEngine { stats.into_iter().next() } + /// Resolves the key and value aggregations a query reads. With two + /// aggregations, the set/delta-set one supplies keys and the other values. fn get_aggregation_id_info( &self, - query_config: &QueryConfig, + query_config_aggregations: &[AggregationReference], ) -> Result { - let query_config_aggregations = &query_config.aggregations; - if query_config_aggregations.is_empty() { return Err("Query config has no aggregations defined".to_string()); } diff --git a/asap-query-engine/src/engines/simple_engine/promql.rs b/asap-query-engine/src/engines/simple_engine/promql.rs index 0cdabfd3..36c48a96 100644 --- a/asap-query-engine/src/engines/simple_engine/promql.rs +++ b/asap-query-engine/src/engines/simple_engine/promql.rs @@ -51,7 +51,7 @@ fn detect_scalar_arm<'a>( /// two arms don't share the same label set — mirroring DataFusion's /// `build_binary_vector_plan`, which fails to resolve a join column that /// only exists on one side. -fn combine_vector_vector( +pub(super) fn combine_vector_vector( lhs_results: Vec, lhs_labels: &[String], rhs_results: Vec, @@ -83,7 +83,7 @@ fn combine_vector_vector( /// Scalar combiner for one binary-expr level (instant query): applies /// `op(scalar, value)` or `op(value, scalar)` per `scalar_on_left` to every /// element of the vector arm's results. -fn combine_scalar( +pub(super) fn combine_scalar( vector_results: Vec, scalar: f64, op: &promql_parser::parser::token::TokenType, @@ -119,7 +119,7 @@ impl SimpleEngine { /// boundary, unconditionally — mirroring SQL's `align_end_timestamp_sql`. /// A no-op in the common case (already-aligned timestamps), a safety net /// otherwise, for every PromQL query shape, not just a subset. - fn align_end_timestamp_promql(&self, end_timestamp: u64) -> u64 { + pub(super) fn align_end_timestamp_promql(&self, end_timestamp: u64) -> u64 { let interval_ms = self.data_ingestion_interval_ms; if end_timestamp.is_multiple_of(interval_ms) { return end_timestamp; @@ -286,7 +286,7 @@ impl SimpleEngine { let match_result = self.find_matching_controller_pattern(arm_ast, planned_subquery)?; let agg_info = self - .get_aggregation_id_info(query_config) + .get_aggregation_id_info(&query_config.aggregations) .map_err(|e| { warn!("{}", e); e @@ -1323,7 +1323,7 @@ impl SimpleEngine { // Resolve aggregation: try pre-configured query_configs first, fall back to capability matching. let agg_info: AggregationIdInfo = if let Some(config) = self.find_query_config(query) { - self.get_aggregation_id_info(&config) + self.get_aggregation_id_info(&config.aggregations) .map_err(|e| { warn!("{}", e); e diff --git a/asap-query-engine/src/engines/simple_engine/rqe.rs b/asap-query-engine/src/engines/simple_engine/rqe.rs new file mode 100644 index 00000000..69958235 --- /dev/null +++ b/asap-query-engine/src/engines/simple_engine/rqe.rs @@ -0,0 +1,318 @@ +//! Answers an RQE expression tree at one query time. A leaf reads the +//! aggregations its `rqe_configs` entry names (no query text is parsed), an +//! `agg` node regroups its child, and a `binary` node combines its sides the +//! way PromQL arithmetic does. + +use super::promql::{combine_scalar, combine_vector_vector}; +use super::{QueryExecutionError, QueryMetadata, QueryTimestamps, SimpleEngine}; +use crate::data_model::RqeConfig; +use crate::engines::query_result::{InstantVectorElement, QueryResult}; +use crate::engines::query_time_aggregation::{apply_instant_pipeline, pipeline_supports_labels}; +use asap_types::query_config::{ + QueryTimeAggregation, QueryTimeAggregationOperator, QueryTimeGrouping, QueryTimeGroupingMode, +}; +use asap_types::rqe_expr::{RqeAggOp, RqeBinaryOp, RqeCapability, RqeExpr, RqeLeaf}; +use promql_parser::parser::token::{TokenType, T_ADD, T_DIV, T_MUL, T_SUB}; +use promql_utilities::data_model::KeyByLabelNames; +use promql_utilities::query_logics::enums::Statistic; +use std::collections::HashMap; + +#[derive(Debug, thiserror::Error)] +pub enum RqeQueryError { + /// The request is wrong: an invalid tree, an unknown RQE id or a bad time. + #[error("{0}")] + BadRequest(String), + /// The request is valid but the engine could not answer it. + #[error(transparent)] + Execution(#[from] QueryExecutionError), +} + +/// A node's result: an instant vector with its label names, or a literal. +enum RqeValue { + Vector { + labels: KeyByLabelNames, + values: Vec, + }, + Scalar(f64), +} + +impl SimpleEngine { + /// Evaluates `expr` at `time` (unix seconds) into an instant vector. + /// + /// Output labels are the root's labels (see `RqeExpr::output_labels`), + /// never `__name__`: every leaf reads a range like `sum_over_time`, whose + /// result Prometheus strips of its name. A leaf whose window holds no + /// data yet contributes an empty vector. + pub fn handle_rqe_query( + &self, + expr: &RqeExpr, + time: f64, + ) -> Result<(KeyByLabelNames, QueryResult), RqeQueryError> { + expr.validate() + .map_err(|error| RqeQueryError::BadRequest(error.to_string()))?; + if !time.is_finite() || time < 0.0 { + return Err(RqeQueryError::BadRequest(format!( + "time must be a non-negative number of unix seconds, got {time}" + ))); + } + let leaves = expr.leaves(); + if leaves.is_empty() { + return Err(RqeQueryError::BadRequest( + "an RQE expression must read at least one RQE".to_string(), + )); + } + // One snapshot of the configs, so every leaf reads the same plan. + let configs: HashMap<&str, RqeConfig> = { + let inference_config = self.inference_config.read().unwrap(); + leaves + .iter() + .map(|leaf| { + inference_config + .rqe_configs + .get(&leaf.id) + .map(|config| (leaf.id.as_str(), config.clone())) + .ok_or_else(|| { + RqeQueryError::BadRequest(format!("unknown RQE id {:?}", leaf.id)) + }) + }) + .collect::>()? + }; + + let query_time = Self::convert_query_time_to_data_time(time); + match self.evaluate_rqe_node(expr, &configs, query_time)? { + RqeValue::Vector { labels, values } => { + Ok((labels, QueryResult::vector(values, query_time))) + } + RqeValue::Scalar(_) => Err(RqeQueryError::BadRequest( + "an RQE expression must evaluate to a vector".to_string(), + )), + } + } + + fn evaluate_rqe_node( + &self, + expr: &RqeExpr, + configs: &HashMap<&str, RqeConfig>, + query_time: u64, + ) -> Result { + match expr { + RqeExpr::Rqe { rqe } => { + let (labels, values) = + self.evaluate_rqe_leaf(rqe, &configs[rqe.id.as_str()], query_time)?; + Ok(RqeValue::Vector { labels, values }) + } + RqeExpr::Agg { agg, by, of } => { + let RqeValue::Vector { labels, values } = + self.evaluate_rqe_node(of, configs, query_time)? + else { + return Err(RqeQueryError::BadRequest( + "an agg node needs a vector operand, not a scalar".to_string(), + )); + }; + let (labels, values) = + apply_instant_pipeline(labels, values, &[regroup_stage(*agg, by)]) + .map_err(QueryExecutionError::Native)?; + Ok(RqeValue::Vector { labels, values }) + } + RqeExpr::Binary { op, lhs, rhs } => { + let op = binary_token(*op); + let lhs = self.evaluate_rqe_node(lhs, configs, query_time)?; + let rhs = self.evaluate_rqe_node(rhs, configs, query_time)?; + match (lhs, rhs) { + ( + RqeValue::Vector { + labels, + values: lhs_values, + }, + RqeValue::Vector { + labels: rhs_labels, + values: rhs_values, + }, + ) => { + let values = combine_vector_vector( + lhs_values, + &labels.labels, + rhs_values, + &rhs_labels.labels, + &op, + ) + .ok_or_else(|| { + RqeQueryError::BadRequest(format!( + "binary op sides have different labels: {:?} vs {:?}", + labels.labels, rhs_labels.labels + )) + })?; + Ok(RqeValue::Vector { labels, values }) + } + (RqeValue::Vector { labels, values }, RqeValue::Scalar(scalar)) => { + Ok(RqeValue::Vector { + labels, + values: combine_scalar(values, scalar, &op, false), + }) + } + (RqeValue::Scalar(scalar), RqeValue::Vector { labels, values }) => { + Ok(RqeValue::Vector { + labels, + values: combine_scalar(values, scalar, &op, true), + }) + } + (RqeValue::Scalar(_), RqeValue::Scalar(_)) => Err(RqeQueryError::BadRequest( + "binary op needs at least one vector side".to_string(), + )), + } + } + RqeExpr::Scalar { scalar } => Ok(RqeValue::Scalar(*scalar)), + } + } + + /// Reads `leaf` from the aggregations `config` names, over the window + /// ending at `query_time` aligned as a PromQL instant query aligns it. + /// A roll-up config reads at the aggregation's own grouping, then its + /// query-time stages regroup to the leaf's. + fn evaluate_rqe_leaf( + &self, + leaf: &RqeLeaf, + config: &RqeConfig, + query_time: u64, + ) -> Result<(KeyByLabelNames, Vec), RqeQueryError> { + let failed = + |error: String| QueryExecutionError::Native(format!("RQE {:?}: {error}", leaf.id)); + let agg_info = self + .get_aggregation_id_info(&config.aggregations) + .map_err(failed)?; + let streaming_config = self.streaming_config.read().unwrap().clone(); + let value_config = streaming_config + .get_aggregation_config(agg_info.aggregation_id_for_value) + .ok_or_else(|| { + failed(format!( + "no streaming config for aggregation_id {}", + agg_info.aggregation_id_for_value + )) + })?; + + let end_timestamp = self.align_end_timestamp_promql(query_time); + let start_timestamp = end_timestamp.checked_sub(leaf.lookback_ms).ok_or_else(|| { + RqeQueryError::BadRequest(format!( + "RQE {:?}: its {} ms lookback starts before the unix epoch", + leaf.id, leaf.lookback_ms + )) + })?; + + let leaf_labels = KeyByLabelNames::new(leaf.grouping.clone()); + let read_labels = if config.query_time_aggregations.is_empty() { + leaf_labels.clone() + } else { + let mut labels = value_config.grouping_labels.labels.clone(); + labels.extend(value_config.aggregated_labels.labels.iter().cloned()); + KeyByLabelNames::new(labels) + }; + let (statistic_to_compute, query_kwargs) = leaf_statistic(leaf)?; + let metadata = QueryMetadata { + query_output_labels: read_labels, + statistic_to_compute, + query_kwargs, + keep_metric_name: false, + }; + let context = self + .build_execution_context_tail( + &value_config.metric, + &QueryTimestamps { + start_timestamp, + end_timestamp, + }, + metadata, + agg_info, + String::new(), + query_time, + ) + .map_err(failed)?; + + let Some((labels, result)) = self.execute_context_result(context, true, true)? else { + return Ok((leaf_labels, Vec::new())); + }; + let QueryResult::Vector(vector) = result else { + return Err(failed("an instant read returned a range result".to_string()).into()); + }; + let (labels, values) = if config.query_time_aggregations.is_empty() { + (labels, vector.values) + } else { + if !pipeline_supports_labels(&labels, &config.query_time_aggregations) + .map_err(failed)? + { + return Err(failed(format!( + "query-time aggregations can't regroup labels {:?}", + labels.labels + )) + .into()); + } + apply_instant_pipeline(labels, vector.values, &config.query_time_aggregations) + .map_err(failed)? + }; + if labels != leaf_labels { + return Err(failed(format!( + "read labels {:?}, not the leaf's grouping {:?}", + labels.labels, leaf_labels.labels + )) + .into()); + } + Ok((labels, values)) + } +} + +/// The statistic and its kwargs a leaf's capability computes. Top-k is a +/// single global ranking: a leaf has no per-group bucketing field. +fn leaf_statistic(leaf: &RqeLeaf) -> Result<(Statistic, HashMap), RqeQueryError> { + let missing = + |field: &str| RqeQueryError::BadRequest(format!("RQE {:?}: missing {field}", leaf.id)); + Ok(match leaf.capability { + RqeCapability::Sum => (Statistic::Sum, HashMap::new()), + RqeCapability::Cardinality => (Statistic::Cardinality, HashMap::new()), + RqeCapability::Quantile => { + let phi = leaf.phi.ok_or_else(|| missing("phi"))?; + ( + Statistic::Quantile, + HashMap::from([("quantile".to_string(), phi.to_string())]), + ) + } + RqeCapability::TopkByValue => { + let k = leaf.k.ok_or_else(|| missing("k"))?; + ( + Statistic::Topk, + HashMap::from([("k".to_string(), k.to_string())]), + ) + } + }) +} + +/// The query-time stage that regroups a vector by `by` (all into one group +/// when `by` is empty). +fn regroup_stage(op: RqeAggOp, by: &[String]) -> QueryTimeAggregation { + let operator = match op { + RqeAggOp::Avg => QueryTimeAggregationOperator::Avg, + RqeAggOp::Min => QueryTimeAggregationOperator::Min, + RqeAggOp::Max => QueryTimeAggregationOperator::Max, + RqeAggOp::Sum => QueryTimeAggregationOperator::Sum, + }; + let mode = if by.is_empty() { + QueryTimeGroupingMode::All + } else { + QueryTimeGroupingMode::By + }; + QueryTimeAggregation { + operator, + grouping: QueryTimeGrouping { + mode, + labels: by.to_vec(), + }, + parameter: None, + } +} + +fn binary_token(op: RqeBinaryOp) -> TokenType { + TokenType::new(match op { + RqeBinaryOp::Add => T_ADD, + RqeBinaryOp::Sub => T_SUB, + RqeBinaryOp::Mul => T_MUL, + RqeBinaryOp::Div => T_DIV, + }) +} diff --git a/asap-query-engine/src/engines/simple_engine/sql.rs b/asap-query-engine/src/engines/simple_engine/sql.rs index dda452bc..da37272c 100644 --- a/asap-query-engine/src/engines/simple_engine/sql.rs +++ b/asap-query-engine/src/engines/simple_engine/sql.rs @@ -369,55 +369,6 @@ impl SimpleEngine { Some((ctx, post)) } - /// Shared context-building tail for both SQL context builders. - /// - /// Called by `build_query_execution_context_sql` and `build_spatiotemporal_context` - /// after labels, statistic, metadata, timestamps, and `agg_info` are resolved. - /// Builds the query plan, derives grouping/aggregated labels, and returns the - /// final `QueryExecutionContext`. - #[allow(clippy::too_many_arguments)] - fn build_sql_execution_context_tail( - &self, - metric: &str, - timestamps: &QueryTimestamps, - metadata: QueryMetadata, - agg_info: AggregationIdInfo, - spatial_filter: String, - query_time: u64, - ) -> Option { - let (query_plan, do_merge, value_window_type) = self - .create_store_query_plan(metric, timestamps, &agg_info) - .map_err(|e| { - warn!("Failed to create store query plan: {}", e); - e - }) - .ok()?; - - let sc = self.streaming_config.read().unwrap().clone(); - let grouping_labels = sc - .get_aggregation_config(agg_info.aggregation_id_for_value) - .map(|config| config.grouping_labels.clone()) - .unwrap_or_else(|| metadata.query_output_labels.clone()); - - let aggregated_labels = sc - .get_aggregation_config(agg_info.aggregation_id_for_key) - .map(|config| config.aggregated_labels.clone()) - .unwrap_or_else(KeyByLabelNames::empty); - - Some(QueryExecutionContext { - metric: metric.to_string(), - metadata, - store_plan: query_plan, - agg_info, - value_window_type, - do_merge, - spatial_filter, - query_time, - grouping_labels, - aggregated_labels, - }) - } - /// Build execution context for SpatioTemporal queries. /// These queries span multiple scrape intervals but GROUP BY a subset of labels. fn build_spatiotemporal_context( @@ -492,7 +443,7 @@ impl SimpleEngine { let agg_info: AggregationIdInfo = if let Some(config) = self.find_query_config_sql(query_data) { - self.get_aggregation_id_info(&config) + self.get_aggregation_id_info(&config.aggregations) .map_err(|e| { warn!("{}", e); e @@ -512,7 +463,7 @@ impl SimpleEngine { }; let metric = &match_result.outer_data()?.metric; - self.build_sql_execution_context_tail( + self.build_execution_context_tail( metric, ×tamps, metadata, @@ -520,6 +471,8 @@ impl SimpleEngine { String::new(), query_time, ) + .map_err(|e| warn!("Failed to create store query plan: {}", e)) + .ok() } } diff --git a/asap-query-engine/src/tests/mod.rs b/asap-query-engine/src/tests/mod.rs index 47cab84a..44f78544 100644 --- a/asap-query-engine/src/tests/mod.rs +++ b/asap-query-engine/src/tests/mod.rs @@ -13,6 +13,7 @@ pub mod query_equivalence_tests; pub mod query_time_aggregation_public_tests; pub mod range_query_arithmetic_tests; pub mod rollup_tests; +pub mod rqe_query_tests; pub mod sql_pattern_matching_tests; pub mod stage_e_instant_range_equivalence_tests; pub mod store_correctness_tests; diff --git a/asap-query-engine/src/tests/rqe_query_tests.rs b/asap-query-engine/src/tests/rqe_query_tests.rs new file mode 100644 index 00000000..fc66bfa2 --- /dev/null +++ b/asap-query-engine/src/tests/rqe_query_tests.rs @@ -0,0 +1,830 @@ +//! RQE expression trees answered by `SimpleEngine::handle_rqe_query` and the +//! `POST /api/v1/rqe_query` route. + +#[cfg(test)] +mod tests { + use crate::data_model::{ + AggregationConfig, AggregationReference, AggregationType, CleanupPolicy, InferenceConfig, + KeyByLabelValues, PrecomputedOutput, PromQLSchema, QueryConfig, QueryLanguage, RqeConfig, + SchemaConfig, StreamingConfig, WindowType, + }; + use crate::drivers::query::adapters::AdapterConfig; + use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig, RQE_QUERY_ENDPOINT}; + use crate::engines::{QueryResult, RqeQueryError, SimpleEngine}; + use crate::precompute_operators::{ + CountMinSketchWithHeapAccumulator, DDSketchAccumulator, HllAccumulator, + MultipleSumAccumulator, SumAccumulator, + }; + use crate::stores::simple_map_store::SimpleMapStore; + use crate::stores::Store; + use crate::AggregateCore; + use asap_types::query_config::{ + QueryTimeAggregation, QueryTimeAggregationOperator, QueryTimeGrouping, + QueryTimeGroupingMode, + }; + use asap_types::rqe_expr::RqeExpr; + use promql_utilities::data_model::KeyByLabelNames; + use serde_json::{json, Value}; + use std::collections::HashMap; + use std::sync::Arc; + + const WINDOW_MS: u64 = 1_000; + const QUERY_TIME_S: f64 = 1_000.0; + const QUERY_TIME_MS: u64 = 1_000_000; + /// Two tumbling windows: ending one window before the query time, and at it. + const TWO_WINDOWS_MS: u64 = 2 * WINDOW_MS; + + const SUM_AGG: u64 = 1; + const DDSKETCH_AGG: u64 = 2; + const HLL_AGG: u64 = 3; + const TOPK_AGG: u64 = 4; + const MULTI_SUM_AGG: u64 = 5; + + type Row = (Vec, f64); + + fn strings(values: &[&str]) -> Vec { + values.iter().map(|value| value.to_string()).collect() + } + + fn aggregation( + id: u64, + aggregation_type: AggregationType, + metric: &str, + grouping: &[&str], + aggregated: &[&str], + ) -> (u64, AggregationConfig) { + ( + id, + AggregationConfig { + aggregation_id: id, + aggregation_type, + aggregation_sub_type: String::new(), + parameters: HashMap::new(), + grouping_labels: KeyByLabelNames::new(strings(grouping)), + aggregated_labels: KeyByLabelNames::new(strings(aggregated)), + rollup_labels: KeyByLabelNames::empty(), + original_yaml: String::new(), + window_size_ms: WINDOW_MS, + slide_interval_ms: WINDOW_MS, + window_type: WindowType::Tumbling, + spatial_filter: String::new(), + spatial_filter_normalized: String::new(), + metric: metric.to_string(), + num_aggregates_to_retain: None, + read_count_threshold: None, + table_name: None, + value_column: None, + }, + ) + } + + fn sum_by_job() -> QueryTimeAggregation { + QueryTimeAggregation { + operator: QueryTimeAggregationOperator::Sum, + grouping: QueryTimeGrouping { + mode: QueryTimeGroupingMode::By, + labels: strings(&["job"]), + }, + parameter: None, + } + } + + fn rqe_config(aggregation_id: u64, stages: Vec) -> RqeConfig { + RqeConfig { + aggregations: vec![AggregationReference::new(aggregation_id, None)], + query_time_aggregations: stages, + } + } + + fn query_config(query: &str, aggregation_id: u64) -> QueryConfig { + QueryConfig::new(query.to_string()) + .add_aggregation(AggregationReference::new(aggregation_id, None)) + } + + /// DDSketch, not KLL: its merge is deterministic, so two reads of the + /// same windows agree exactly. + fn ddsketch(values: impl IntoIterator) -> DDSketchAccumulator { + let mut sketch = DDSketchAccumulator::new(0.01); + values.into_iter().for_each(|value| sketch.update(value)); + sketch + } + + fn hll(values: std::ops::Range) -> HllAccumulator { + let mut sketch = HllAccumulator::with_default_precision(); + values.for_each(|value| sketch.update(f64::from(value))); + sketch + } + + fn multiple_sum(sums: &[(&[&str], f64)]) -> MultipleSumAccumulator { + let mut accumulator = MultipleSumAccumulator::new(); + for (labels, sum) in sums { + accumulator.add_sum(KeyByLabelValues::new_with_labels(strings(labels)), *sum); + } + accumulator + } + + /// One engine holding every aggregation the tests read, each RQE id in + /// `rqe_configs`, and the PromQL queries the equivalence tests compare + /// against. + fn engine() -> SimpleEngine { + let streaming_config = Arc::new(StreamingConfig { + aggregation_configs: HashMap::from([ + aggregation( + SUM_AGG, + AggregationType::Sum, + "http_requests", + &["instance", "job"], + &[], + ), + aggregation( + DDSKETCH_AGG, + AggregationType::DDSketch, + "latency", + &["job"], + &[], + ), + aggregation(HLL_AGG, AggregationType::HLL, "users", &["job"], &[]), + aggregation( + TOPK_AGG, + AggregationType::CountMinSketchWithHeap, + "transfers", + &[], + &["srcip"], + ), + aggregation( + MULTI_SUM_AGG, + AggregationType::MultipleSum, + "events", + &[], + &["instance", "job"], + ), + ]), + }); + let store = Arc::new(SimpleMapStore::new( + streaming_config.clone(), + CleanupPolicy::NoCleanup, + )); + let insert = |aggregation_id: u64, + end_ms: u64, + key: Option<&[&str]>, + accumulator: Box| { + let key = key.map(|labels| KeyByLabelValues::new_with_labels(strings(labels))); + store + .insert_precomputed_output( + PrecomputedOutput::new(end_ms - WINDOW_MS, end_ms, key, aggregation_id), + accumulator, + ) + .unwrap(); + }; + let previous_window_ms = QUERY_TIME_MS - WINDOW_MS; + + // Sums: (a, api) 1 then 2, (b, api) 3 then 4, (a, web) 5 then 6. + for (labels, earlier, latest) in [ + (["a", "api"], 1.0, 2.0), + (["b", "api"], 3.0, 4.0), + (["a", "web"], 5.0, 6.0), + ] { + insert( + SUM_AGG, + previous_window_ms, + Some(&labels), + Box::new(SumAccumulator::with_sum(earlier)), + ); + insert( + SUM_AGG, + QUERY_TIME_MS, + Some(&labels), + Box::new(SumAccumulator::with_sum(latest)), + ); + } + + for (job, offset) in [("api", 0.0), ("web", 1_000.0)] { + insert( + DDSKETCH_AGG, + previous_window_ms, + Some(&[job]), + Box::new(ddsketch((0..500).map(|v| f64::from(v) + offset))), + ); + insert( + DDSKETCH_AGG, + QUERY_TIME_MS, + Some(&[job]), + Box::new(ddsketch((500..1_000).map(|v| f64::from(v) + offset))), + ); + } + + insert( + HLL_AGG, + QUERY_TIME_MS, + Some(&["api"]), + Box::new(hll(0..1_000)), + ); + insert(HLL_AGG, QUERY_TIME_MS, Some(&["web"]), Box::new(hll(0..50))); + + let mut heap = CountMinSketchWithHeapAccumulator::new(3, 1024, 32); + for i in 1..=5u64 { + heap.inner.update(&format!("10.0.0.{i}"), (i * 10) as f64); + } + insert(TOPK_AGG, QUERY_TIME_MS, None, Box::new(heap)); + + insert( + MULTI_SUM_AGG, + previous_window_ms, + None, + Box::new(multiple_sum(&[(&["a", "api"], 1.0), (&["b", "api"], 2.0)])), + ); + insert( + MULTI_SUM_AGG, + QUERY_TIME_MS, + None, + Box::new(multiple_sum(&[(&["a", "api"], 4.0), (&["a", "web"], 8.0)])), + ); + + let schema = PromQLSchema::new() + .add_metric( + "http_requests".to_string(), + KeyByLabelNames::new(strings(&["instance", "job"])), + ) + .add_metric( + "latency".to_string(), + KeyByLabelNames::new(strings(&["job"])), + ) + .add_metric("users".to_string(), KeyByLabelNames::new(strings(&["job"]))) + .add_metric( + "transfers".to_string(), + KeyByLabelNames::new(strings(&["srcip"])), + ) + .add_metric( + "events".to_string(), + KeyByLabelNames::new(strings(&["instance", "job"])), + ); + let inference_config = InferenceConfig { + schema: SchemaConfig::PromQL(schema), + query_configs: vec![ + query_config("sum_over_time(http_requests[2s])", SUM_AGG), + query_config("sum by (job) (sum_over_time(http_requests[2s]))", SUM_AGG), + query_config("quantile_over_time(0.9, latency[2s])", DDSKETCH_AGG), + query_config("topk(3, sum_over_time(transfers[1s]))", TOPK_AGG), + QueryConfig::with_plan( + "sum by (job) (sum_over_time(events[2s]))".to_string(), + "sum by (instance, job) (sum_over_time(events[2s]))".to_string(), + vec![sum_by_job()], + ) + .add_aggregation(AggregationReference::new(MULTI_SUM_AGG, None)), + ], + cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: HashMap::from([ + ("sum".to_string(), rqe_config(SUM_AGG, vec![])), + ("sum_by_job".to_string(), rqe_config(SUM_AGG, vec![])), + ("latest".to_string(), rqe_config(SUM_AGG, vec![])), + ("p90".to_string(), rqe_config(DDSKETCH_AGG, vec![])), + ("distinct".to_string(), rqe_config(HLL_AGG, vec![])), + ("top3".to_string(), rqe_config(TOPK_AGG, vec![])), + ( + "rolled_up".to_string(), + rqe_config(MULTI_SUM_AGG, vec![sum_by_job()]), + ), + ( + // Rolls up to `job` though its leaves group by `instance`. + "misrolled".to_string(), + rqe_config(MULTI_SUM_AGG, vec![sum_by_job()]), + ), + ]), + }; + SimpleEngine::new( + store, + inference_config, + streaming_config, + WINDOW_MS, + QueryLanguage::promql, + ) + } + + fn leaf( + id: &str, + metric: &str, + capability: &str, + lookback_ms: u64, + grouping: &[&str], + ) -> Value { + json!({"rqe": {"id": id, "metric": metric, "capability": capability, + "lookback_ms": lookback_ms, "grouping": grouping}}) + } + + /// `sum_over_time(http_requests[2s])`, or its `sum by` when `grouping` + /// drops a label. + fn sum_leaf(id: &str, lookback_ms: u64, grouping: &[&str]) -> Value { + leaf(id, "http_requests", "sum", lookback_ms, grouping) + } + + fn series_sum() -> Value { + sum_leaf("sum", TWO_WINDOWS_MS, &["instance", "job"]) + } + + fn latest_sum() -> Value { + sum_leaf("latest", WINDOW_MS, &["instance", "job"]) + } + + fn quantile_leaf() -> Value { + let mut quantile = leaf("p90", "latency", "quantile", TWO_WINDOWS_MS, &["job"]); + quantile["rqe"]["phi"] = json!(0.9); + quantile + } + + fn topk_leaf() -> Value { + let mut topk = leaf("top3", "transfers", "topk_by_value", WINDOW_MS, &["srcip"]); + topk["rqe"]["k"] = json!(3); + topk + } + + fn expr(tree: Value) -> RqeExpr { + serde_json::from_value(tree).expect("a well-formed RQE tree") + } + + /// Label names and rows, sorted by label values. + fn sorted(labels: KeyByLabelNames, result: QueryResult) -> (Vec, Vec) { + let QueryResult::Vector(vector) = result else { + panic!("expected an instant vector"); + }; + let mut rows: Vec = vector + .values + .into_iter() + .map(|element| (element.labels.labels, element.value)) + .collect(); + rows.sort_by(|left, right| left.0.cmp(&right.0)); + (labels.labels, rows) + } + + fn rqe(engine: &SimpleEngine, tree: Value) -> (Vec, Vec) { + let (labels, result) = engine + .handle_rqe_query(&expr(tree), QUERY_TIME_S) + .expect("the RQE query should be answered"); + sorted(labels, result) + } + + fn promql(engine: &SimpleEngine, query: &str) -> (Vec, Vec) { + let (labels, result) = engine + .handle_query_promql(query.to_string(), QUERY_TIME_S) + .expect("the PromQL query should execute") + .expect("the PromQL query should be served locally"); + sorted(labels, result) + } + + fn rqe_error(engine: &SimpleEngine, tree: Value, time: f64) -> RqeQueryError { + engine + .handle_rqe_query(&expr(tree), time) + .expect_err("the RQE query should be refused") + } + + fn row(labels: &[&str], value: f64) -> Row { + (strings(labels), value) + } + + // --- leaves --- + + #[test] + fn sum_leaf_sums_its_lookback_per_series_without_metric_name() { + assert_eq!( + rqe(&engine(), series_sum()), + ( + strings(&["instance", "job"]), + vec![ + row(&["a", "api"], 3.0), + row(&["a", "web"], 11.0), + row(&["b", "api"], 7.0), + ] + ) + ); + } + + #[test] + fn sum_leaf_equals_promql_sum_over_time() { + let engine = engine(); + assert_eq!( + rqe(&engine, series_sum()), + promql(&engine, "sum_over_time(http_requests[2s])") + ); + } + + #[test] + fn grouped_sum_leaf_equals_promql_sum_by_of_sum_over_time() { + let engine = engine(); + let answer = rqe(&engine, sum_leaf("sum_by_job", TWO_WINDOWS_MS, &["job"])); + assert_eq!( + answer, + ( + strings(&["job"]), + vec![row(&["api"], 10.0), row(&["web"], 11.0)] + ) + ); + assert_eq!( + answer, + promql(&engine, "sum by (job) (sum_over_time(http_requests[2s]))") + ); + } + + #[test] + fn quantile_leaf_equals_promql_quantile_over_time() { + let engine = engine(); + let answer = rqe(&engine, quantile_leaf()); + assert_eq!(answer.0, strings(&["job"])); + assert_eq!(answer.1.len(), 2); + for ((labels, value), expected) in answer.1.iter().zip([900.0, 1_900.0]) { + assert!( + (value - expected).abs() <= 1_000.0 * 0.02, + "{labels:?}: p90 {value} should be near {expected}" + ); + } + assert_eq!( + answer, + promql(&engine, "quantile_over_time(0.9, latency[2s])") + ); + } + + #[test] + fn cardinality_leaf_estimates_distinct_values_per_group() { + let answer = rqe( + &engine(), + leaf("distinct", "users", "cardinality", WINDOW_MS, &["job"]), + ); + assert_eq!( + answer, + ( + strings(&["job"]), + vec![ + row(&["api"], hll(0..1_000).estimate()), + row(&["web"], hll(0..50).estimate()), + ] + ) + ); + } + + #[test] + fn topk_leaf_ranks_globally_and_equals_promql_topk_of_sum_over_time() { + let engine = engine(); + let (labels, result) = engine + .handle_rqe_query(&expr(topk_leaf()), QUERY_TIME_S) + .unwrap(); + let QueryResult::Vector(vector) = &result else { + panic!("expected an instant vector"); + }; + let ranked: Vec = vector + .values + .iter() + .map(|element| (element.labels.labels.clone(), element.value)) + .collect(); + assert_eq!(labels.labels, strings(&["srcip"])); + assert_eq!( + ranked, + vec![ + row(&["10.0.0.5"], 50.0), + row(&["10.0.0.4"], 40.0), + row(&["10.0.0.3"], 30.0), + ], + "top-k rows come back ranked, highest first" + ); + assert_eq!( + sorted(labels, result), + promql(&engine, "topk(3, sum_over_time(transfers[1s]))") + ); + } + + #[test] + fn rolled_up_leaf_applies_its_query_time_stage_like_the_promql_plan() { + let engine = engine(); + let answer = rqe( + &engine, + leaf("rolled_up", "events", "sum", TWO_WINDOWS_MS, &["job"]), + ); + assert_eq!( + answer, + ( + strings(&["job"]), + vec![row(&["api"], 7.0), row(&["web"], 8.0)] + ) + ); + assert_eq!( + answer, + promql(&engine, "sum by (job) (sum_over_time(events[2s]))") + ); + } + + #[test] + fn a_roll_up_to_other_labels_than_the_leaf_fails_loudly() { + let error = rqe_error( + &engine(), + leaf("misrolled", "events", "sum", TWO_WINDOWS_MS, &["instance"]), + QUERY_TIME_S, + ); + assert!(matches!(&error, RqeQueryError::Execution(_)), "{error:?}"); + assert!(error.to_string().contains("misrolled"), "{error}"); + } + + /// Between ingestion ticks a leaf reads what the PromQL instant path + /// reads at the same time. + #[test] + fn misaligned_time_answers_like_the_promql_path() { + let engine = engine(); + let time = QUERY_TIME_S + 0.4; + let (labels, result) = engine.handle_rqe_query(&expr(series_sum()), time).unwrap(); + let promql_rows = engine + .handle_query_promql("sum_over_time(http_requests[2s])".to_string(), time) + .expect("the PromQL query should execute") + .map(|(labels, result)| sorted(labels, result).1) + .unwrap_or_default(); + assert_eq!(sorted(labels, result).1, promql_rows); + } + + // --- agg --- + + #[test] + fn agg_regroups_its_child_by_each_operator() { + let engine = engine(); + for (op, api, web) in [ + ("avg", 5.0, 11.0), + ("min", 3.0, 11.0), + ("max", 7.0, 11.0), + ("sum", 10.0, 11.0), + ] { + assert_eq!( + rqe( + &engine, + json!({"agg": op, "by": ["job"], "of": series_sum()}) + ), + ( + strings(&["job"]), + vec![row(&["api"], api), row(&["web"], web)] + ), + "{op}" + ); + } + } + + #[test] + fn agg_by_nothing_folds_every_series_into_one() { + assert_eq!( + rqe( + &engine(), + json!({"agg": "avg", "by": [], "of": series_sum()}) + ), + (vec![], vec![row(&[], 7.0)]) + ); + } + + // --- binary --- + + #[test] + fn binary_vector_vector_matches_series_by_labels() { + let engine = engine(); + assert_eq!( + rqe( + &engine, + json!({"op": "-", "lhs": series_sum(), "rhs": latest_sum()}) + ), + ( + strings(&["instance", "job"]), + vec![ + row(&["a", "api"], 1.0), + row(&["a", "web"], 5.0), + row(&["b", "api"], 3.0), + ] + ) + ); + assert_eq!( + rqe( + &engine, + json!({"op": "/", "lhs": series_sum(), "rhs": latest_sum()}) + ) + .1, + vec![ + row(&["a", "api"], 1.5), + row(&["a", "web"], 11.0 / 6.0), + row(&["b", "api"], 7.0 / 4.0), + ] + ); + } + + #[test] + fn binary_vector_scalar_keeps_operand_order() { + let engine = engine(); + assert_eq!( + rqe( + &engine, + json!({"op": "*", "lhs": latest_sum(), "rhs": {"scalar": 2.0}}) + ) + .1, + vec![ + row(&["a", "api"], 4.0), + row(&["a", "web"], 12.0), + row(&["b", "api"], 8.0), + ] + ); + assert_eq!( + rqe( + &engine, + json!({"op": "-", "lhs": {"scalar": 10.0}, "rhs": latest_sum()}) + ), + ( + strings(&["instance", "job"]), + vec![ + row(&["a", "api"], 8.0), + row(&["a", "web"], 4.0), + row(&["b", "api"], 6.0), + ] + ) + ); + } + + #[test] + fn nested_tree_aggregates_a_binary_of_leaves() { + let tree = json!({"agg": "max", "by": ["job"], "of": + {"op": "+", "lhs": series_sum(), "rhs": latest_sum()}}); + assert_eq!( + rqe(&engine(), tree), + ( + strings(&["job"]), + vec![row(&["api"], 11.0), row(&["web"], 17.0)] + ) + ); + } + + // --- refusals and empty windows --- + + #[test] + fn refuses_an_invalid_tree() { + let mut quantile = quantile_leaf(); + quantile["rqe"]["phi"] = json!(1.5); + let error = rqe_error(&engine(), quantile, QUERY_TIME_S); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{error:?}"); + assert!(error.to_string().contains("phi"), "{error}"); + } + + #[test] + fn refuses_an_unknown_rqe_id_before_reading_anything() { + let tree = json!({"op": "+", "lhs": series_sum(), + "rhs": sum_leaf("nope", WINDOW_MS, &["instance", "job"])}); + let error = rqe_error(&engine(), tree, QUERY_TIME_S); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{error:?}"); + assert!(error.to_string().contains("\"nope\""), "{error}"); + } + + #[test] + fn refuses_a_bad_time() { + for time in [-1.0, f64::NAN, f64::INFINITY] { + let error = rqe_error(&engine(), series_sum(), time); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{time}"); + } + // A lookback reaching before the epoch. + let error = rqe_error(&engine(), series_sum(), 1.0); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{error:?}"); + } + + #[test] + fn refuses_trees_without_a_vector() { + let engine = engine(); + let error = rqe_error(&engine, json!({"scalar": 1.0}), QUERY_TIME_S); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{error:?}"); + let agg_of_scalar = json!({"op": "+", + "lhs": {"agg": "sum", "by": [], "of": {"scalar": 1.0}}, + "rhs": {"agg": "sum", "by": [], "of": series_sum()}}); + let error = rqe_error(&engine, agg_of_scalar, QUERY_TIME_S); + assert!(matches!(error, RqeQueryError::BadRequest(_)), "{error:?}"); + } + + #[test] + fn a_window_without_data_yet_answers_empty() { + let engine = engine(); + let later = QUERY_TIME_S + 3_600.0; + for tree in [ + series_sum(), + topk_leaf(), + json!({"agg": "sum", "by": ["job"], "of": series_sum()}), + json!({"op": "*", "lhs": series_sum(), "rhs": {"scalar": 2.0}}), + ] { + let parsed = expr(tree.clone()); + let (labels, result) = engine.handle_rqe_query(&parsed, later).unwrap(); + let (labels, rows) = sorted(labels, result); + assert!(rows.is_empty(), "{tree}: {rows:?}"); + assert_eq!(labels, parsed.output_labels(), "{tree}"); + } + } + + // --- HTTP route --- + + async fn server() -> (HttpServer, String) { + let engine = Arc::new(engine()); + let store: Arc = Arc::new(SimpleMapStore::new( + Arc::new(StreamingConfig::default()), + CleanupPolicy::NoCleanup, + )); + let config = HttpServerConfig { + port: 0, + handle_http_requests: true, + // Forwarding on, so a test would see any fallback attempt fail. + adapter_config: AdapterConfig::prometheus_promql( + "http://127.0.0.1:1".to_string(), + true, + 30, + ), + }; + let server = HttpServer::new(config, engine, store, None); + let port = server.start_test_server().await.unwrap(); + ( + server, + format!("http://127.0.0.1:{port}{RQE_QUERY_ENDPOINT}"), + ) + } + + async fn post(url: &str, body: String) -> (u16, Value) { + let response = reqwest::Client::new() + .post(url) + .header("content-type", "application/json") + .body(body) + .send() + .await + .unwrap(); + let status = response.status().as_u16(); + (status, response.json().await.unwrap()) + } + + #[tokio::test] + async fn route_answers_prometheus_vector_json() { + let (_server, url) = server().await; + let body = json!({"expr": {"agg": "sum", "by": ["job"], "of": series_sum()}, + "time": QUERY_TIME_S}); + let (status, response) = post(&url, body.to_string()).await; + assert_eq!(status, 200); + assert_eq!(response["status"], "success"); + assert_eq!(response["data"]["resultType"], "vector"); + let mut result = response["data"]["result"].as_array().unwrap().clone(); + result.sort_by_key(|series| series["metric"]["job"].as_str().unwrap().to_string()); + assert_eq!( + result, + vec![ + json!({"metric": {"job": "api"}, "value": [QUERY_TIME_S, "10"]}), + json!({"metric": {"job": "web"}, "value": [QUERY_TIME_S, "11"]}), + ] + ); + } + + #[tokio::test] + async fn route_answers_an_empty_window_with_an_empty_result() { + let (_server, url) = server().await; + let body = json!({"expr": series_sum(), "time": QUERY_TIME_S + 3_600.0}); + let (status, response) = post(&url, body.to_string()).await; + assert_eq!(status, 200); + assert_eq!( + response, + json!({"status": "success", "data": {"resultType": "vector", "result": []}}) + ); + } + + #[tokio::test] + async fn route_refuses_bad_requests_with_400_bad_data() { + let (_server, url) = server().await; + let mut bad_quantile = quantile_leaf(); + bad_quantile["rqe"]["phi"] = json!(2.0); + for (body, reason) in [ + ("not json".to_string(), "invalid RQE query body"), + ( + json!({"expr": series_sum()}).to_string(), + "missing field `time`", + ), + ( + json!({"expr": {"rqe": {"id": "sum"}}, "time": 1.0}).to_string(), + "invalid RQE query body", + ), + ( + json!({"expr": series_sum(), "time": 1.0, "extra": 1}).to_string(), + "unknown field `extra`", + ), + ( + json!({"expr": bad_quantile, "time": QUERY_TIME_S}).to_string(), + "phi", + ), + ( + json!({"expr": sum_leaf("nope", WINDOW_MS, &["job"]), "time": QUERY_TIME_S}) + .to_string(), + "unknown RQE id", + ), + ] { + let (status, response) = post(&url, body.clone()).await; + assert_eq!(status, 400, "{body}"); + assert_eq!(response["status"], "error", "{body}"); + assert_eq!(response["errorType"], "bad_data", "{body}"); + let error = response["error"].as_str().unwrap(); + assert!(error.contains(reason), "{body}: {error}"); + } + } + + #[tokio::test] + async fn route_reports_an_execution_failure_as_500() { + let (_server, url) = server().await; + let body = json!({"expr": leaf("misrolled", "events", "sum", TWO_WINDOWS_MS, &["instance"]), + "time": QUERY_TIME_S}); + let (status, response) = post(&url, body.to_string()).await; + assert_eq!(status, 500); + assert_eq!(response["status"], "error"); + assert_eq!(response["errorType"], "internal"); + } +} From 7c77405e4267b2a837a554b043843ca266a398eb Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 20:16:43 -0400 Subject: [PATCH 04/12] feat(asap-common): per-bucket top-k (`topk_by`) on RQE leaves An RQE leaf may name `topk_by` labels: like PromQL `topk by (...)`, a top-k leaf then ranks its input independently within each value combination of those labels, while `grouping` stays its output labels. Absent, the whole input is one bucket. Validation allows `topk_by` only on `topk_by_value` leaves, non-empty and within `grouping`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../rs/asap_types/src/rqe_expr.rs | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs index 05e6000d..d320909e 100644 --- a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs +++ b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs @@ -27,7 +27,13 @@ pub struct RqeLeaf { pub metric: String, pub capability: RqeCapability, pub lookback_ms: u64, + /// Output labels: one series per value combination. pub grouping: Vec, + /// Top-k buckets, as in PromQL `topk by (topk_by) (k, ...)`: the input is + /// ranked independently within each value combination of these labels, a + /// subset of `grouping`. Absent ranks the whole input as one bucket. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topk_by: Option>, /// Quantile rank; set iff `capability` is `quantile`. #[serde(default, skip_serializing_if = "Option::is_none")] pub phi: Option, @@ -159,6 +165,16 @@ pub enum RqeExprError { InvalidK { id: String, k: Option }, #[error("RQE {id}: k is only allowed for topk_by_value")] UnexpectedK { id: String }, + #[error("RQE {id}: topk_by is only allowed for topk_by_value")] + UnexpectedTopkBy { id: String }, + #[error("RQE {id}: topk_by must name at least one label; omit it to rank as one bucket")] + EmptyTopkBy { id: String }, + #[error("RQE {id}: topk_by {topk_by:?} is not a subset of grouping {grouping:?}")] + TopkByNotSubset { + id: String, + topk_by: Vec, + grouping: Vec, + }, #[error("RQE {id}: topk_by_value is only allowed at the root")] TopkNotAtRoot { id: String }, #[error("agg by {by:?} is not a subset of child labels {child:?}")] @@ -267,6 +283,22 @@ impl RqeLeaf { (_, Some(_)) => return Err(RqeExprError::UnexpectedK { id: id() }), (_, None) => {} } + match (self.capability, &self.topk_by) { + (RqeCapability::TopkByValue, Some(topk_by)) if topk_by.is_empty() => { + return Err(RqeExprError::EmptyTopkBy { id: id() }) + } + (RqeCapability::TopkByValue, Some(topk_by)) + if !label_set(topk_by).is_subset(&label_set(&self.grouping)) => + { + return Err(RqeExprError::TopkByNotSubset { + id: id(), + topk_by: topk_by.clone(), + grouping: self.grouping.clone(), + }) + } + (RqeCapability::TopkByValue, _) | (_, None) => {} + (_, Some(_)) => return Err(RqeExprError::UnexpectedTopkBy { id: id() }), + } if self.capability == RqeCapability::TopkByValue && !is_root { return Err(RqeExprError::TopkNotAtRoot { id: id() }); } @@ -326,6 +358,11 @@ mod tests { "id": "q2", "metric": "events", "capability": "topk_by_value", "lookback_ms": 1000, "grouping": ["user_id"], "k": 10, }})); + assert_round_trip(json!({"rqe": { + "id": "q4", "metric": "data", "capability": "topk_by_value", + "lookback_ms": 60000, "grouping": ["region", "job"], "k": 4, + "topk_by": ["region"], + }})); assert_round_trip(leaf_json("q3", "cardinality", &["job"])); } @@ -458,6 +495,67 @@ mod tests { )); } + // --- validation: topk_by --- + + fn topk_by_leaf(grouping: &[&str], topk_by: Value) -> Value { + let mut v = leaf_json("t", "topk_by_value", grouping); + v["rqe"]["k"] = json!(4); + v["rqe"]["topk_by"] = topk_by; + v + } + + #[test] + fn topk_by_rules() { + let leaf = parse(topk_by_leaf(&["region", "job"], json!(["region"]))); + assert_eq!(leaf.validate(), Ok(())); + let RqeExpr::Rqe { rqe } = &leaf else { + unreachable!() + }; + assert_eq!(rqe.topk_by.as_deref(), Some(&["region".to_string()][..])); + // The output labels stay the leaf's grouping, not its buckets. + assert_eq!(leaf.output_labels(), ["region", "job"]); + // topk_by equal to grouping keeps one series per bucket. + let all = topk_by_leaf(&["region", "job"], json!(["job", "region"])); + assert_eq!(parse(all).validate(), Ok(())); + } + + #[test] + fn rejects_topk_by_outside_grouping() { + let v = topk_by_leaf(&["region", "job"], json!(["region", "pod"])); + assert!(matches!( + parse(v).validate(), + Err(RqeExprError::TopkByNotSubset { .. }) + )); + } + + #[test] + fn rejects_empty_topk_by() { + let v = topk_by_leaf(&["region"], json!([])); + assert_eq!( + parse(v).validate(), + Err(RqeExprError::EmptyTopkBy { id: "t".into() }) + ); + } + + #[test] + fn rejects_topk_by_on_non_topk() { + for capability in ["sum", "cardinality"] { + let mut v = leaf_json("q", capability, &["job"]); + v["rqe"]["topk_by"] = json!(["job"]); + assert_eq!( + parse(v).validate(), + Err(RqeExprError::UnexpectedTopkBy { id: "q".into() }) + ); + } + let mut quantile = leaf_json("q", "quantile", &["job"]); + quantile["rqe"]["phi"] = json!(0.5); + quantile["rqe"]["topk_by"] = json!(["job"]); + assert!(matches!( + parse(quantile).validate(), + Err(RqeExprError::UnexpectedTopkBy { .. }) + )); + } + // --- validation: top-k placement --- #[test] From f8ae11fdee53f2dc4afc7a8fbb7c3801beea5da9 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 20:19:11 -0400 Subject: [PATCH 05/12] feat(planner): plan a top-k RQE leaf by its `topk_by` buckets A top-k leaf's optimizer item now carries its `topk_by` labels as `topk_by_labels`, as a PromQL `topk by (...)` query's does, so its Raqe is grouped by the buckets (the plan's top-k RQE grouping) and its heap is one per bucket, keyed by the remaining labels. Without `topk_by` the Raqe is a single bucket and can't share the deployment of a `topk by (...)` query. Tests run the pilot random workload (seed-862/workload-01-pilot) and its saved plans: `topk by (region, service)` leaves read the same heaps as their PromQL queries. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- asap-planner-rs/src/optimizer/saved_plan.rs | 40 +- .../tests/saved_plan_e2e_tables.rs | 158 +++++- .../random-seed862-workload01-pilot.json | 448 ++++++++++++++++++ ...dom-seed862-workload01-pilot.plans.json.gz | Bin 0 -> 2288 bytes 4 files changed, 638 insertions(+), 8 deletions(-) create mode 100644 asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json create mode 100644 asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index 4929de92..f4584aed 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -523,8 +523,8 @@ pub fn saved_plan_output( } /// The optimizer item of an RQE leaf, read every `interval_ms`: what a -/// PromQL or SQL leaf with the same capability, metric, lookback and grouping -/// needs. It has no query string, so it gets no `query_configs` entry. +/// PromQL or SQL leaf with the same capability, metric, lookback, grouping +/// and top-k buckets needs. It has no query string, so it gets no `query_configs` entry. fn leaf_item( leaf: &RqeLeaf, interval_ms: u64, @@ -579,7 +579,9 @@ fn leaf_item( grouping_labels: KeyByLabelNames::new(leaf.grouping.clone()), spatial_filter_normalized: normalize_spatial_filter(""), topk_count_events, - topk_by_labels: None, + // As a PromQL `topk by`: the Raqe is grouped by these buckets, + // as the plan's top-k RQE is. + topk_by_labels: leaf.topk_by.clone().map(KeyByLabelNames::new), }, query_strings: Vec::new(), query_frequency_hz: 1000.0 / interval_ms as f64, @@ -1590,6 +1592,38 @@ mod tests { assert!(err.contains(r#"doesn't serve RQE "unplanned""#), "{err}"); } + /// A `topk_by` leaf is a Raqe grouped by its buckets: one heap per `job`, + /// keyed by the remaining series label. + #[test] + fn writes_a_tree_only_topk_by_leaf_as_one_heap_per_bucket() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"interval_secs": 60, "expr": {"rqe": {"id": "top", "metric": "data", + "capability": "topk_by_value", "k": 5, "lookback_ms": 300_000, + "grouping": ["job", "label_0"], "topk_by": ["job"]}}}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let top = pick( + "top", + 0, + "cms-heap-topk-fastpath-vector2d", + json!({"rows": 3, "cols": 1024, "heap": 50}), + 300, + ); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [top]}, + ]}); + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + let [(id, _)] = rqe_aggregations(&inference, "top")[..] else { + panic!("{:?}", inference.rqe_configs); + }; + assert_eq!(stages(&inference, "top"), json!([])); + let streaming = output.to_streaming_config(QueryLanguage::promql).unwrap(); + let heap = streaming.get_aggregation_config(id).unwrap(); + assert_eq!(heap.grouping_labels.labels, ["job"]); + assert_eq!(heap.aggregated_labels.labels, ["label_0"]); + } + /// A topk deployment can't roll up, for an RQE as for a query. #[test] fn refuses_a_tree_only_topk_roll_up() { diff --git a/asap-planner-rs/tests/saved_plan_e2e_tables.rs b/asap-planner-rs/tests/saved_plan_e2e_tables.rs index cdcd3b6b..c82a3229 100644 --- a/asap-planner-rs/tests/saved_plan_e2e_tables.rs +++ b/asap-planner-rs/tests/saved_plan_e2e_tables.rs @@ -1,6 +1,8 @@ -//! `--planner saved` on a real sketch-bench eval table: the `all` workload of -//! sketch-bench's `rqe-optimizer/results/e2e-rate10` (sketch-bench c1fd812), -//! whose e2e queries carry RQE trees, and its saved plans. +//! `--planner saved` on real sketch-bench eval tables and their saved plans: +//! the `all` workload of sketch-bench's `rqe-optimizer/results/e2e-rate10` +//! (sketch-bench c1fd812), whose e2e queries carry RQE trees, and the pilot +//! random workload `seed-862/workload-01-pilot` (sketch-bench PR #226), whose +//! top-k queries are `topk by (region, service)`. use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; @@ -22,6 +24,19 @@ fn table_path() -> PathBuf { } fn convert(table: &Path, language: QueryLanguage) -> PlannerOutput { + convert_with_plans( + table, + "synthetic-templatesall-shared1.plans.json.gz", + language, + ) + .unwrap() +} + +fn convert_with_plans( + table: &Path, + plans: &str, + language: QueryLanguage, +) -> anyhow::Result { let selector = PlanSelector { method: "asap", weights: "cpu", @@ -29,12 +44,11 @@ fn convert(table: &Path, language: QueryLanguage) -> PlannerOutput { }; saved_plan_output( table, - &data_dir().join("synthetic-templatesall-shared1.plans.json.gz"), + &data_dir().join(plans), &selector, language, SCRAPE_INTERVAL_MS, ) - .unwrap() } fn table_queries() -> Vec { @@ -151,3 +165,137 @@ fn trees_add_only_rqe_configs() { assert_eq!(with_inference, without_inference, "{language:?}"); } } + +const PILOT_PLANS: &str = "random-seed862-workload01-pilot.plans.json.gz"; + +/// The pilot table without its `avg by (...) (quantile_over_time(...))` +/// queries, which `--planner saved` can't write yet. +fn pilot_table() -> Value { + let path = data_dir().join("random-seed862-workload01-pilot.json"); + let mut table: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + .retain(|q| !q["promql"].as_str().unwrap().starts_with("avg by")); + table +} + +/// The pilot table with a tree on each `topk by (region, service) (4, +/// sum_over_time(data[w]))` query: a leaf over every series label, ranked +/// within `topk_by` buckets. Returns the table and the top-k RQE ids. +fn pilot_with_topk_trees(topk_by: Option<&[&str]>) -> (Value, Vec) { + let mut table = pilot_table(); + let mut ids = Vec::new(); + for q in table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + { + let promql = q["promql"].as_str().unwrap(); + if !promql.starts_with("topk by (region, service) (4, sum_over_time(data[") { + continue; + } + let id = q["rqes"][0].as_str().unwrap().to_string(); + let lookback_ms = if promql.contains("[1m]") { + 60_000 + } else { + assert!(promql.contains("[20m]"), "{promql}"); + 1_200_000 + }; + let mut leaf = serde_json::json!({ + "id": id, "metric": "data", "capability": "topk_by_value", "k": 4, + "lookback_ms": lookback_ms, + "grouping": ["region", "service", "job", "label_0"], + }); + if let Some(topk_by) = topk_by { + leaf["topk_by"] = serde_json::json!(topk_by); + } + q["expr"] = serde_json::json!({"rqe": leaf}); + ids.push(id); + } + assert_eq!(ids.len(), 2, "the pilot has two top-k queries"); + (table, ids) +} + +fn write_table(dir: &tempfile::TempDir, table: &Value) -> PathBuf { + let path = dir.path().join("table.json"); + std::fs::write(&path, table.to_string()).unwrap(); + path +} + +/// A `topk by (region, service)` leaf is the plan's top-k RQE, grouped by +/// its buckets: it reads the same per-bucket heap as its PromQL query and +/// adds only its `rqe_configs` entry. +#[test] +fn pilot_topk_by_leaf_reads_its_promql_querys_heap() { + let dir = tempfile::tempdir().unwrap(); + let (with_trees, ids) = pilot_with_topk_trees(Some(&["region", "service"])); + let with = convert_with_plans( + &write_table(&dir, &with_trees), + PILOT_PLANS, + QueryLanguage::promql, + ) + .unwrap(); + let without_dir = tempfile::tempdir().unwrap(); + let without = convert_with_plans( + &write_table(&without_dir, &pilot_table()), + PILOT_PLANS, + QueryLanguage::promql, + ) + .unwrap(); + + let yaml = |s: String| serde_yaml::from_str::(&s).unwrap(); + assert_eq!( + yaml(with.to_streaming_yaml_string().unwrap()), + yaml(without.to_streaming_yaml_string().unwrap()) + ); + let inference = with.to_inference_config(QueryLanguage::promql).unwrap(); + let written: BTreeSet<&String> = inference.rqe_configs.keys().collect(); + assert_eq!(written, ids.iter().collect()); + let streaming = with.to_streaming_config(QueryLanguage::promql).unwrap(); + for q in pilot_with_topk_trees(None).0["workloads"][0]["e2e"]["queries"] + .as_array() + .unwrap() + { + let Some(rqe) = inference.rqe_configs.get(q["rqes"][0].as_str().unwrap()) else { + continue; + }; + let query = inference + .query_configs + .iter() + .find(|c| c.query == q["promql"].as_str().unwrap()) + .unwrap(); + let refs = |refs: &[asap_types::AggregationReference]| { + refs.iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect::>() + }; + assert_eq!(refs(&rqe.aggregations), refs(&query.aggregations), "{q}"); + assert!(rqe.query_time_aggregations.is_empty(), "{q}"); + // One heap per (region, service) bucket, keyed by the other labels. + let heap = streaming + .get_aggregation_config(rqe.aggregations[0].aggregation_id) + .unwrap(); + assert_eq!(heap.grouping_labels.labels, ["region", "service"], "{q}"); + assert_eq!(heap.aggregated_labels.labels, ["job", "label_0"], "{q}"); + } +} + +/// Without `topk_by` the leaf ranks every series as one bucket: not the +/// plan's per-(region, service) RQE, so the deployment its PromQL query +/// shares can't serve it. +#[test] +fn pilot_topk_leaf_without_topk_by_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (table, _) = pilot_with_topk_trees(None); + let Err(error) = convert_with_plans( + &write_table(&dir, &table), + PILOT_PLANS, + QueryLanguage::promql, + ) else { + panic!("a single-bucket leaf is not the plan's RQE"); + }; + assert!( + error.to_string().contains("need different sketches"), + "{error:#}" + ); +} diff --git a/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json new file mode 100644 index 00000000..38e91a55 --- /dev/null +++ b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json @@ -0,0 +1,448 @@ +{ + "inputs": { + "workload-01.json": "edacdc8662d75c0cd1c8bcc3136614225fd9f273d98360e73d8c563016586ae4" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "85699c9edef79ed6362ff19020d4c1f486af531d", + "workloads": [ + { + "dataset": "random/seed=862/workload-01", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz new file mode 100644 index 0000000000000000000000000000000000000000..dcd2f8832c63a7339bfe8355e73217752d34d8c7 GIT binary patch literal 2288 zcmZwGdpy$%AHeZd5*?H&9O_x=XE~F5>?oJx8r8}rO2%GphaoJFTw*Gh+Hl6ET$UUo zWoB{-+k=Uf(~y|9v4TQc_hiuLHM=!-Bm+ z!nFg#LqkCC^E^VwY?W=rJIAO)lKO%l39Pb3%HYh&r&BI9mU5@BrM+lLF5(1=ulk>+ zyccC;@3yQo#;X}KnQv<<&hjT^VlSbTCTu)?6BcL778~1_=*P6Grhd=9>)I)6c2OaZ zuhlJI8P^?{l|>hwI*hGEotx(uDc70Mu){N5ie)JG_|^HTth4dr`GN)anI~`zX~+#r z)d)4aTiwZw7D|>?k^3l@;?2?E zg^}_PDRb5Qp;fk@!Lz8SsIj+&lzU{CH%Hp*uLwp1Jh?w~Dtev4EWTWr6lhLP;^^nl zun3qF`o1Ep=5X8gT2rHa`CLLtOV?3LYZ$Jm`&j1a;@(eCukeg_P^WASF4P`;|F-Cq z??Up`ZSb}fGLGDLyx2-F#UX|7-PfR1DzmHygTXzT25BRxFU^;GO7S*tb}jQ)+x)l` zWS@E?k3X!xActS7L)KSLyA;x7f=(+%3n%%Z!(FFVQZXJv&!3yHbC2d?K0rKwdsmD; zVv}jNm)t__?VZrRi7juQ-^H+m(Tv(!X`T}^wG(dO#f9tPQ=gpJXKJ$l*~xN_j2R7~ zI-~kwjnrvI$8zDLrsb|I?ISK-ovdyBrO0@<$*|0odI#i4m&n~L_?6KAQ?g%0_E&JS zpm){jE_YTIzE7zq2IUcLxTq@THFj3u#(M2EoeBi+jA!<7BZNwBZu{%IkJ%@=!0<&2 zq++*GM6=3CZd%!Gkq|!YV7X$oeaOvB)vpa6CkUi=PNSF0{r2UDE0FC>c0&Gx7rhtIHfk-V+95IHa*90wB9qexRCGJhpPD}rEw|xoh8*;a zp)l&(0aAebXSUj*+qxC}FvhF%*YVhegi1*7s zH__H?&C)82_3-Nrtxo04zSWAwkjuO2Q-R+ucBK(@4cIhR8pk>d2C=?{=8xd`D2k3>~^N+_I-5g)L z@@jnV(8mmTWL1MR-3YZQ^s+FrN31ayG_Bbbu%BFe#q;5BOKE+2aefi01a}sbO;LzR zVa;ekbNOTQ6CBOPsC&1qP|Xp)?qO8@{^>!5Lxy5Tn)BTg59KE_zUBT!g~{uPjs+Q& z>E2V0pd^Z%ch1W9wp>kAJ@}70`H)1_QD)?fX-JZ-7<;rS-Dsgd+b)OgZN5L~! z!7;EP!rLmGGt%RO+iHJPemnSYME*eruF;Q?It`|JFGYP_56r=5eJqIqr0)`PfkB-c zxv^k?jgyx)XAnLf`X=>fe4Mf4J=j}MaQ|bJmA6EQrT#1kR6gHar^4jw(vvZUsij`X zj^mK?RjsKzO^GxWQ(_c(-6Alz=ru^%X$l^_Fga5(n18`{95iu+bLw@Ue6a;$}gDz%bxdDF(pR zt($THV<5r1;Y8py4v@C6Oy3PitYsW4fm7c|+Cn&xU`^T#d7sHre>G3rw-iW^+`m%h zVbd)>1e`0f@fGZGCN>=^28`kV<i5X-ZpJn(G2G z(pHT0S=_N8-36)XPp8oSb-YEj-e}jRzc|*V)6W85*R` z+|9`Q{D~h`{E3a^b*}2spDmur>_CHs*0~aD*X<5~Z>{Ar>Q+ncMu}?qukqI8&3Gn% zNM0g-QMp8HA+PgP$9=Xq5mks_0t7f(;kzChAToQe9NJ?4Tmb{dC#n_LX{?34@)J0GlhoLg2H=Q|M8i~4{ Date: Wed, 14 Oct 2026 20:21:23 -0400 Subject: [PATCH 06/12] feat(query-engine): rank RQE top-k leaves within their `topk_by` buckets A top-k leaf now ranks per bucket the way the PromQL path ranks `topk by (...)`: its aggregation keeps one heap per bucket (the aggregation's grouping) and the instant plan's top-k limit ranks within each. The leaf's `topk_by` (none: one bucket) must equal that grouping; otherwise the leaf fails as an execution error instead of answering a different ranking. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../src/engines/simple_engine/rqe.rs | 18 +++- .../src/tests/rqe_query_tests.rs | 82 +++++++++++++++++++ 2 files changed, 97 insertions(+), 3 deletions(-) diff --git a/asap-query-engine/src/engines/simple_engine/rqe.rs b/asap-query-engine/src/engines/simple_engine/rqe.rs index 69958235..ae0daec8 100644 --- a/asap-query-engine/src/engines/simple_engine/rqe.rs +++ b/asap-query-engine/src/engines/simple_engine/rqe.rs @@ -168,7 +168,9 @@ impl SimpleEngine { /// Reads `leaf` from the aggregations `config` names, over the window /// ending at `query_time` aligned as a PromQL instant query aligns it. /// A roll-up config reads at the aggregation's own grouping, then its - /// query-time stages regroup to the leaf's. + /// query-time stages regroup to the leaf's. A top-k leaf ranks within + /// each bucket of its aggregation's grouping, as PromQL `topk by` does, + /// so that grouping must be the leaf's `topk_by`. fn evaluate_rqe_leaf( &self, leaf: &RqeLeaf, @@ -190,6 +192,17 @@ impl SimpleEngine { )) })?; + if leaf.capability == RqeCapability::TopkByValue { + let buckets = KeyByLabelNames::new(leaf.topk_by.clone().unwrap_or_default()); + if value_config.grouping_labels != buckets { + return Err(failed(format!( + "ranks within topk_by buckets {:?}, but its aggregation keeps a heap per {:?}", + buckets.labels, value_config.grouping_labels.labels + )) + .into()); + } + } + let end_timestamp = self.align_end_timestamp_promql(query_time); let start_timestamp = end_timestamp.checked_sub(leaf.lookback_ms).ok_or_else(|| { RqeQueryError::BadRequest(format!( @@ -259,8 +272,7 @@ impl SimpleEngine { } } -/// The statistic and its kwargs a leaf's capability computes. Top-k is a -/// single global ranking: a leaf has no per-group bucketing field. +/// The statistic and its kwargs a leaf's capability computes. fn leaf_statistic(leaf: &RqeLeaf) -> Result<(Statistic, HashMap), RqeQueryError> { let missing = |field: &str| RqeQueryError::BadRequest(format!("RQE {:?}: missing {field}", leaf.id)); diff --git a/asap-query-engine/src/tests/rqe_query_tests.rs b/asap-query-engine/src/tests/rqe_query_tests.rs index fc66bfa2..9b85e47e 100644 --- a/asap-query-engine/src/tests/rqe_query_tests.rs +++ b/asap-query-engine/src/tests/rqe_query_tests.rs @@ -39,6 +39,9 @@ mod tests { const HLL_AGG: u64 = 3; const TOPK_AGG: u64 = 4; const MULTI_SUM_AGG: u64 = 5; + /// One heap per `job`, keyed by `srcip`: serves `topk by (job)`. + const TOPK_BY_JOB_AGG: u64 = 6; + const TOPK_BY_JOB_QUERY: &str = "topk by (job) (2, sum_over_time(flows[1s]))"; type Row = (Vec, f64); @@ -158,6 +161,13 @@ mod tests { &[], &["instance", "job"], ), + aggregation( + TOPK_BY_JOB_AGG, + AggregationType::CountMinSketchWithHeap, + "flows", + &["job"], + &["srcip"], + ), ]), }); let store = Arc::new(SimpleMapStore::new( @@ -227,6 +237,16 @@ mod tests { } insert(TOPK_AGG, QUERY_TIME_MS, None, Box::new(heap)); + // `api`'s smallest sources outweigh `web`'s largest, so a global + // ranking would keep only `api` rows. + for (job, sums) in [("api", [100.0, 200.0, 300.0]), ("web", [10.0, 20.0, 30.0])] { + let mut heap = CountMinSketchWithHeapAccumulator::new(3, 1024, 32); + for (i, sum) in sums.into_iter().enumerate() { + heap.inner.update(&format!("10.0.0.{}", i + 1), sum); + } + insert(TOPK_BY_JOB_AGG, QUERY_TIME_MS, Some(&[job]), Box::new(heap)); + } + insert( MULTI_SUM_AGG, previous_window_ms, @@ -257,6 +277,10 @@ mod tests { .add_metric( "events".to_string(), KeyByLabelNames::new(strings(&["instance", "job"])), + ) + .add_metric( + "flows".to_string(), + KeyByLabelNames::new(strings(&["job", "srcip"])), ); let inference_config = InferenceConfig { schema: SchemaConfig::PromQL(schema), @@ -265,6 +289,7 @@ mod tests { query_config("sum by (job) (sum_over_time(http_requests[2s]))", SUM_AGG), query_config("quantile_over_time(0.9, latency[2s])", DDSKETCH_AGG), query_config("topk(3, sum_over_time(transfers[1s]))", TOPK_AGG), + query_config(TOPK_BY_JOB_QUERY, TOPK_BY_JOB_AGG), QueryConfig::with_plan( "sum by (job) (sum_over_time(events[2s]))".to_string(), "sum by (instance, job) (sum_over_time(events[2s]))".to_string(), @@ -280,6 +305,10 @@ mod tests { ("p90".to_string(), rqe_config(DDSKETCH_AGG, vec![])), ("distinct".to_string(), rqe_config(HLL_AGG, vec![])), ("top3".to_string(), rqe_config(TOPK_AGG, vec![])), + ( + "top2_by_job".to_string(), + rqe_config(TOPK_BY_JOB_AGG, vec![]), + ), ( "rolled_up".to_string(), rqe_config(MULTI_SUM_AGG, vec![sum_by_job()]), @@ -337,6 +366,22 @@ mod tests { topk } + /// `topk by (job) (2, sum_over_time(flows[1s]))`, or with other buckets. + fn topk_by_job_leaf(topk_by: Option<&[&str]>) -> Value { + let mut topk = leaf( + "top2_by_job", + "flows", + "topk_by_value", + WINDOW_MS, + &["job", "srcip"], + ); + topk["rqe"]["k"] = json!(2); + if let Some(topk_by) = topk_by { + topk["rqe"]["topk_by"] = json!(topk_by); + } + topk + } + fn expr(tree: Value) -> RqeExpr { serde_json::from_value(tree).expect("a well-formed RQE tree") } @@ -489,6 +534,43 @@ mod tests { ); } + #[test] + fn topk_by_leaf_ranks_within_each_bucket_and_equals_promql_topk_by() { + let engine = engine(); + let answer = rqe(&engine, topk_by_job_leaf(Some(&["job"]))); + assert_eq!( + answer, + ( + strings(&["job", "srcip"]), + vec![ + row(&["api", "10.0.0.2"], 200.0), + row(&["api", "10.0.0.3"], 300.0), + row(&["web", "10.0.0.2"], 20.0), + row(&["web", "10.0.0.3"], 30.0), + ] + ), + "each job keeps its own top 2" + ); + assert_eq!(answer, promql(&engine, TOPK_BY_JOB_QUERY)); + } + + /// The buckets are the aggregation's heaps: a leaf ranking other buckets + /// than its aggregation keeps fails instead of answering another ranking. + #[test] + fn a_topk_leaf_whose_buckets_are_not_its_heaps_fails_loudly() { + let engine = engine(); + for tree in [topk_by_job_leaf(None), topk_by_job_leaf(Some(&["srcip"]))] { + let error = rqe_error(&engine, tree, QUERY_TIME_S); + assert!(matches!(&error, RqeQueryError::Execution(_)), "{error:?}"); + assert!(error.to_string().contains("top2_by_job"), "{error}"); + assert!(error.to_string().contains("topk_by"), "{error}"); + } + let mut global = topk_leaf(); + global["rqe"]["topk_by"] = json!(["srcip"]); + let error = rqe_error(&engine, global, QUERY_TIME_S); + assert!(error.to_string().contains("top3"), "{error}"); + } + #[test] fn rolled_up_leaf_applies_its_query_time_stage_like_the_promql_plan() { let engine = engine(); From 1b07c8364a6d0c8d7a90ca5f95c70f1d03e5a2e9 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 22:32:57 -0400 Subject: [PATCH 07/12] fix(asap-common): tighten RQE tree validation Require every subtree that feeds the result (root, agg operand, one side of each binary op) to read at least one RQE leaf; allow a repeated leaf id only for identical leaves; reject duplicate labels in grouping, by and topk_by; reject an empty id or metric and a zero lookback. Document that label lists are sets matched by name. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../rs/asap_types/src/rqe_expr.rs | 168 +++++++++++++++++- 1 file changed, 160 insertions(+), 8 deletions(-) diff --git a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs index d320909e..53907354 100644 --- a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs +++ b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs @@ -5,8 +5,12 @@ //! {"agg": "avg", "by": ["job"], "of": //! {"op": "/", "lhs": {"rqe": {...}}, "rhs": {"scalar": 2.0}}} //! ``` +//! +//! Label lists (`grouping`, `by`, `topk_by`) are sets: their order carries no +//! meaning and series are matched by label name, so an engine must sort or +//! match by name rather than pair values by position. -use std::collections::BTreeSet; +use std::collections::{BTreeSet, HashMap}; use serde::{Deserialize, Serialize}; @@ -179,8 +183,18 @@ pub enum RqeExprError { TopkNotAtRoot { id: String }, #[error("agg by {by:?} is not a subset of child labels {child:?}")] AggByNotSubset { by: Vec, child: Vec }, - #[error("binary op needs at least one non-scalar side")] - BinaryBothScalar, + #[error("RQE id must not be empty")] + EmptyId, + #[error("RQE {id}: metric must not be empty")] + EmptyMetric { id: String }, + #[error("RQE {id}: lookback_ms must be > 0")] + ZeroLookback { id: String }, + #[error("RQE {id} appears more than once with different leaves")] + ConflictingLeafId { id: String }, + #[error("{field} lists label {label:?} more than once")] + DuplicateLabel { field: &'static str, label: String }, + #[error("{node} reads no RQE leaf")] + NoLeaf { node: &'static str }, #[error("binary op sides have different labels: {lhs:?} vs {rhs:?}")] BinaryLabelMismatch { lhs: Vec, rhs: Vec }, } @@ -188,7 +202,20 @@ pub enum RqeExprError { impl RqeExpr { /// Checks the whole tree; call once after loading. pub fn validate(&self) -> Result<(), RqeExprError> { - self.validate_node(true) + self.validate_node(true)?; + if !self.has_leaf() { + return Err(RqeExprError::NoLeaf { node: "root" }); + } + // A leaf may be repeated (e.g. `x - x`) but an id names one leaf. + let mut by_id: HashMap<&str, &RqeLeaf> = HashMap::new(); + for leaf in self.leaves() { + if *by_id.entry(&leaf.id).or_insert(leaf) != leaf { + return Err(RqeExprError::ConflictingLeafId { + id: leaf.id.clone(), + }); + } + } + Ok(()) } fn validate_node(&self, is_root: bool) -> Result<(), RqeExprError> { @@ -196,6 +223,10 @@ impl RqeExpr { RqeExpr::Rqe { rqe } => rqe.validate(is_root), RqeExpr::Agg { by, of, .. } => { of.validate_node(false)?; + if !of.has_leaf() { + return Err(RqeExprError::NoLeaf { node: "agg" }); + } + check_unique_labels("agg by", by)?; let child = label_set(of.output_labels()); if !label_set(by).is_subset(&child) { return Err(RqeExprError::AggByNotSubset { @@ -208,9 +239,9 @@ impl RqeExpr { RqeExpr::Binary { lhs, rhs, .. } => { lhs.validate_node(false)?; rhs.validate_node(false)?; - match (lhs.is_scalar(), rhs.is_scalar()) { - (true, true) => Err(RqeExprError::BinaryBothScalar), - (false, false) + match (lhs.has_leaf(), rhs.has_leaf()) { + (false, false) => Err(RqeExprError::NoLeaf { node: "binary op" }), + (true, true) if label_set(lhs.output_labels()) != label_set(rhs.output_labels()) => { Err(RqeExprError::BinaryLabelMismatch { @@ -264,11 +295,33 @@ impl RqeExpr { fn is_scalar(&self) -> bool { matches!(self, RqeExpr::Scalar { .. }) } + + fn has_leaf(&self) -> bool { + match self { + RqeExpr::Rqe { .. } => true, + RqeExpr::Agg { of, .. } => of.has_leaf(), + RqeExpr::Binary { lhs, rhs, .. } => lhs.has_leaf() || rhs.has_leaf(), + RqeExpr::Scalar { .. } => false, + } + } } impl RqeLeaf { fn validate(&self, is_root: bool) -> Result<(), RqeExprError> { let id = || self.id.clone(); + if self.id.is_empty() { + return Err(RqeExprError::EmptyId); + } + if self.metric.is_empty() { + return Err(RqeExprError::EmptyMetric { id: id() }); + } + if self.lookback_ms == 0 { + return Err(RqeExprError::ZeroLookback { id: id() }); + } + check_unique_labels("grouping", &self.grouping)?; + if let Some(topk_by) = &self.topk_by { + check_unique_labels("topk_by", topk_by)?; + } match (self.capability, self.phi) { (RqeCapability::Quantile, Some(phi)) if (0.0..=1.0).contains(&phi) => {} (RqeCapability::Quantile, phi) => { @@ -310,6 +363,17 @@ fn label_set(labels: &[String]) -> BTreeSet<&str> { labels.iter().map(String::as_str).collect() } +fn check_unique_labels(field: &'static str, labels: &[String]) -> Result<(), RqeExprError> { + let mut seen = BTreeSet::new(); + match labels.iter().find(|l| !seen.insert(l.as_str())) { + Some(label) => Err(RqeExprError::DuplicateLabel { + field, + label: label.clone(), + }), + None => Ok(()), + } +} + #[cfg(test)] mod tests { use super::*; @@ -618,7 +682,95 @@ mod tests { let scalar_lhs = json!({"op": "-", "lhs": {"scalar": 1.0}, "rhs": sum_leaf("a", &["job"])}); assert_eq!(parse(scalar_lhs).validate(), Ok(())); let both = json!({"op": "+", "lhs": {"scalar": 1.0}, "rhs": {"scalar": 2.0}}); - assert_eq!(parse(both).validate(), Err(RqeExprError::BinaryBothScalar)); + let node = "binary op"; + assert_eq!(parse(both).validate(), Err(RqeExprError::NoLeaf { node })); + } + + // --- validation: every subtree reads a leaf --- + + #[test] + fn rejects_subtrees_without_leaves() { + let scalar_agg = |x: f64| json!({"agg": "sum", "by": [], "of": {"scalar": x}}); + // Neither side is a literal scalar, yet neither reads a leaf. + let binary = json!({"op": "+", "lhs": scalar_agg(1.0), "rhs": scalar_agg(2.0)}); + let nested_binary = json!({"op": "*", + "lhs": {"op": "+", "lhs": {"scalar": 1.0}, "rhs": {"scalar": 2.0}}, + "rhs": sum_leaf("a", &[])}); + for (v, node) in [ + (json!({"scalar": 1.0}), "root"), + (scalar_agg(1.0), "agg"), + (binary, "agg"), + (nested_binary, "binary op"), + ] { + assert_eq!(parse(v).validate(), Err(RqeExprError::NoLeaf { node })); + } + // A side that reads a leaf through an agg is not a scalar side. + let agg_side = json!({"op": "+", + "lhs": {"agg": "sum", "by": [], "of": sum_leaf("a", &["job"])}, + "rhs": {"scalar": 1.0}}); + assert_eq!(parse(agg_side).validate(), Ok(())); + } + + // --- validation: leaf ids --- + + #[test] + fn repeated_identical_leaf_is_allowed() { + let v = json!({"op": "-", "lhs": sum_leaf("x", &["job"]), "rhs": sum_leaf("x", &["job"])}); + let expr = parse(v); + assert_eq!(expr.validate(), Ok(())); + assert_eq!(expr.leaves().len(), 2); + } + + #[test] + fn rejects_one_id_for_different_leaves() { + let mut other = sum_leaf("x", &["job"]); + other["rqe"]["metric"] = json!("other_metric"); + let v = json!({"op": "/", "lhs": sum_leaf("x", &["job"]), "rhs": other}); + assert_eq!( + parse(v).validate(), + Err(RqeExprError::ConflictingLeafId { id: "x".into() }) + ); + } + + // --- validation: leaf fields --- + + #[test] + fn rejects_empty_or_zero_leaf_fields() { + let with = |key: &str, value: Value| { + let mut v = sum_leaf("q", &[]); + v["rqe"][key] = value; + parse(v).validate() + }; + assert_eq!(with("id", json!("")), Err(RqeExprError::EmptyId)); + assert_eq!( + with("metric", json!("")), + Err(RqeExprError::EmptyMetric { id: "q".into() }) + ); + assert_eq!( + with("lookback_ms", json!(0)), + Err(RqeExprError::ZeroLookback { id: "q".into() }) + ); + } + + // --- validation: duplicate labels --- + + #[test] + fn rejects_duplicate_labels() { + let dup = |field: &'static str| { + Err(RqeExprError::DuplicateLabel { + field, + label: "job".into(), + }) + }; + let grouping = sum_leaf("q", &["job", "instance", "job"]); + assert_eq!(parse(grouping).validate(), dup("grouping")); + let by = json!({"agg": "sum", "by": ["job", "job"], "of": sum_leaf("q", &["job"])}); + assert_eq!(parse(by).validate(), dup("agg by")); + let topk_by = topk_by_leaf(&["job", "region"], json!(["job", "job"])); + assert_eq!(parse(topk_by).validate(), dup("topk_by")); + // A duplicated grouping no longer slips past the binary label-set check. + let binary = json!({"op": "+", "lhs": sum_leaf("a", &["job", "job"]), "rhs": sum_leaf("b", &["job"])}); + assert_eq!(parse(binary).validate(), dup("grouping")); } #[test] From 33ddb10d1999f39f2c7d62ac3c3680b064834c20 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 22:38:48 -0400 Subject: [PATCH 08/12] fix(planner): saved-plan RQE checks per review; add --expr-only - Compare a query's `rqes` with its tree's leaves as sets. - Allow one RQE leaf at several repeat intervals: each interval is a Raqe, and the interval-independent rqe_configs entry is written once. Different leaves under one id are still refused. - Name a query in errors by its text, else its tree's RQE ids. - Compute and check a query's RQE ids once. - Check a tree's leaves read one kind of metric without dedup(). - Describe the e2e-tables fixture instead of citing its PR. - --expr-only (with --planner saved) plans only the e2e queries that have an RQE tree; the others (e.g. rate) get no deployment. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../rs/asap_types/src/streaming_config.rs | 80 ++++- asap-planner-rs/src/main.rs | 12 +- asap-planner-rs/src/optimizer/saved_plan.rs | 313 ++++++++++++++---- .../tests/saved_plan_e2e_tables.rs | 73 +++- 4 files changed, 397 insertions(+), 81 deletions(-) diff --git a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs index 4ab02498..d8e9e34f 100644 --- a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs @@ -55,8 +55,19 @@ impl StreamingConfig { let mut read_count_threshold_map: HashMap = HashMap::new(); if let Some(inference_config) = inference_config { - for query_config in &inference_config.query_configs { - for aggregation in &query_config.aggregations { + // Every reader of an aggregation: text queries and RQE leaves. + let references = inference_config + .query_configs + .iter() + .map(|q| &q.aggregations) + .chain( + inference_config + .rqe_configs + .values() + .map(|r| &r.aggregations), + ); + for aggregations in references { + for aggregation in aggregations { let aggregation_id = aggregation.aggregation_id; let num_aggregates = aggregation.num_aggregates_to_retain; retention_map @@ -152,7 +163,7 @@ mod tests { use crate::aggregation_config::AggregationConfigError; use crate::aggregation_reference::AggregationReference; use crate::enums::{CleanupPolicy, QueryLanguage}; - use crate::inference_config::InferenceConfig; + use crate::inference_config::{InferenceConfig, RqeConfig}; use crate::query_config::QueryConfig; use promql_utilities::query_logics::enums::AggregationType; @@ -523,4 +534,67 @@ aggregations: .expect_err("an overflowing threshold must not load"); assert!(error.to_string().contains("overflowed"), "{error}"); } + + fn rqe_config(reference: AggregationReference) -> RqeConfig { + RqeConfig { + aggregations: vec![reference], + query_time_aggregations: Vec::new(), + } + } + + #[test] + fn rqe_only_reference_bounds_retention() { + // An aggregation only an RQE leaf reads keeps that leaf's windows, + // not every window ever built. + let yaml = minmax_yaml("MinMax", "max"); + let mut inference = + InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::CircularBuffer); + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::new(1, Some(4))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("an RQE-only aggregation should load"); + + assert_eq!(config[1].num_aggregates_to_retain, Some(4)); + } + + #[test] + fn query_and_rqe_references_keep_the_largest_retention() { + // A shared aggregation keeps the longer lookback, whichever reader has it. + let yaml = minmax_yaml("MinMax", "max"); + for (query_retain, rqe_retain) in [(7, 2), (2, 7)] { + let mut inference = + InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::CircularBuffer); + inference.query_configs = vec![QueryConfig::new("max(metric[1m])".into()) + .add_aggregation(AggregationReference::new(1, Some(query_retain)))]; + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::new(1, Some(rqe_retain))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("a shared aggregation should load"); + + assert_eq!(config[1].num_aggregates_to_retain, Some(7)); + } + } + + #[test] + fn query_and_rqe_references_sum_read_based_thresholds() { + let yaml = minmax_yaml("MinMax", "max"); + let mut inference = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::ReadBased); + inference.query_configs = vec![QueryConfig::new("max(metric[1m])".into()) + .add_aggregation(AggregationReference::with_read_count_threshold(1, Some(2)))]; + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::with_read_count_threshold(1, Some(3))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("a shared aggregation should load"); + + assert_eq!(config[1].read_count_threshold, Some(5)); + } } diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index 32dda2b6..4e831643 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -87,6 +87,11 @@ struct Args { #[arg(long = "sla-ms")] sla_ms: Option, + /// Saved only. Plan only the e2e queries that have an RQE tree (`expr`); + /// the others (e.g. `rate`) are skipped, so nothing is deployed for them. + #[arg(long = "expr-only")] + expr_only: bool, + /// MILP only. YAML workload facts: per metric, positive `value_range` and /// `cardinality` per label set, including all labels (the series count), /// plus the `shape` of each grouping sketches may serve. @@ -155,8 +160,10 @@ fn main() -> anyhow::Result<()> { && args.workload_table.is_none() && args.method.is_none() && args.weights.is_none() - && args.sla_ms.is_none(), - "--plan, --workload-table, --method, --weights and --sla-ms require --planner saved" + && args.sla_ms.is_none() + && !args.expr_only, + "--plan, --workload-table, --method, --weights, --sla-ms and --expr-only require \ + --planner saved" ); } if args.planner != PlannerArg::Milp { @@ -358,6 +365,7 @@ fn run_saved(args: &Args) -> anyhow::Result<()> { }, args.query_language, scrape_interval_ms, + args.expr_only, )? .write_to_dir(&args.output_dir)?; Ok(()) diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index f4584aed..d2b73e7e 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -5,7 +5,7 @@ //! expression tree (`expr`) also gets each of its leaves an `rqe_configs` //! entry, so the engine can answer the tree without parsing PromQL or SQL. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Read; use std::path::Path; @@ -69,73 +69,99 @@ struct E2eQuery { } impl E2eQuery { + /// How errors name the query: its text, else its tree's RQEs. + fn name(&self) -> String { + match (self.promql.as_ref().or(self.sql.as_ref()), &self.expr) { + (Some(text), _) => format!("{text:?}"), + (None, Some(expr)) => format!("with RQEs {:?}", leaf_ids(expr)), + (None, None) if self.rqes.is_empty() => "with no text, `expr` or RQEs".to_string(), + (None, None) => format!("reading {:?}", self.rqes), + } + } + /// The query's PromQL or SQL text, if any. fn text(&self) -> anyhow::Result> { match (&self.promql, &self.sql) { (Some(_), Some(_)) => bail!( - "e2e query reading {:?} needs exactly one of `promql` and `sql`", - self.rqes + "e2e query {} needs exactly one of `promql` and `sql`", + self.name() ), (promql, sql) => Ok(promql.as_deref().or(sql.as_deref())), } } /// Whether the query is SQL: written in `sql`, or, with no text, a tree - /// whose leaves read `records` metrics (SQL tables). + /// whose leaves read `records` metrics (SQL tables). Its tree, if any, + /// must be valid. fn is_sql(&self, metrics: &BTreeMap) -> anyhow::Result { let text_sql = self.text()?.map(|_| self.sql.is_some()); let tree_sql = match &self.expr { None => None, Some(expr) => { - let mut kinds = Vec::new(); - for leaf in expr.leaves() { + let reads_records = |leaf: &RqeLeaf| { let metric = metrics.get(&leaf.metric).with_context(|| { format!("RQE {:?} reads unknown metric {:?}", leaf.id, leaf.metric) })?; - kinds.push(metric.kind == RECORDS_METRIC_TYPE); - } - kinds.dedup(); - match kinds.as_slice() { - [] => bail!("e2e query reading {:?}: its `expr` reads no RQE", self.rqes), - [sql] => Some(*sql), - _ => bail!( - "e2e query reading {:?}: its `expr` mixes records and non-records metrics", - self.rqes - ), + anyhow::Ok(metric.kind == RECORDS_METRIC_TYPE) + }; + let leaves = expr.leaves(); + let (first, rest) = leaves.split_first().expect("a valid tree reads a leaf"); + let sql = reads_records(first)?; + for leaf in rest { + ensure!( + reads_records(leaf)? == sql, + "e2e query {}: its `expr` mixes records and non-records metrics", + self.name() + ); } + Some(sql) } }; match (text_sql, tree_sql) { (Some(text), Some(tree)) if text != tree => bail!( - "e2e query reading {:?} is written in {} but its `expr` reads {} metrics", - self.rqes, + "e2e query {} is written in {} but its `expr` reads {} metrics", + self.name(), if text { "SQL" } else { "PromQL" }, if tree { "records" } else { "non-records" } ), (Some(sql), _) | (None, Some(sql)) => Ok(sql), - (None, None) => bail!( - "e2e query reading {:?} needs `promql`, `sql` or `expr`", - self.rqes - ), + (None, None) => bail!("e2e query {} needs `promql`, `sql` or `expr`", self.name()), } } - /// The plan RQEs the query reads: its tree's leaves, if it has one, - /// which `rqes` must then list in order or omit. + /// The plan RQEs the query reads: `rqes`, or, if it omits them, its + /// tree's leaves. With both, they must name the same RQEs, in any order + /// and however often each appears. fn rqe_ids(&self) -> anyhow::Result> { let rqes: Vec<&str> = self.rqes.iter().map(String::as_str).collect(); let Some(expr) = &self.expr else { return Ok(rqes); }; - let leaves: Vec<&str> = expr.leaves().iter().map(|l| l.id.as_str()).collect(); + let leaves = leaf_ids(expr); + if rqes.is_empty() { + return Ok(leaves); + } ensure!( - rqes.is_empty() || rqes == leaves, - "e2e query names RQEs {rqes:?} but its `expr` reads {leaves:?}" + rqes.iter().collect::>() == leaves.iter().collect(), + "e2e query {} names RQEs {rqes:?} but its `expr` reads {leaves:?}", + self.name() ); - Ok(leaves) + Ok(rqes) } } +/// The ids of a tree's leaves, left to right. +fn leaf_ids(expr: &RqeExpr) -> Vec<&str> { + expr.leaves().iter().map(|l| l.id.as_str()).collect() +} + +/// A query planned from its text, and the plan RQEs it reads. +struct TextQuery<'a> { + query: &'a E2eQuery, + text: &'a str, + rqes: Vec<&'a str>, +} + #[derive(Deserialize)] struct E2eMetric { #[serde(rename = "type")] @@ -190,13 +216,13 @@ pub struct PlanSelector<'a> { /// The selected queries' texts by repeat interval in seconds: one query group /// each. -fn by_interval(selected: &[(&E2eQuery, &str)]) -> BTreeMap> { +fn by_interval(selected: &[TextQuery]) -> BTreeMap> { let mut groups: BTreeMap> = BTreeMap::new(); - for (q, text) in selected { + for q in selected { groups - .entry(q.interval_secs) + .entry(q.query.interval_secs) .or_default() - .push(text.to_string()); + .push(q.text.to_string()); } groups } @@ -204,7 +230,7 @@ fn by_interval(selected: &[(&E2eQuery, &str)]) -> BTreeMap> { /// The PromQL controller config of the selected queries, with every /// non-`records` metric's labels as hints. fn e2e_promql_config( - selected: &[(&E2eQuery, &str)], + selected: &[TextQuery], metrics: &BTreeMap, ) -> anyhow::Result { let config = json!({ @@ -225,7 +251,7 @@ fn e2e_promql_config( /// is a table, its labels the metadata columns and its values the value /// columns. fn e2e_sql_config( - selected: &[(&E2eQuery, &str)], + selected: &[TextQuery], metrics: &BTreeMap, ) -> anyhow::Result { let mut tables = Vec::new(); @@ -275,13 +301,14 @@ fn read_json(path: &Path) -> anyhow::Result { /// Streaming and inference YAML for the selected plan of `plan_path` (JSON, /// gzipped or not) over the workload of `table_path`: its queries written in -/// `language`. +/// `language`, and only those with an RQE tree if `expr_only`. pub fn saved_plan_output( table_path: &Path, plan_path: &Path, selector: &PlanSelector, language: QueryLanguage, scrape_interval_ms: u64, + expr_only: bool, ) -> anyhow::Result { let sql = match language { QueryLanguage::sql => true, @@ -332,18 +359,26 @@ pub fn saved_plan_output( ); } // Queries in `language`: those with text, and those with a tree. - let mut selected: Vec<(&E2eQuery, &str)> = Vec::new(); + let mut selected: Vec = Vec::new(); let mut trees: Vec<(&E2eQuery, &RqeExpr)> = Vec::new(); for q in &e2e.queries { + if expr_only && q.expr.is_none() { + continue; + } if let Some(expr) = &q.expr { expr.validate() - .with_context(|| format!("e2e query reading {:?}: invalid `expr`", q.rqes))?; + .with_context(|| format!("e2e query {}: invalid `expr`", q.name()))?; } if q.is_sql(&e2e.metrics)? != sql { continue; } + let rqes = q.rqe_ids()?; if let Some(text) = q.text()? { - selected.push((q, text)); + selected.push(TextQuery { + query: q, + text, + rqes, + }); } if let Some(expr) = &q.expr { trees.push((q, expr)); @@ -351,8 +386,9 @@ pub fn saved_plan_output( } ensure!( !selected.is_empty() || !trees.is_empty(), - "{}: no e2e query is written in {language:?}", - table_path.display() + "{}: no e2e query{} is written in {language:?}", + table_path.display(), + if expr_only { " with an `expr`" } else { "" } ); let (promql_config, sql_config); let config = if sql { @@ -369,8 +405,13 @@ pub fn saved_plan_output( // Leaf query string and repeat interval -> the plan RQEs it reads. let mut rqes_of: HashMap<(String, u64), Vec<&str>> = HashMap::new(); - for &(q, text) in &selected { - let rqe_ids = q.rqe_ids()?; + for TextQuery { + query: q, + text, + rqes: rqe_ids, + } in &selected + { + let rqe_ids = rqe_ids.clone(); let leaves: Vec<(String, Vec<&str>)> = match config { // A SQL query is one leaf, and its RQEs a set. MilpConfig::Sql(_) => { @@ -406,36 +447,37 @@ pub fn saved_plan_output( } } - // Every tree leaf, by RQE id, and the repeat interval of its queries. - let mut tree_leaves: BTreeMap<&str, (&RqeLeaf, u64)> = BTreeMap::new(); + // Every tree leaf, by RQE id, and the repeat intervals of its queries. + let mut tree_leaves: BTreeMap<&str, (&RqeLeaf, BTreeSet)> = BTreeMap::new(); for (q, expr) in &trees { - // A tree-only query's `rqes` must agree with its tree too. - q.rqe_ids()?; for leaf in expr.leaves() { - let interval_ms = q.interval_secs * 1000; - if let Some((other, other_interval_ms)) = - tree_leaves.insert(&leaf.id, (leaf, interval_ms)) - { - ensure!( - other == leaf && other_interval_ms == interval_ms, - "RQE {:?} is two different leaves: {other:?} every {other_interval_ms} ms \ - and {leaf:?} every {interval_ms} ms", - leaf.id - ); - } + let (other, intervals_ms) = tree_leaves + .entry(&leaf.id) + .or_insert_with(|| (leaf, BTreeSet::new())); + ensure!( + *other == leaf, + "RQE {:?} is two different leaves: {other:?} and {leaf:?}", + leaf.id + ); + intervals_ms.insert(q.interval_secs * 1000); } } let mut workload = workload_from_config(config, scrape_interval_ms)?; let text_items = workload.items.len(); - // Each tree leaf is one more item and Raqe, read directly by its RQE - // id; `leaf_rqes[i]` is item `text_items + i`'s. + // Each tree leaf at each repeat interval is one more item and Raqe, + // read directly by its RQE id; `leaf_rqes[i]` is item `text_items + i`'s. let mut rqe_raqes: BTreeMap = BTreeMap::new(); let mut leaf_rqes: Vec<&str> = Vec::new(); - for (&id, &(leaf, interval_ms)) in &tree_leaves { - let item = leaf_item(leaf, interval_ms, &e2e.metrics)?; - rqe_raqes.insert(id.to_string(), workload.push_item(item, id)?); - leaf_rqes.push(id); + for (&id, (leaf, intervals_ms)) in &tree_leaves { + for &interval_ms in intervals_ms { + let item = leaf_item(leaf, interval_ms, &e2e.metrics)?; + let raqe = workload.push_item(item, id)?; + // One pick serves every interval, and the interval doesn't reach + // the `rqe_configs` entry: any interval's Raqe writes it. + rqe_raqes.entry(id.to_string()).or_insert(raqe); + leaf_rqes.push(id); + } } // Plan deployment index -> its position in `deployments`, in order of @@ -700,6 +742,16 @@ mod tests { plan: &[u8], sla_ms: Option, language: QueryLanguage, + ) -> anyhow::Result { + run_filtered(table, plan, sla_ms, language, false) + } + + fn run_filtered( + table: &str, + plan: &[u8], + sla_ms: Option, + language: QueryLanguage, + expr_only: bool, ) -> anyhow::Result { let dir = tempfile::tempdir().unwrap(); let (table_path, plan_path) = (dir.path().join("table.json"), dir.path().join("plan.json")); @@ -710,7 +762,14 @@ mod tests { weights: "cpu", sla_ms, }; - saved_plan_output(&table_path, &plan_path, &selector, language, 1000) + saved_plan_output( + &table_path, + &plan_path, + &selector, + language, + 1000, + expr_only, + ) } fn refusal(plan: Value, sla_ms: Option) -> String { @@ -1138,7 +1197,12 @@ mod tests { .err() .expect("refused") .to_string(); - assert!(err.contains("exactly one of `promql` and `sql`"), "{err}"); + assert!( + err.contains( + r#"e2e query "sum by (job) (data)" needs exactly one of `promql` and `sql`"# + ), + "{err}" + ); } #[test] @@ -1457,14 +1521,103 @@ mod tests { #[test] fn refuses_a_tree_disagreeing_with_its_rqes() { let mut table = tree_table(); - queries(&mut table)[1]["rqes"] = json!(["p50", "p90"]); + queries(&mut table)[1]["rqes"] = json!(["p90", "sum"]); let err = tree_refusal(&table); assert!( - err.contains(r#"names RQEs ["p50", "p90"] but its `expr` reads ["p90", "p50"]"#), + err.contains(r#"names RQEs ["p90", "sum"] but its `expr` reads ["p90", "p50"]"#), "{err}" ); } + /// `rqes` and the tree's leaves are compared as sets: a SQL query's + /// `rqes` in another order, or a tree reading one leaf twice, agree. + #[test] + fn reads_a_tree_whose_rqes_are_reordered_or_repeated() { + let users = |id: &str| http_leaf(id, "cardinality", &["service"], "user_id"); + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + // `rqes` is ["users_1pct", "users_5pct"]. + queries(&mut table)[1]["expr"] = + json!({"op": "-", "lhs": users("users_5pct"), "rhs": users("users_1pct")}); + let output = run_table(&table, &mixed_plan(1), QueryLanguage::sql).unwrap(); + assert_eq!( + inference_of(&output, QueryLanguage::sql).rqe_configs.len(), + 2 + ); + + let mut table = tree_table(); + let p50 = quantile_leaf("p50", 0.5, 300_000, &["job", "label_0"]); + queries(&mut table)[2]["expr"] = json!({"op": "-", "lhs": p50.clone(), "rhs": p50}); + assert_eq!(queries(&mut table)[2]["rqes"], json!(["p50"])); + run_table(&table, &plans(shared_p50()), QueryLanguage::promql).unwrap(); + } + + /// The `rqe_configs` entry doesn't depend on the interval, so one leaf + /// run at two intervals is one entry on one aggregation. + #[test] + fn plans_one_rqe_read_at_two_intervals() { + let plan = plans(shared_p50()); + let once = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + let mut table = tree_table(); + queries(&mut table) + .push(json!({"interval_secs": 10, "expr": leaf("sum", "sum", 1000, &["job"])})); + let twice = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!( + twice.to_streaming_yaml_string().unwrap(), + once.to_streaming_yaml_string().unwrap() + ); + assert_eq!( + rqe_aggregations(&inference_of(&twice, QueryLanguage::promql), "sum"), + rqe_aggregations(&inference_of(&once, QueryLanguage::promql), "sum") + ); + } + + /// `--expr-only` skips the queries without a tree: nothing is deployed + /// or written for them. + #[test] + fn expr_only_plans_only_the_queries_with_a_tree() { + let mut table = tree_table(); + queries(&mut table)[0] + .as_object_mut() + .unwrap() + .remove("expr"); + let plan = plans(shared_p50()).to_string(); + let run = |expr_only| { + run_filtered( + &table.to_string(), + plan.as_bytes(), + None, + QueryLanguage::promql, + expr_only, + ) + .unwrap() + }; + + let all = run(false); + assert_eq!(all.streaming_aggregation_count(), 2); + assert_eq!(all.inference_query_count(), 3); + + let expr_only = run(true); + // Only the DDSketch the quantile trees read; no sum. + assert_eq!(expr_only.streaming_aggregation_count(), 1); + assert_eq!(expr_only.inference_query_count(), 2); + let inference = inference_of(&expr_only, QueryLanguage::promql); + assert!(inference + .query_configs + .iter() + .all(|q| q.query != "sum by (job) (data)")); + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["p50", "p90"]); + + // A table with no tree in the language has nothing to plan. + let err = run_filtered(TABLE, plan.as_bytes(), None, QueryLanguage::promql, true) + .err() + .expect("refused") + .to_string(); + assert!(err.contains("no e2e query with an `expr`"), "{err}"); + } + /// `rqes` may be omitted: the tree names the RQEs. #[test] fn reads_the_rqes_of_a_tree_without_rqes() { @@ -1488,8 +1641,20 @@ mod tests { queries(&mut table)[0]["expr"] = json!({"agg": "sum", "by": ["region"], "of": leaf("sum", "sum", 1000, &["job"])}); let err = tree_refusal(&table); - assert!(err.contains("invalid `expr`"), "{err}"); + assert!( + err.contains(r#"e2e query "sum by (job) (data)": invalid `expr`"#), + "{err}" + ); assert!(err.contains("not a subset"), "{err}"); + // A tree-only query is named by its leaves. + let mut table = tree_table(); + queries(&mut table).push(json!({"interval_secs": 1, "expr": + {"agg": "sum", "by": ["region"], "of": leaf("sum", "sum", 1000, &["job"])}})); + let err = tree_refusal(&table); + assert!( + err.contains(r#"e2e query with RQEs ["sum"]: invalid `expr`"#), + "{err}" + ); } #[test] @@ -1509,22 +1674,26 @@ mod tests { let cases = [ ( json!({"interval_secs": 1, "rqes": ["sum"]}), - "needs `promql`, `sql` or `expr`", + r#"e2e query reading ["sum"] needs `promql`, `sql` or `expr`"#, + ), + ( + json!({"interval_secs": 1}), + "e2e query with no text, `expr` or RQEs needs `promql`, `sql` or `expr`", ), ( json!({"promql": "sum by (job) (data)", "interval_secs": 1, "expr": http_leaf("sum", "sum", &[], "latency")}), - "written in PromQL but its `expr` reads records metrics", + r#"e2e query "sum by (job) (data)" is written in PromQL but its `expr` reads records metrics"#, ), ( json!({"interval_secs": 1, "expr": {"op": "+", "lhs": leaf("sum", "sum", 1000, &[]), "rhs": http_leaf("sum2", "sum", &[], "latency")}}), - "mixes records and non-records metrics", + r#"e2e query with RQEs ["sum", "sum2"]: its `expr` mixes records and non-records metrics"#, ), ( json!({"interval_secs": 1, "expr": {"scalar": 1.0}}), - "its `expr` reads no RQE", + "root reads no RQE leaf", ), ]; for (query, reason) in cases { diff --git a/asap-planner-rs/tests/saved_plan_e2e_tables.rs b/asap-planner-rs/tests/saved_plan_e2e_tables.rs index c82a3229..9475675c 100644 --- a/asap-planner-rs/tests/saved_plan_e2e_tables.rs +++ b/asap-planner-rs/tests/saved_plan_e2e_tables.rs @@ -1,8 +1,8 @@ //! `--planner saved` on real sketch-bench eval tables and their saved plans: -//! the `all` workload of sketch-bench's `rqe-optimizer/results/e2e-rate10` -//! (sketch-bench c1fd812), whose e2e queries carry RQE trees, and the pilot -//! random workload `seed-862/workload-01-pilot` (sketch-bench PR #226), whose -//! top-k queries are `topk by (region, service)`. +//! the `all` workload of sketch-bench's `rqe-optimizer/results/e2e-rate10`, +//! whose e2e queries carry RQE trees (all but its `rate` queries), and a +//! pilot random workload, `seed-862/workload-01-pilot`, whose top-k queries +//! are `topk by (region, service)`. use std::collections::{BTreeSet, HashMap}; use std::path::{Path, PathBuf}; @@ -36,6 +36,15 @@ fn convert_with_plans( table: &Path, plans: &str, language: QueryLanguage, +) -> anyhow::Result { + convert_filtered(table, plans, language, false) +} + +fn convert_filtered( + table: &Path, + plans: &str, + language: QueryLanguage, + expr_only: bool, ) -> anyhow::Result { let selector = PlanSelector { method: "asap", @@ -48,6 +57,7 @@ fn convert_with_plans( &selector, language, SCRAPE_INTERVAL_MS, + expr_only, ) } @@ -166,6 +176,61 @@ fn trees_add_only_rqe_configs() { } } +/// The `rate` queries have no tree: `--expr-only` deploys nothing for them +/// (no aggregation reads `data_total`) and leaves the tree queries' configs +/// as they are. +#[test] +fn expr_only_skips_the_rate_queries() { + let rate_queries: Vec = table_queries() + .iter() + .filter(|q| q.get("expr").is_none()) + .map(|q| q["promql"].as_str().unwrap().to_string()) + .collect(); + assert!(!rate_queries.is_empty()); + assert!(rate_queries.iter().all(|q| q.starts_with("rate("))); + let reads_rate_metric = |output: &PlannerOutput| { + output + .to_streaming_config(QueryLanguage::promql) + .unwrap() + .aggregation_configs + .values() + .any(|a| a.metric == "data_total") + }; + + let all = convert(&table_path(), QueryLanguage::promql); + assert!(reads_rate_metric(&all)); + + let plans = "synthetic-templatesall-shared1.plans.json.gz"; + let expr_only = convert_filtered(&table_path(), plans, QueryLanguage::promql, true).unwrap(); + assert!(!reads_rate_metric(&expr_only)); + let inference = expr_only + .to_inference_config(QueryLanguage::promql) + .unwrap(); + assert!(inference + .query_configs + .iter() + .all(|c| !rate_queries.contains(&c.query))); + assert_eq!( + inference.rqe_configs.len(), + all.to_inference_config(QueryLanguage::promql) + .unwrap() + .rqe_configs + .len() + ); + assert_eq!( + inference.query_configs.len() + rate_queries.len(), + all.inference_query_count() + ); + + // The SQL queries all have trees: nothing changes. + let sql_all = convert(&table_path(), QueryLanguage::sql); + let sql_expr_only = convert_filtered(&table_path(), plans, QueryLanguage::sql, true).unwrap(); + assert_eq!( + sql_expr_only.to_streaming_yaml_string().unwrap(), + sql_all.to_streaming_yaml_string().unwrap() + ); +} + const PILOT_PLANS: &str = "random-seed862-workload01-pilot.plans.json.gz"; /// The pilot table without its `avg by (...) (quantile_over_time(...))` From a791ac53fbbe2036b004fa2765696f7e96be5b59 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Wed, 14 Oct 2026 22:44:35 -0400 Subject: [PATCH 09/12] fix(query-engine): check each RQE leaf against its plan before reading Before reading a leaf, check it against the aggregation its rqe_configs entry names: same metric (table.value_column for SQL), a type that serves its capability with value weighting, a lookback its windows tile, top-k buckets equal to the heap grouping, and a grouping the aggregation serves (its own labels, a mergeable subset of its groups, or what its query-time stages regroup to). A mismatch is the request's fault: 400 bad_data, not 500. Also: drop error paths validation already rules out (missing phi/k, scalar root), make the end-timestamp alignment warning language-neutral, and stop claiming a config snapshot. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../src/engines/simple_engine/promql.rs | 2 +- .../src/engines/simple_engine/rqe.rs | 224 +++++++++++++----- .../src/tests/rqe_query_tests.rs | 154 +++++++++++- 3 files changed, 322 insertions(+), 58 deletions(-) diff --git a/asap-query-engine/src/engines/simple_engine/promql.rs b/asap-query-engine/src/engines/simple_engine/promql.rs index 36c48a96..f9f7bd29 100644 --- a/asap-query-engine/src/engines/simple_engine/promql.rs +++ b/asap-query-engine/src/engines/simple_engine/promql.rs @@ -126,7 +126,7 @@ impl SimpleEngine { } let aligned = (end_timestamp / interval_ms) * interval_ms; warn!( - "PromQL query end timestamp {} is not aligned with data ingestion interval of {} ms; \ + "Query end timestamp {} is not aligned with data ingestion interval of {} ms; \ aligning down to {}.", end_timestamp, interval_ms, aligned ); diff --git a/asap-query-engine/src/engines/simple_engine/rqe.rs b/asap-query-engine/src/engines/simple_engine/rqe.rs index ae0daec8..053fcef4 100644 --- a/asap-query-engine/src/engines/simple_engine/rqe.rs +++ b/asap-query-engine/src/engines/simple_engine/rqe.rs @@ -7,7 +7,12 @@ use super::promql::{combine_scalar, combine_vector_vector}; use super::{QueryExecutionError, QueryMetadata, QueryTimestamps, SimpleEngine}; use crate::data_model::RqeConfig; use crate::engines::query_result::{InstantVectorElement, QueryResult}; -use crate::engines::query_time_aggregation::{apply_instant_pipeline, pipeline_supports_labels}; +use crate::engines::query_time_aggregation::{ + apply_instant_pipeline, output_labels_for_aggregation, pipeline_supports_labels, +}; +use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_mode::{AggregationMode, CountMode}; +use asap_types::capability_matching::{compatible_agg_types, window_compatible}; use asap_types::query_config::{ QueryTimeAggregation, QueryTimeAggregationOperator, QueryTimeGrouping, QueryTimeGroupingMode, }; @@ -19,7 +24,8 @@ use std::collections::HashMap; #[derive(Debug, thiserror::Error)] pub enum RqeQueryError { - /// The request is wrong: an invalid tree, an unknown RQE id or a bad time. + /// The request is wrong: an invalid tree, an unknown RQE id, a leaf its + /// RQE's plan can't serve, or a bad time. #[error("{0}")] BadRequest(String), /// The request is valid but the engine could not answer it. @@ -56,12 +62,8 @@ impl SimpleEngine { ))); } let leaves = expr.leaves(); - if leaves.is_empty() { - return Err(RqeQueryError::BadRequest( - "an RQE expression must read at least one RQE".to_string(), - )); - } - // One snapshot of the configs, so every leaf reads the same plan. + // The leaves' entries, copied so the lock isn't held while reading. + // Configs are loaded once and don't change while a tree is answered. let configs: HashMap<&str, RqeConfig> = { let inference_config = self.inference_config.read().unwrap(); leaves @@ -83,9 +85,7 @@ impl SimpleEngine { RqeValue::Vector { labels, values } => { Ok((labels, QueryResult::vector(values, query_time))) } - RqeValue::Scalar(_) => Err(RqeQueryError::BadRequest( - "an RQE expression must evaluate to a vector".to_string(), - )), + RqeValue::Scalar(_) => unreachable!("a valid tree's root reads a leaf"), } } @@ -105,9 +105,7 @@ impl SimpleEngine { let RqeValue::Vector { labels, values } = self.evaluate_rqe_node(of, configs, query_time)? else { - return Err(RqeQueryError::BadRequest( - "an agg node needs a vector operand, not a scalar".to_string(), - )); + unreachable!("a valid agg operand reads a leaf"); }; let (labels, values) = apply_instant_pipeline(labels, values, &[regroup_stage(*agg, by)]) @@ -156,9 +154,9 @@ impl SimpleEngine { values: combine_scalar(values, scalar, &op, true), }) } - (RqeValue::Scalar(_), RqeValue::Scalar(_)) => Err(RqeQueryError::BadRequest( - "binary op needs at least one vector side".to_string(), - )), + (RqeValue::Scalar(_), RqeValue::Scalar(_)) => { + unreachable!("a valid binary op reads a leaf on one side") + } } } RqeExpr::Scalar { scalar } => Ok(RqeValue::Scalar(*scalar)), @@ -192,16 +190,8 @@ impl SimpleEngine { )) })?; - if leaf.capability == RqeCapability::TopkByValue { - let buckets = KeyByLabelNames::new(leaf.topk_by.clone().unwrap_or_default()); - if value_config.grouping_labels != buckets { - return Err(failed(format!( - "ranks within topk_by buckets {:?}, but its aggregation keeps a heap per {:?}", - buckets.labels, value_config.grouping_labels.labels - )) - .into()); - } - } + let (statistic_to_compute, query_kwargs) = leaf_statistic(leaf); + check_leaf_against_aggregation(leaf, statistic_to_compute, value_config)?; let end_timestamp = self.align_end_timestamp_promql(query_time); let start_timestamp = end_timestamp.checked_sub(leaf.lookback_ms).ok_or_else(|| { @@ -213,13 +203,15 @@ impl SimpleEngine { let leaf_labels = KeyByLabelNames::new(leaf.grouping.clone()); let read_labels = if config.query_time_aggregations.is_empty() { + check_read_grouping(leaf, &leaf_labels, statistic_to_compute, value_config)?; leaf_labels.clone() } else { let mut labels = value_config.grouping_labels.labels.clone(); labels.extend(value_config.aggregated_labels.labels.iter().cloned()); - KeyByLabelNames::new(labels) + let labels = KeyByLabelNames::new(labels); + check_roll_up_grouping(leaf, &leaf_labels, &labels, &config.query_time_aggregations)?; + labels }; - let (statistic_to_compute, query_kwargs) = leaf_statistic(leaf)?; let metadata = QueryMetadata { query_output_labels: read_labels, statistic_to_compute, @@ -249,51 +241,177 @@ impl SimpleEngine { let (labels, values) = if config.query_time_aggregations.is_empty() { (labels, vector.values) } else { - if !pipeline_supports_labels(&labels, &config.query_time_aggregations) - .map_err(failed)? - { - return Err(failed(format!( - "query-time aggregations can't regroup labels {:?}", - labels.labels - )) - .into()); - } apply_instant_pipeline(labels, vector.values, &config.query_time_aggregations) .map_err(failed)? }; - if labels != leaf_labels { - return Err(failed(format!( - "read labels {:?}, not the leaf's grouping {:?}", - labels.labels, leaf_labels.labels - )) - .into()); - } + debug_assert_eq!(labels, leaf_labels, "checked before reading"); Ok((labels, values)) } } -/// The statistic and its kwargs a leaf's capability computes. -fn leaf_statistic(leaf: &RqeLeaf) -> Result<(Statistic, HashMap), RqeQueryError> { - let missing = - |field: &str| RqeQueryError::BadRequest(format!("RQE {:?}: missing {field}", leaf.id)); - Ok(match leaf.capability { +fn bad_leaf(leaf: &RqeLeaf, reason: String) -> RqeQueryError { + RqeQueryError::BadRequest(format!("RQE {:?}: {reason}", leaf.id)) +} + +/// Whether the aggregation `leaf`'s RQE reads can answer it: same metric +/// (`table.value` for SQL), a type serving its capability, and windows that +/// tile its lookback. A top-k leaf must rank within the aggregation's heaps. +fn check_leaf_against_aggregation( + leaf: &RqeLeaf, + statistic: Statistic, + aggregation: &AggregationConfig, +) -> Result<(), RqeQueryError> { + // A SQL aggregation's metric is `table.value_column`. + let metric = match &leaf.value { + Some(value) => format!("{}.{value}", leaf.metric), + None => leaf.metric.clone(), + }; + if metric != aggregation.metric { + return Err(bad_leaf( + leaf, + format!( + "reads {metric:?}, but its aggregation {} is on {:?}", + aggregation.aggregation_id, aggregation.metric + ), + )); + } + // RQE sums and top-k rank values, so a SUM/COUNT sketch must weight by value. + let value_weighted = match aggregation.mode() { + Ok(AggregationMode::SumOrCount(mode)) => mode == CountMode::Sum, + Ok(_) => true, + Err(error) => { + return Err(QueryExecutionError::Native(format!("RQE {:?}: {error}", leaf.id)).into()) + } + }; + if !compatible_agg_types(statistic).contains(&aggregation.aggregation_type) || !value_weighted { + return Err(bad_leaf( + leaf, + format!( + "{:?} can't be answered by its aggregation {} ({} {:?})", + leaf.capability, + aggregation.aggregation_id, + aggregation.aggregation_type, + aggregation.aggregation_sub_type + ), + )); + } + if !window_compatible(aggregation, leaf.lookback_ms) { + return Err(bad_leaf( + leaf, + format!( + "its {} ms lookback is not a whole number of its aggregation {}'s {} ms windows \ + (sliding every {} ms)", + leaf.lookback_ms, + aggregation.aggregation_id, + aggregation.window_size_ms, + aggregation.slide_interval_ms + ), + )); + } + if leaf.capability == RqeCapability::TopkByValue { + let buckets = KeyByLabelNames::new(leaf.topk_by.clone().unwrap_or_default()); + if aggregation.grouping_labels != buckets { + return Err(bad_leaf( + leaf, + format!( + "ranks within topk_by buckets {:?}, but its aggregation keeps a heap per {:?}", + buckets.labels, aggregation.grouping_labels.labels + ), + )); + } + } + Ok(()) +} + +/// A leaf read with no query-time stage gets the aggregation's own labels +/// (its groups and keys), or, from a single-population aggregation that +/// merges across groups, a subset of its groups. +fn check_read_grouping( + leaf: &RqeLeaf, + leaf_labels: &KeyByLabelNames, + statistic: Statistic, + aggregation: &AggregationConfig, +) -> Result<(), RqeQueryError> { + let groups = &aggregation.grouping_labels.labels; + let mut served = groups.clone(); + served.extend(aggregation.aggregated_labels.labels.iter().cloned()); + let served = KeyByLabelNames::new(served); + let rolls_up = aggregation.aggregated_labels.labels.is_empty() + && aggregation.aggregation_type.merges_across_groups() + && statistic != Statistic::Topk + && leaf_labels + .labels + .iter() + .all(|label| groups.contains(label)); + if *leaf_labels == served || rolls_up { + return Ok(()); + } + Err(bad_leaf( + leaf, + format!( + "groups by {:?}, but its aggregation {} serves {:?}", + leaf_labels.labels, aggregation.aggregation_id, served.labels + ), + )) +} + +/// A roll-up's query-time stages must regroup the labels read into the +/// leaf's grouping. +fn check_roll_up_grouping( + leaf: &RqeLeaf, + leaf_labels: &KeyByLabelNames, + read_labels: &KeyByLabelNames, + stages: &[QueryTimeAggregation], +) -> Result<(), RqeQueryError> { + let invalid_stage = + |error: String| QueryExecutionError::Native(format!("RQE {:?}: {error}", leaf.id)); + if !pipeline_supports_labels(read_labels, stages).map_err(invalid_stage)? { + return Err(bad_leaf( + leaf, + format!( + "its query-time aggregations can't regroup labels {:?}", + read_labels.labels + ), + )); + } + let output = stages + .iter() + .try_fold(read_labels.clone(), |labels, stage| { + output_labels_for_aggregation(&labels, stage) + }) + .map_err(invalid_stage)?; + if output != *leaf_labels { + return Err(bad_leaf( + leaf, + format!( + "groups by {:?}, but its query-time aggregations regroup to {:?}", + leaf_labels.labels, output.labels + ), + )); + } + Ok(()) +} + +/// The statistic and its kwargs a validated leaf's capability computes. +fn leaf_statistic(leaf: &RqeLeaf) -> (Statistic, HashMap) { + match leaf.capability { RqeCapability::Sum => (Statistic::Sum, HashMap::new()), RqeCapability::Cardinality => (Statistic::Cardinality, HashMap::new()), RqeCapability::Quantile => { - let phi = leaf.phi.ok_or_else(|| missing("phi"))?; + let phi = leaf.phi.expect("a valid quantile leaf has phi"); ( Statistic::Quantile, HashMap::from([("quantile".to_string(), phi.to_string())]), ) } RqeCapability::TopkByValue => { - let k = leaf.k.ok_or_else(|| missing("k"))?; + let k = leaf.k.expect("a valid top-k leaf has k"); ( Statistic::Topk, HashMap::from([("k".to_string(), k.to_string())]), ) } - }) + } } /// The query-time stage that regroups a vector by `by` (all into one group diff --git a/asap-query-engine/src/tests/rqe_query_tests.rs b/asap-query-engine/src/tests/rqe_query_tests.rs index 9b85e47e..3e9f3171 100644 --- a/asap-query-engine/src/tests/rqe_query_tests.rs +++ b/asap-query-engine/src/tests/rqe_query_tests.rs @@ -42,6 +42,10 @@ mod tests { /// One heap per `job`, keyed by `srcip`: serves `topk by (job)`. const TOPK_BY_JOB_AGG: u64 = 6; const TOPK_BY_JOB_QUERY: &str = "topk by (job) (2, sum_over_time(flows[1s]))"; + /// A SQL table's `latency` column, as the planner names it. + const SQL_DDSKETCH_AGG: u64 = 7; + /// In `rqe_configs` but not in the streaming config. + const MISSING_AGG: u64 = 99; type Row = (Vec, f64); @@ -61,7 +65,13 @@ mod tests { AggregationConfig { aggregation_id: id, aggregation_type, - aggregation_sub_type: String::new(), + // Sums and top-k heaps weight by value. + aggregation_sub_type: match aggregation_type { + AggregationType::MultipleSum | AggregationType::CountMinSketchWithHeap => { + "sum".to_string() + } + _ => String::new(), + }, parameters: HashMap::new(), grouping_labels: KeyByLabelNames::new(strings(grouping)), aggregated_labels: KeyByLabelNames::new(strings(aggregated)), @@ -168,6 +178,13 @@ mod tests { &["job"], &["srcip"], ), + aggregation( + SQL_DDSKETCH_AGG, + AggregationType::DDSketch, + "http.latency", + &["job"], + &[], + ), ]), }); let store = Arc::new(SimpleMapStore::new( @@ -318,6 +335,10 @@ mod tests { "misrolled".to_string(), rqe_config(MULTI_SUM_AGG, vec![sum_by_job()]), ), + // The keyed sums at their own labels, with no stage. + ("events".to_string(), rqe_config(MULTI_SUM_AGG, vec![])), + ("sql_p90".to_string(), rqe_config(SQL_DDSKETCH_AGG, vec![])), + ("dangling".to_string(), rqe_config(MISSING_AGG, vec![])), ]), }; SimpleEngine::new( @@ -561,7 +582,7 @@ mod tests { let engine = engine(); for tree in [topk_by_job_leaf(None), topk_by_job_leaf(Some(&["srcip"]))] { let error = rqe_error(&engine, tree, QUERY_TIME_S); - assert!(matches!(&error, RqeQueryError::Execution(_)), "{error:?}"); + assert!(matches!(&error, RqeQueryError::BadRequest(_)), "{error:?}"); assert!(error.to_string().contains("top2_by_job"), "{error}"); assert!(error.to_string().contains("topk_by"), "{error}"); } @@ -598,8 +619,122 @@ mod tests { leaf("misrolled", "events", "sum", TWO_WINDOWS_MS, &["instance"]), QUERY_TIME_S, ); - assert!(matches!(&error, RqeQueryError::Execution(_)), "{error:?}"); + assert!(matches!(&error, RqeQueryError::BadRequest(_)), "{error:?}"); assert!(error.to_string().contains("misrolled"), "{error}"); + assert!(error.to_string().contains("regroup to"), "{error}"); + } + + // --- leaves checked against their RQE's aggregation --- + + fn assert_bad_leaf(tree: Value, reason: &str) { + let error = rqe_error(&engine(), tree.clone(), QUERY_TIME_S); + assert!( + matches!(&error, RqeQueryError::BadRequest(_)), + "{tree}: {error:?}" + ); + assert!(error.to_string().contains(reason), "{tree}: {error}"); + } + + #[test] + fn refuses_a_leaf_on_another_metric() { + assert_bad_leaf( + leaf("sum", "other", "sum", TWO_WINDOWS_MS, &["instance", "job"]), + r#"reads "other", but its aggregation 1 is on "http_requests""#, + ); + } + + #[test] + fn refuses_a_capability_its_aggregation_cannot_answer() { + let mut quantile = series_sum(); + quantile["rqe"]["capability"] = json!("quantile"); + quantile["rqe"]["phi"] = json!(0.5); + assert_bad_leaf(quantile, "can't be answered by its aggregation 1"); + let mut distinct = quantile_leaf(); + distinct["rqe"]["capability"] = json!("cardinality"); + distinct["rqe"].as_object_mut().unwrap().remove("phi"); + assert_bad_leaf(distinct, "can't be answered by its aggregation 2"); + } + + #[test] + fn refuses_a_lookback_its_windows_do_not_tile() { + assert_bad_leaf( + sum_leaf("sum", WINDOW_MS + WINDOW_MS / 2, &["instance", "job"]), + "1500 ms lookback is not a whole number of its aggregation 1's 1000 ms windows", + ); + // Any whole number of windows is served. + let three_windows = sum_leaf("sum", 3 * WINDOW_MS, &["instance", "job"]); + engine() + .handle_rqe_query(&expr(three_windows), QUERY_TIME_S) + .unwrap(); + } + + /// Without a stage, a leaf gets the aggregation's labels; a single- + /// population aggregation can also be merged to a subset of its groups. + #[test] + fn refuses_a_grouping_its_aggregation_does_not_serve() { + assert_bad_leaf( + sum_leaf("sum", TWO_WINDOWS_MS, &["instance", "job", "pod"]), + r#"groups by ["instance", "job", "pod"], but its aggregation 1 serves ["instance", "job"]"#, + ); + // Keyed sums keep their keys: they can't drop `instance` without a stage. + assert_bad_leaf( + leaf("events", "events", "sum", TWO_WINDOWS_MS, &["job"]), + r#"but its aggregation 5 serves ["instance", "job"]"#, + ); + let answer = rqe( + &engine(), + leaf( + "events", + "events", + "sum", + TWO_WINDOWS_MS, + &["instance", "job"], + ), + ); + assert_eq!( + answer.1, + vec![ + row(&["a", "api"], 5.0), + row(&["a", "web"], 8.0), + row(&["b", "api"], 2.0), + ] + ); + // A top-k heap ranks whole series: it is never merged to its buckets. + assert_bad_leaf( + { + let mut top = topk_by_job_leaf(Some(&["job"])); + top["rqe"]["grouping"] = json!(["job"]); + top + }, + "but its aggregation 6 serves", + ); + } + + /// A SQL leaf names its table and value column; the aggregation's + /// metric is `table.value_column`. + #[test] + fn reads_a_sql_leaf_by_table_and_value_column() { + let mut sql = leaf("sql_p90", "http", "quantile", TWO_WINDOWS_MS, &["job"]); + sql["rqe"]["phi"] = json!(0.9); + sql["rqe"]["value"] = json!("latency"); + engine() + .handle_rqe_query(&expr(sql.clone()), QUERY_TIME_S) + .expect("checks pass; the window has no data"); + sql["rqe"].as_object_mut().unwrap().remove("value"); + assert_bad_leaf( + sql, + r#"reads "http", but its aggregation 7 is on "http.latency""#, + ); + } + + /// Repeated labels would mislabel series that are matched by position. + #[test] + fn refuses_duplicate_labels() { + assert_bad_leaf( + json!({"op": "+", "lhs": sum_leaf("sum", TWO_WINDOWS_MS, &["instance", "job", "job"]), + "rhs": series_sum()}), + "more than once", + ); } /// Between ingestion ticks a leaf reads what the PromQL instant path @@ -889,6 +1024,16 @@ mod tests { .to_string(), "unknown RQE id", ), + ( + json!({"expr": topk_by_job_leaf(None), "time": QUERY_TIME_S}).to_string(), + "topk_by", + ), + ( + json!({"expr": leaf("misrolled", "events", "sum", TWO_WINDOWS_MS, &["instance"]), + "time": QUERY_TIME_S}) + .to_string(), + "regroup to", + ), ] { let (status, response) = post(&url, body.clone()).await; assert_eq!(status, 400, "{body}"); @@ -899,10 +1044,11 @@ mod tests { } } + /// The engine's own configs are broken: no fault of the request. #[tokio::test] async fn route_reports_an_execution_failure_as_500() { let (_server, url) = server().await; - let body = json!({"expr": leaf("misrolled", "events", "sum", TWO_WINDOWS_MS, &["instance"]), + let body = json!({"expr": leaf("dangling", "events", "sum", TWO_WINDOWS_MS, &["job"]), "time": QUERY_TIME_S}); let (status, response) = post(&url, body.to_string()).await; assert_eq!(status, 500); From 60a78443d69dcd3f08ee5a114057babcf875ffe7 Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 00:23:37 -0400 Subject: [PATCH 10/12] style(asap-common): state behaviour in duplicate-label test comment Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- asap-common/dependencies/rs/asap_types/src/rqe_expr.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs index 53907354..22d91b52 100644 --- a/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs +++ b/asap-common/dependencies/rs/asap_types/src/rqe_expr.rs @@ -768,7 +768,7 @@ mod tests { assert_eq!(parse(by).validate(), dup("agg by")); let topk_by = topk_by_leaf(&["job", "region"], json!(["job", "job"])); assert_eq!(parse(topk_by).validate(), dup("topk_by")); - // A duplicated grouping no longer slips past the binary label-set check. + // A duplicated grouping fails the binary label-set check. let binary = json!({"op": "+", "lhs": sum_leaf("a", &["job", "job"]), "rhs": sum_leaf("b", &["job"])}); assert_eq!(parse(binary).validate(), dup("grouping")); } From 2f18617a15f51652b371b5709ecac992c1f9448a Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 00:25:40 -0400 Subject: [PATCH 11/12] fix(planner): require rqes with text; --expr-only plans tree leaves only - A query with `promql`/`sql` must name its `rqes`: text leaves map to plan RQEs by `rqes`, never by the tree's leaf order. Only a tree-only query may omit them. - `--expr-only` no longer plans query text: it writes only the tree leaves' deployments and `rqe_configs`, and no `query_configs`. - `leaf_item` drops the lookback check `RqeExpr::validate` owns, and shares one metric lookup (`leaf_metric`) with `is_sql`. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- asap-planner-rs/src/main.rs | 6 +- asap-planner-rs/src/optimizer/saved_plan.rs | 86 +++++++++++-------- .../tests/saved_plan_e2e_tables.rs | 69 ++++++++------- 3 files changed, 89 insertions(+), 72 deletions(-) diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index 4e831643..ae063089 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -87,8 +87,10 @@ struct Args { #[arg(long = "sla-ms")] sla_ms: Option, - /// Saved only. Plan only the e2e queries that have an RQE tree (`expr`); - /// the others (e.g. `rate`) are skipped, so nothing is deployed for them. + /// Saved only. Plan only the tree leaves of the e2e queries that have an + /// RQE tree (`expr`): no query text is planned, so no `query_configs` are + /// written, only deployments and `rqe_configs`. Queries without a tree + /// (e.g. `rate`) are skipped. #[arg(long = "expr-only")] expr_only: bool, diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index d2b73e7e..afcff27a 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -60,8 +60,8 @@ struct E2eQuery { sql: Option, interval_secs: u64, /// Plan RQE ids: for `promql`, one per leaf in order; for `sql`, a single - /// leaf, every RQE it reads. With `expr`, its leaves' ids, which this - /// may omit. + /// leaf, every RQE it reads. Required with text; a query with only `expr` + /// may omit them, and then reads its leaves' ids. #[serde(default)] rqes: Vec, /// Leaves name plan RQE ids. @@ -99,10 +99,7 @@ impl E2eQuery { None => None, Some(expr) => { let reads_records = |leaf: &RqeLeaf| { - let metric = metrics.get(&leaf.metric).with_context(|| { - format!("RQE {:?} reads unknown metric {:?}", leaf.id, leaf.metric) - })?; - anyhow::Ok(metric.kind == RECORDS_METRIC_TYPE) + anyhow::Ok(leaf_metric(leaf, metrics)?.kind == RECORDS_METRIC_TYPE) }; let leaves = expr.leaves(); let (first, rest) = leaves.split_first().expect("a valid tree reads a leaf"); @@ -129,11 +126,18 @@ impl E2eQuery { } } - /// The plan RQEs the query reads: `rqes`, or, if it omits them, its - /// tree's leaves. With both, they must name the same RQEs, in any order - /// and however often each appears. + /// The plan RQEs the query reads: `rqes`, or, for a query with only a + /// tree that omits them, its leaves. With both, they must name the same + /// RQEs, in any order and however often each appears. fn rqe_ids(&self) -> anyhow::Result> { let rqes: Vec<&str> = self.rqes.iter().map(String::as_str).collect(); + // A text query's `rqes` map its text leaves to plan RQEs, which a + // tree's leaf order can't. + ensure!( + self.text()?.is_none() || !rqes.is_empty(), + "e2e query {} has text but no `rqes`", + self.name() + ); let Some(expr) = &self.expr else { return Ok(rqes); }; @@ -155,6 +159,16 @@ fn leaf_ids(expr: &RqeExpr) -> Vec<&str> { expr.leaves().iter().map(|l| l.id.as_str()).collect() } +/// The table metric an RQE leaf reads. +fn leaf_metric<'a>( + leaf: &RqeLeaf, + metrics: &'a BTreeMap, +) -> anyhow::Result<&'a E2eMetric> { + metrics + .get(&leaf.metric) + .with_context(|| format!("RQE {:?} reads unknown metric {:?}", leaf.id, leaf.metric)) +} + /// A query planned from its text, and the plan RQEs it reads. struct TextQuery<'a> { query: &'a E2eQuery, @@ -301,7 +315,8 @@ fn read_json(path: &Path) -> anyhow::Result { /// Streaming and inference YAML for the selected plan of `plan_path` (JSON, /// gzipped or not) over the workload of `table_path`: its queries written in -/// `language`, and only those with an RQE tree if `expr_only`. +/// `language`. With `expr_only`, only the queries with an RQE tree, planned +/// from their tree leaves alone: no text items and no `query_configs`. pub fn saved_plan_output( table_path: &Path, plan_path: &Path, @@ -358,7 +373,8 @@ pub fn saved_plan_output( c.rqe ); } - // Queries in `language`: those with text, and those with a tree. + // Queries in `language`: those with text (unless `expr_only`), and those + // with a tree. let mut selected: Vec = Vec::new(); let mut trees: Vec<(&E2eQuery, &RqeExpr)> = Vec::new(); for q in &e2e.queries { @@ -373,7 +389,7 @@ pub fn saved_plan_output( continue; } let rqes = q.rqe_ids()?; - if let Some(text) = q.text()? { + if let (Some(text), false) = (q.text()?, expr_only) { selected.push(TextQuery { query: q, text, @@ -415,7 +431,6 @@ pub fn saved_plan_output( let leaves: Vec<(String, Vec<&str>)> = match config { // A SQL query is one leaf, and its RQEs a set. MilpConfig::Sql(_) => { - ensure!(!rqe_ids.is_empty(), "query {text:?} names no RQEs"); let mut rqes = rqe_ids; rqes.sort_unstable(); vec![(text.to_string(), rqes)] @@ -573,9 +588,7 @@ fn leaf_item( metrics: &BTreeMap, ) -> anyhow::Result { let id = &leaf.id; - let m = metrics - .get(&leaf.metric) - .with_context(|| format!("RQE {id:?} reads unknown metric {:?}", leaf.metric))?; + let m = leaf_metric(leaf, metrics)?; let metric = match (m.kind == RECORDS_METRIC_TYPE, &leaf.value) { (true, Some(value)) => { ensure!( @@ -602,7 +615,6 @@ fn leaf_item( leaf.metric ); } - ensure!(leaf.lookback_ms > 0, "RQE {id:?} looks back 0 ms"); ensure!(interval_ms > 0, "RQE {id:?} repeats every 0 s"); let (statistic, topk_count_events) = match leaf.capability { RqeCapability::Sum => (Statistic::Sum, None), @@ -1380,7 +1392,7 @@ mod tests { #[test] fn writes_a_sketch_roll_up_rqe_without_a_stage() { let table = json!({"workloads": [{"e2e": { - "queries": [{"promql": "quantile by (job) (0.5, data)", "interval_secs": 1, + "queries": [{"promql": "quantile by (job) (0.5, data)", "interval_secs": 1, "rqes": ["p50"], "expr": quantile_leaf("p50", 0.5, 1000, &["job"])}], "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} }}]}); @@ -1572,8 +1584,9 @@ mod tests { ); } - /// `--expr-only` skips the queries without a tree: nothing is deployed - /// or written for them. + /// `--expr-only` skips the queries without a tree, and plans the others + /// from their tree leaves alone: only deployments and `rqe_configs`, no + /// `query_configs`. #[test] fn expr_only_plans_only_the_queries_with_a_tree() { let mut table = tree_table(); @@ -1600,12 +1613,9 @@ mod tests { let expr_only = run(true); // Only the DDSketch the quantile trees read; no sum. assert_eq!(expr_only.streaming_aggregation_count(), 1); - assert_eq!(expr_only.inference_query_count(), 2); + assert_eq!(expr_only.inference_query_count(), 0); let inference = inference_of(&expr_only, QueryLanguage::promql); - assert!(inference - .query_configs - .iter() - .all(|q| q.query != "sum by (job) (data)")); + assert!(inference.query_configs.is_empty()); let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); ids.sort(); assert_eq!(ids, ["p50", "p90"]); @@ -1618,20 +1628,22 @@ mod tests { assert!(err.contains("no e2e query with an `expr`"), "{err}"); } - /// `rqes` may be omitted: the tree names the RQEs. + /// A query with text needs `rqes`, even with a tree: its text leaves + /// map to RQEs by `rqes`, not by the tree's leaf order. Only a query + /// with just a tree may omit them (`plans_a_tree_only_query`). #[test] - fn reads_the_rqes_of_a_tree_without_rqes() { + fn refuses_a_query_with_text_but_no_rqes() { let mut table = tree_table(); - for q in queries(&mut table).iter_mut() { - q.as_object_mut().unwrap().remove("rqes"); - } - let output = run_table(&table, &plans(shared_p50()), QueryLanguage::promql).unwrap(); - assert_eq!(output.inference_query_count(), 3); - assert_eq!( - inference_of(&output, QueryLanguage::promql) - .rqe_configs - .len(), - 3 + queries(&mut table)[1] + .as_object_mut() + .unwrap() + .remove("rqes"); + let err = tree_refusal(&table); + assert!( + err.contains( + r#"e2e query "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])" has text but no `rqes`"# + ), + "{err}" ); } diff --git a/asap-planner-rs/tests/saved_plan_e2e_tables.rs b/asap-planner-rs/tests/saved_plan_e2e_tables.rs index 9475675c..d4abc599 100644 --- a/asap-planner-rs/tests/saved_plan_e2e_tables.rs +++ b/asap-planner-rs/tests/saved_plan_e2e_tables.rs @@ -176,9 +176,9 @@ fn trees_add_only_rqe_configs() { } } -/// The `rate` queries have no tree: `--expr-only` deploys nothing for them -/// (no aggregation reads `data_total`) and leaves the tree queries' configs -/// as they are. +/// `--expr-only` plans tree leaves alone: no `query_configs`, every tree +/// leaf's `rqe_configs` entry, and only the aggregations those entries read. +/// The `rate` queries have no tree, so nothing reads `data_total`. #[test] fn expr_only_skips_the_rate_queries() { let rate_queries: Vec = table_queries() @@ -196,39 +196,42 @@ fn expr_only_skips_the_rate_queries() { .values() .any(|a| a.metric == "data_total") }; - - let all = convert(&table_path(), QueryLanguage::promql); - assert!(reads_rate_metric(&all)); + assert!(reads_rate_metric(&convert( + &table_path(), + QueryLanguage::promql + ))); let plans = "synthetic-templatesall-shared1.plans.json.gz"; - let expr_only = convert_filtered(&table_path(), plans, QueryLanguage::promql, true).unwrap(); - assert!(!reads_rate_metric(&expr_only)); - let inference = expr_only - .to_inference_config(QueryLanguage::promql) - .unwrap(); - assert!(inference - .query_configs - .iter() - .all(|c| !rate_queries.contains(&c.query))); - assert_eq!( - inference.rqe_configs.len(), - all.to_inference_config(QueryLanguage::promql) - .unwrap() + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let all = convert(&table_path(), language); + let expr_only = convert_filtered(&table_path(), plans, language, true).unwrap(); + if language == QueryLanguage::promql { + assert!(!reads_rate_metric(&expr_only)); + } + let inference = expr_only.to_inference_config(language).unwrap(); + assert!(inference.query_configs.is_empty(), "{language:?}"); + let rqe_ids = |i: &asap_types::InferenceConfig| { + i.rqe_configs.keys().cloned().collect::>() + }; + assert_eq!( + rqe_ids(&inference), + rqe_ids(&all.to_inference_config(language).unwrap()), + "{language:?}" + ); + let read: BTreeSet = inference .rqe_configs - .len() - ); - assert_eq!( - inference.query_configs.len() + rate_queries.len(), - all.inference_query_count() - ); - - // The SQL queries all have trees: nothing changes. - let sql_all = convert(&table_path(), QueryLanguage::sql); - let sql_expr_only = convert_filtered(&table_path(), plans, QueryLanguage::sql, true).unwrap(); - assert_eq!( - sql_expr_only.to_streaming_yaml_string().unwrap(), - sql_all.to_streaming_yaml_string().unwrap() - ); + .values() + .flat_map(|c| c.aggregations.iter().map(|a| a.aggregation_id)) + .collect(); + let deployed: BTreeSet = expr_only + .to_streaming_config(language) + .unwrap() + .aggregation_configs + .keys() + .copied() + .collect(); + assert_eq!(deployed, read, "{language:?}"); + } } const PILOT_PLANS: &str = "random-seed862-workload01-pilot.plans.json.gz"; From c2064d2e582a617800a27cbf455e19056e8c025d Mon Sep 17 00:00:00 2001 From: Milind Srivastava Date: Thu, 15 Oct 2026 00:30:38 -0400 Subject: [PATCH 12/12] fix(query-engine): refuse RQE leaves their plan can't serve; honour disabled handling - A leaf whose lookback reads more closed windows than its `rqe_configs` entry retains of the value aggregation is a 400. Windows read are `(lookback - window) / slide + 1`, as the planner sizes retention. - A top-k leaf whose `k` exceeds its CountMinSketchWithHeap's `heapsize` is a 400 (`cms_heap_params` is now shared with the accumulator factory). - `POST /api/v1/rqe_query` answers 500 `internal` when request handling is disabled: it has no fallback, like the instant route without one. - The tree reads one streaming-config snapshot for all its leaves. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_011gKTCAEz3sJ1Hzde16xxU1 --- .../src/drivers/query/servers/http.rs | 12 +- .../src/engines/simple_engine/rqe.rs | 127 ++++++++++++++++-- .../precompute_engine/accumulator_factory.rs | 2 +- .../src/tests/rqe_query_tests.rs | 78 ++++++++++- 4 files changed, 204 insertions(+), 15 deletions(-) diff --git a/asap-query-engine/src/drivers/query/servers/http.rs b/asap-query-engine/src/drivers/query/servers/http.rs index 16f21c0c..2e3806b7 100644 --- a/asap-query-engine/src/drivers/query/servers/http.rs +++ b/asap-query-engine/src/drivers/query/servers/http.rs @@ -469,8 +469,18 @@ fn prometheus_error(status: StatusCode, error_type: &str, error: &str) -> Respon /// `POST /api/v1/rqe_query`: answers with Prometheus instant-vector JSON. Never /// forwards to a fallback backend: an invalid request is a 400, a failed -/// execution a 500, and a window without data yet an empty result. +/// execution a 500, and a window without data yet an empty result. With +/// request handling disabled it answers 500 `internal`, as the instant route +/// does when it has no fallback either. async fn handle_rqe_query(State(state): State, body: Bytes) -> Response { + if !state.config.handle_http_requests { + debug!("HTTP request handling is disabled; RQE queries have no fallback"); + return prometheus_error( + StatusCode::INTERNAL_SERVER_ERROR, + "internal", + "RQE query handling is disabled", + ); + } let start_time = Instant::now(); let request: RqeQueryRequest = match serde_json::from_slice(&body) { Ok(request) => request, diff --git a/asap-query-engine/src/engines/simple_engine/rqe.rs b/asap-query-engine/src/engines/simple_engine/rqe.rs index 053fcef4..63065672 100644 --- a/asap-query-engine/src/engines/simple_engine/rqe.rs +++ b/asap-query-engine/src/engines/simple_engine/rqe.rs @@ -5,11 +5,12 @@ use super::promql::{combine_scalar, combine_vector_vector}; use super::{QueryExecutionError, QueryMetadata, QueryTimestamps, SimpleEngine}; -use crate::data_model::RqeConfig; +use crate::data_model::{RqeConfig, StreamingConfig}; use crate::engines::query_result::{InstantVectorElement, QueryResult}; use crate::engines::query_time_aggregation::{ apply_instant_pipeline, output_labels_for_aggregation, pipeline_supports_labels, }; +use crate::precompute_engine::accumulator_factory::cms_heap_params; use asap_types::aggregation_config::AggregationConfig; use asap_types::aggregation_mode::{AggregationMode, CountMode}; use asap_types::capability_matching::{compatible_agg_types, window_compatible}; @@ -62,6 +63,8 @@ impl SimpleEngine { ))); } let leaves = expr.leaves(); + // One snapshot (a shared pointer, not a copy) for every leaf to read. + let streaming_config = self.streaming_config.read().unwrap().clone(); // The leaves' entries, copied so the lock isn't held while reading. // Configs are loaded once and don't change while a tree is answered. let configs: HashMap<&str, RqeConfig> = { @@ -81,7 +84,7 @@ impl SimpleEngine { }; let query_time = Self::convert_query_time_to_data_time(time); - match self.evaluate_rqe_node(expr, &configs, query_time)? { + match self.evaluate_rqe_node(expr, &configs, &streaming_config, query_time)? { RqeValue::Vector { labels, values } => { Ok((labels, QueryResult::vector(values, query_time))) } @@ -93,17 +96,22 @@ impl SimpleEngine { &self, expr: &RqeExpr, configs: &HashMap<&str, RqeConfig>, + streaming_config: &StreamingConfig, query_time: u64, ) -> Result { match expr { RqeExpr::Rqe { rqe } => { - let (labels, values) = - self.evaluate_rqe_leaf(rqe, &configs[rqe.id.as_str()], query_time)?; + let (labels, values) = self.evaluate_rqe_leaf( + rqe, + &configs[rqe.id.as_str()], + streaming_config, + query_time, + )?; Ok(RqeValue::Vector { labels, values }) } RqeExpr::Agg { agg, by, of } => { let RqeValue::Vector { labels, values } = - self.evaluate_rqe_node(of, configs, query_time)? + self.evaluate_rqe_node(of, configs, streaming_config, query_time)? else { unreachable!("a valid agg operand reads a leaf"); }; @@ -114,8 +122,8 @@ impl SimpleEngine { } RqeExpr::Binary { op, lhs, rhs } => { let op = binary_token(*op); - let lhs = self.evaluate_rqe_node(lhs, configs, query_time)?; - let rhs = self.evaluate_rqe_node(rhs, configs, query_time)?; + let lhs = self.evaluate_rqe_node(lhs, configs, streaming_config, query_time)?; + let rhs = self.evaluate_rqe_node(rhs, configs, streaming_config, query_time)?; match (lhs, rhs) { ( RqeValue::Vector { @@ -173,6 +181,7 @@ impl SimpleEngine { &self, leaf: &RqeLeaf, config: &RqeConfig, + streaming_config: &StreamingConfig, query_time: u64, ) -> Result<(KeyByLabelNames, Vec), RqeQueryError> { let failed = @@ -180,7 +189,6 @@ impl SimpleEngine { let agg_info = self .get_aggregation_id_info(&config.aggregations) .map_err(failed)?; - let streaming_config = self.streaming_config.read().unwrap().clone(); let value_config = streaming_config .get_aggregation_config(agg_info.aggregation_id_for_value) .ok_or_else(|| { @@ -192,6 +200,7 @@ impl SimpleEngine { let (statistic_to_compute, query_kwargs) = leaf_statistic(leaf); check_leaf_against_aggregation(leaf, statistic_to_compute, value_config)?; + check_leaf_retention(leaf, config, value_config)?; let end_timestamp = self.align_end_timestamp_promql(query_time); let start_timestamp = end_timestamp.checked_sub(leaf.lookback_ms).ok_or_else(|| { @@ -255,7 +264,8 @@ fn bad_leaf(leaf: &RqeLeaf, reason: String) -> RqeQueryError { /// Whether the aggregation `leaf`'s RQE reads can answer it: same metric /// (`table.value` for SQL), a type serving its capability, and windows that -/// tile its lookback. A top-k leaf must rank within the aggregation's heaps. +/// tile its lookback. A top-k leaf must rank within the aggregation's heaps, +/// and ask for no more than a heap holds. fn check_leaf_against_aggregation( leaf: &RqeLeaf, statistic: Statistic, @@ -309,6 +319,19 @@ fn check_leaf_against_aggregation( )); } if leaf.capability == RqeCapability::TopkByValue { + // Top-k is served only by a CountMinSketchWithHeap (checked above). + let (_, _, heap_size) = cms_heap_params(aggregation) + .map_err(|error| QueryExecutionError::Native(format!("RQE {:?}: {error}", leaf.id)))?; + let k = leaf.k.expect("a valid top-k leaf has k"); + if k > heap_size as u64 { + return Err(bad_leaf( + leaf, + format!( + "asks for the top {k}, but its aggregation {} keeps a heap of {heap_size}", + aggregation.aggregation_id + ), + )); + } let buckets = KeyByLabelNames::new(leaf.topk_by.clone().unwrap_or_default()); if aggregation.grouping_labels != buckets { return Err(bad_leaf( @@ -323,6 +346,43 @@ fn check_leaf_against_aggregation( Ok(()) } +/// The closed windows a lookback reads: those wholly inside it, which start +/// in `[end - lookback, end - window]`, so `(lookback - window) / slide + 1`. +/// The planner retains this many for the lookback (its `retained_windows`). +/// The lookback must already be `window_compatible`. +fn windows_needed(aggregation: &AggregationConfig, lookback_ms: u64) -> u64 { + (lookback_ms - aggregation.window_size_ms) / SimpleEngine::bucket_step_ms(aggregation) + 1 +} + +/// A leaf may look back only as many windows as its RQE's plan retains of +/// the aggregation it reads: older ones may already be gone. +fn check_leaf_retention( + leaf: &RqeLeaf, + config: &RqeConfig, + aggregation: &AggregationConfig, +) -> Result<(), RqeQueryError> { + let retained = config + .aggregations + .iter() + .find(|reference| reference.aggregation_id == aggregation.aggregation_id) + .and_then(|reference| reference.num_aggregates_to_retain); + let Some(retained) = retained else { + return Ok(()); + }; + let needed = windows_needed(aggregation, leaf.lookback_ms); + if needed > retained { + return Err(bad_leaf( + leaf, + format!( + "its {} ms lookback reads {needed} windows of its aggregation {}, \ + but its plan retains {retained}", + leaf.lookback_ms, aggregation.aggregation_id + ), + )); + } + Ok(()) +} + /// A leaf read with no query-time stage gets the aggregation's own labels /// (its groups and keys), or, from a single-population aggregation that /// merges across groups, a subset of its groups. @@ -446,3 +506,52 @@ fn binary_token(op: RqeBinaryOp) -> TokenType { RqeBinaryOp::Div => T_DIV, }) } + +#[cfg(test)] +mod tests { + use super::*; + use asap_types::enums::{AggregationType, WindowType}; + + fn aggregation(window_size_ms: u64, slide_interval_ms: u64) -> AggregationConfig { + let window_type = if window_size_ms == slide_interval_ms { + WindowType::Tumbling + } else { + WindowType::Sliding + }; + AggregationConfig::new( + 1, + AggregationType::Sum, + String::new(), + HashMap::new(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + window_size_ms, + slide_interval_ms, + window_type, + String::new(), + "m".to_string(), + None, + None, + None, + None, + ) + } + + #[test] + fn windows_needed_counts_the_closed_windows_inside_the_lookback() { + const MINUTE_MS: u64 = 60_000; + // Tumbling: one window per window width. + assert_eq!( + windows_needed(&aggregation(MINUTE_MS, MINUTE_MS), 3 * MINUTE_MS), + 3 + ); + // A tumbling config may leave the slide unset. + assert_eq!(windows_needed(&aggregation(MINUTE_MS, 0), 3 * MINUTE_MS), 3); + // 5m windows sliding every minute: the newest for 5m, eleven for 15m. + let sliding = aggregation(5 * MINUTE_MS, MINUTE_MS); + assert_eq!(windows_needed(&sliding, 5 * MINUTE_MS), 1); + assert_eq!(windows_needed(&sliding, 15 * MINUTE_MS), 11); + } +} diff --git a/asap-query-engine/src/precompute_engine/accumulator_factory.rs b/asap-query-engine/src/precompute_engine/accumulator_factory.rs index e55149b5..a03023b8 100644 --- a/asap-query-engine/src/precompute_engine/accumulator_factory.rs +++ b/asap-query-engine/src/precompute_engine/accumulator_factory.rs @@ -926,7 +926,7 @@ fn hydra_kll_params(config: &AggregationConfig) -> Result<(usize, usize, u16), S /// then falls back to the `row_num`/`col_num`/`heap_size` aliases. All three /// parameters are required — the planner always emits them and their absence /// indicates a malformed config. -fn cms_heap_params(config: &AggregationConfig) -> Result<(usize, usize, usize), String> { +pub(crate) fn cms_heap_params(config: &AggregationConfig) -> Result<(usize, usize, usize), String> { let read = |names: &[&str]| -> Result { names .iter() diff --git a/asap-query-engine/src/tests/rqe_query_tests.rs b/asap-query-engine/src/tests/rqe_query_tests.rs index 3e9f3171..7f9b13f7 100644 --- a/asap-query-engine/src/tests/rqe_query_tests.rs +++ b/asap-query-engine/src/tests/rqe_query_tests.rs @@ -44,6 +44,10 @@ mod tests { const TOPK_BY_JOB_QUERY: &str = "topk by (job) (2, sum_over_time(flows[1s]))"; /// A SQL table's `latency` column, as the planner names it. const SQL_DDSKETCH_AGG: u64 = 7; + /// Each top-k aggregation's heap capacity. + const HEAP_SIZE: usize = 32; + const CMS_DEPTH: usize = 3; + const CMS_WIDTH: usize = 1024; /// In `rqe_configs` but not in the streaming config. const MISSING_AGG: u64 = 99; @@ -72,7 +76,14 @@ mod tests { } _ => String::new(), }, - parameters: HashMap::new(), + parameters: match aggregation_type { + AggregationType::CountMinSketchWithHeap => HashMap::from([ + ("depth".to_string(), json!(CMS_DEPTH)), + ("width".to_string(), json!(CMS_WIDTH)), + ("heapsize".to_string(), json!(HEAP_SIZE)), + ]), + _ => HashMap::new(), + }, grouping_labels: KeyByLabelNames::new(strings(grouping)), aggregated_labels: KeyByLabelNames::new(strings(aggregated)), rollup_labels: KeyByLabelNames::empty(), @@ -248,7 +259,7 @@ mod tests { ); insert(HLL_AGG, QUERY_TIME_MS, Some(&["web"]), Box::new(hll(0..50))); - let mut heap = CountMinSketchWithHeapAccumulator::new(3, 1024, 32); + let mut heap = CountMinSketchWithHeapAccumulator::new(CMS_DEPTH, CMS_WIDTH, HEAP_SIZE); for i in 1..=5u64 { heap.inner.update(&format!("10.0.0.{i}"), (i * 10) as f64); } @@ -257,7 +268,7 @@ mod tests { // `api`'s smallest sources outweigh `web`'s largest, so a global // ranking would keep only `api` rows. for (job, sums) in [("api", [100.0, 200.0, 300.0]), ("web", [10.0, 20.0, 30.0])] { - let mut heap = CountMinSketchWithHeapAccumulator::new(3, 1024, 32); + let mut heap = CountMinSketchWithHeapAccumulator::new(CMS_DEPTH, CMS_WIDTH, HEAP_SIZE); for (i, sum) in sums.into_iter().enumerate() { heap.inner.update(&format!("10.0.0.{}", i + 1), sum); } @@ -319,6 +330,14 @@ mod tests { ("sum".to_string(), rqe_config(SUM_AGG, vec![])), ("sum_by_job".to_string(), rqe_config(SUM_AGG, vec![])), ("latest".to_string(), rqe_config(SUM_AGG, vec![])), + ( + // Planned for a lookback of two windows. + "sum_retains_two".to_string(), + RqeConfig { + aggregations: vec![AggregationReference::new(SUM_AGG, Some(2))], + query_time_aggregations: vec![], + }, + ), ("p90".to_string(), rqe_config(DDSKETCH_AGG, vec![])), ("distinct".to_string(), rqe_config(HLL_AGG, vec![])), ("top3".to_string(), rqe_config(TOPK_AGG, vec![])), @@ -668,6 +687,41 @@ mod tests { .unwrap(); } + #[test] + fn refuses_a_lookback_longer_than_its_plan_retains() { + let series = ["instance", "job"]; + assert_bad_leaf( + sum_leaf("sum_retains_two", 3 * WINDOW_MS, &series), + "its 3000 ms lookback reads 3 windows of its aggregation 1, but its plan retains 2", + ); + // The retained windows are served. + let engine = engine(); + assert_eq!( + rqe( + &engine, + sum_leaf("sum_retains_two", TWO_WINDOWS_MS, &series) + ), + rqe(&engine, series_sum()) + ); + } + + #[test] + fn refuses_a_topk_k_its_heap_cannot_hold() { + let mut too_many = topk_leaf(); + too_many["rqe"]["k"] = json!(HEAP_SIZE + 1); + assert_bad_leaf( + too_many, + &format!( + "asks for the top 33, but its aggregation {TOPK_AGG} keeps a heap of {HEAP_SIZE}" + ), + ); + // A whole heap is served. + let mut whole_heap = topk_leaf(); + whole_heap["rqe"]["k"] = json!(HEAP_SIZE); + let (_, rows) = rqe(&engine(), whole_heap); + assert_eq!(rows.len(), 5, "every source fits the heap"); + } + /// Without a stage, a leaf gets the aggregation's labels; a single- /// population aggregation can also be merged to a subset of its groups. #[test] @@ -929,6 +983,10 @@ mod tests { // --- HTTP route --- async fn server() -> (HttpServer, String) { + server_handling(true).await + } + + async fn server_handling(handle_http_requests: bool) -> (HttpServer, String) { let engine = Arc::new(engine()); let store: Arc = Arc::new(SimpleMapStore::new( Arc::new(StreamingConfig::default()), @@ -936,7 +994,7 @@ mod tests { )); let config = HttpServerConfig { port: 0, - handle_http_requests: true, + handle_http_requests, // Forwarding on, so a test would see any fallback attempt fail. adapter_config: AdapterConfig::prometheus_promql( "http://127.0.0.1:1".to_string(), @@ -1055,4 +1113,16 @@ mod tests { assert_eq!(response["status"], "error"); assert_eq!(response["errorType"], "internal"); } + + /// RQE queries have no fallback: disabled handling refuses them. + #[tokio::test] + async fn route_refuses_when_handling_is_disabled() { + let (_server, url) = server_handling(false).await; + let body = json!({"expr": series_sum(), "time": QUERY_TIME_S}); + let (status, response) = post(&url, body.to_string()).await; + assert_eq!(status, 500); + assert_eq!(response["status"], "error"); + assert_eq!(response["errorType"], "internal"); + assert_eq!(response["error"], "RQE query handling is disabled"); + } }