diff --git a/asap-common/dependencies/rs/asap_types/src/inference_config.rs b/asap-common/dependencies/rs/asap_types/src/inference_config.rs index cb674149..3b96e037 100644 --- a/asap-common/dependencies/rs/asap_types/src/inference_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/inference_config.rs @@ -1,6 +1,6 @@ use anyhow::Result; use serde_yaml::Value; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::fs::File; use std::io::BufReader; @@ -21,11 +21,21 @@ pub enum SchemaConfig { ElasticSQL(SQLSchema), } +/// How the engine answers one RQE leaf directly: the aggregations it reads +/// and the query-time stages that regroup them to the RQE's grouping. +#[derive(Debug, Clone)] +pub struct RqeConfig { + pub aggregations: Vec, + pub query_time_aggregations: Vec, +} + #[derive(Debug, Clone)] pub struct InferenceConfig { pub schema: SchemaConfig, pub query_configs: Vec, pub cleanup_policy: CleanupPolicy, + /// RQE id -> how to answer that RQE leaf; empty unless planned from RQE trees. + pub rqe_configs: HashMap, } impl InferenceConfig { @@ -42,6 +52,7 @@ impl InferenceConfig { schema, query_configs: Vec::new(), cleanup_policy, + rqe_configs: HashMap::new(), } } @@ -75,11 +86,13 @@ impl InferenceConfig { let cleanup_policy = Self::parse_cleanup_policy(data)?; let query_configs = Self::parse_query_configs(data, cleanup_policy)?; + let rqe_configs = Self::parse_rqe_configs(data, cleanup_policy)?; Ok(Self { schema, query_configs, cleanup_policy, + rqe_configs, }) } @@ -263,43 +276,11 @@ impl InferenceConfig { .map_err(anyhow::Error::from) })?; - let aggregations = if let Some(aggregations_data) = - query_data.get("aggregations").and_then(|v| v.as_sequence()) - { - let mut agg_refs = Vec::new(); - for agg_data in aggregations_data { - let aggregation_id = agg_data - .get("aggregation_id") - .and_then(|v| v.as_u64()) - .ok_or_else(|| { - anyhow::anyhow!("Missing aggregation_id in aggregation") - })?; - - let agg_ref = match cleanup_policy { - CleanupPolicy::CircularBuffer => { - let num_aggregates_to_retain = agg_data - .get("num_aggregates_to_retain") - .and_then(|v| v.as_u64()); - AggregationReference::new(aggregation_id, num_aggregates_to_retain) - } - CleanupPolicy::ReadBased => { - let read_count_threshold = agg_data - .get("read_count_threshold") - .and_then(|v| v.as_u64()); - AggregationReference::with_read_count_threshold( - aggregation_id, - read_count_threshold, - ) - } - CleanupPolicy::NoCleanup => { - AggregationReference::new(aggregation_id, None) - } - }; - agg_refs.push(agg_ref); + let aggregations = match query_data.get("aggregations") { + Some(aggregations) => { + Self::parse_aggregation_refs(aggregations, cleanup_policy)? } - agg_refs - } else { - Vec::new() + None => Vec::new(), }; let config = @@ -316,11 +297,97 @@ impl InferenceConfig { }; Ok(query_configs) } + + /// A sequence of `{aggregation_id, num_aggregates_to_retain | read_count_threshold}`. + fn parse_aggregation_refs( + aggregations_data: &Value, + cleanup_policy: CleanupPolicy, + ) -> Result> { + let Some(aggregations_data) = aggregations_data.as_sequence() else { + return Ok(Vec::new()); + }; + let mut agg_refs = Vec::new(); + for agg_data in aggregations_data { + let aggregation_id = agg_data + .get("aggregation_id") + .and_then(|v| v.as_u64()) + .ok_or_else(|| anyhow::anyhow!("Missing aggregation_id in aggregation"))?; + + let agg_ref = match cleanup_policy { + CleanupPolicy::CircularBuffer => { + let num_aggregates_to_retain = agg_data + .get("num_aggregates_to_retain") + .and_then(|v| v.as_u64()); + AggregationReference::new(aggregation_id, num_aggregates_to_retain) + } + CleanupPolicy::ReadBased => { + let read_count_threshold = agg_data + .get("read_count_threshold") + .and_then(|v| v.as_u64()); + AggregationReference::with_read_count_threshold( + aggregation_id, + read_count_threshold, + ) + } + CleanupPolicy::NoCleanup => AggregationReference::new(aggregation_id, None), + }; + agg_refs.push(agg_ref); + } + Ok(agg_refs) + } + + /// The optional `rqe_configs` mapping: RQE id -> `{aggregations, + /// query_time_aggregations}`, both required. + fn parse_rqe_configs( + data: &Value, + cleanup_policy: CleanupPolicy, + ) -> Result> { + let Some(rqe_configs) = data.get("rqe_configs") else { + return Ok(HashMap::new()); + }; + let rqe_configs = rqe_configs + .as_mapping() + .ok_or_else(|| anyhow::anyhow!("rqe_configs must map RQE ids to configs"))?; + let mut configs = HashMap::new(); + for (id, config) in rqe_configs { + let id = id + .as_str() + .ok_or_else(|| anyhow::anyhow!("rqe_configs key {id:?} is not a string"))?; + let aggregations = config + .get("aggregations") + .ok_or_else(|| anyhow::anyhow!("RQE {id:?}: missing aggregations")) + .and_then(|value| Self::parse_aggregation_refs(value, cleanup_policy))?; + if aggregations.is_empty() { + anyhow::bail!("RQE {id:?}: aggregations is empty"); + } + let query_time_aggregations = config + .get("query_time_aggregations") + .ok_or_else(|| anyhow::anyhow!("RQE {id:?}: missing query_time_aggregations")) + .and_then(|value| { + serde_yaml::from_value::>(value.clone()) + .map_err(anyhow::Error::from) + })?; + for stage in &query_time_aggregations { + stage + .validate() + .map_err(|error| anyhow::anyhow!("RQE {id:?}: {error}"))?; + } + configs.insert( + id.to_string(), + RqeConfig { + aggregations, + query_time_aggregations, + }, + ); + } + Ok(configs) + } } #[cfg(test)] mod tests { use super::*; + use crate::query_config::QueryTimeAggregationOperator; #[test] fn rejects_a_query_without_a_planned_subquery() { @@ -390,4 +457,79 @@ queries: assert!(error.to_string().contains("quantile")); } + + #[test] + fn loads_rqe_configs_with_their_roll_up() { + let data: Value = serde_yaml::from_str( + r#" +cleanup_policy: + name: circular_buffer +metrics: {} +queries: [] +rqe_configs: + sum_by_job: + aggregations: + - aggregation_id: 2 + num_aggregates_to_retain: 5 + query_time_aggregations: + - operator: sum + grouping: + mode: by + labels: [job] + parameter: null +"#, + ) + .unwrap(); + + let config = InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).unwrap(); + + let rqe = &config.rqe_configs["sum_by_job"]; + let [aggregation] = rqe.aggregations.as_slice() else { + panic!("{:?}", rqe.aggregations); + }; + assert_eq!(aggregation.aggregation_id, 2); + assert_eq!(aggregation.num_aggregates_to_retain, Some(5)); + let [stage] = rqe.query_time_aggregations.as_slice() else { + panic!("{:?}", rqe.query_time_aggregations); + }; + assert!(matches!(stage.operator, QueryTimeAggregationOperator::Sum)); + assert_eq!(stage.grouping.labels, ["job"]); + } + + #[test] + fn a_config_without_rqe_configs_has_none() { + let data: Value = + serde_yaml::from_str("cleanup_policy: {name: no_cleanup}\nmetrics: {}\n").unwrap(); + let config = InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).unwrap(); + assert!(config.rqe_configs.is_empty()); + } + + #[test] + fn rejects_an_rqe_config_missing_a_field_or_with_an_invalid_stage() { + for (rqe, reason) in [ + ("{query_time_aggregations: []}", "missing aggregations"), + ( + "{aggregations: [], query_time_aggregations: []}", + "aggregations is empty", + ), + ( + "{aggregations: [{aggregation_id: 1}]}", + "missing query_time_aggregations", + ), + ( + "{aggregations: [{aggregation_id: 1}], query_time_aggregations: \ + [{operator: sum, grouping: {mode: by, labels: []}, parameter: null}]}", + "by grouping must name at least one label", + ), + ] { + let yaml = format!( + "cleanup_policy: {{name: no_cleanup}}\nmetrics: {{}}\nrqe_configs: {{r: {rqe}}}\n" + ); + let data: Value = serde_yaml::from_str(&yaml).unwrap(); + let error = + InferenceConfig::from_yaml_data(&data, QueryLanguage::promql).expect_err("refused"); + assert!(error.to_string().contains(reason), "{error}"); + assert!(error.to_string().contains("\"r\""), "{error}"); + } + } } diff --git a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs index 4ab02498..d8e9e34f 100644 --- a/asap-common/dependencies/rs/asap_types/src/streaming_config.rs +++ b/asap-common/dependencies/rs/asap_types/src/streaming_config.rs @@ -55,8 +55,19 @@ impl StreamingConfig { let mut read_count_threshold_map: HashMap = HashMap::new(); if let Some(inference_config) = inference_config { - for query_config in &inference_config.query_configs { - for aggregation in &query_config.aggregations { + // Every reader of an aggregation: text queries and RQE leaves. + let references = inference_config + .query_configs + .iter() + .map(|q| &q.aggregations) + .chain( + inference_config + .rqe_configs + .values() + .map(|r| &r.aggregations), + ); + for aggregations in references { + for aggregation in aggregations { let aggregation_id = aggregation.aggregation_id; let num_aggregates = aggregation.num_aggregates_to_retain; retention_map @@ -152,7 +163,7 @@ mod tests { use crate::aggregation_config::AggregationConfigError; use crate::aggregation_reference::AggregationReference; use crate::enums::{CleanupPolicy, QueryLanguage}; - use crate::inference_config::InferenceConfig; + use crate::inference_config::{InferenceConfig, RqeConfig}; use crate::query_config::QueryConfig; use promql_utilities::query_logics::enums::AggregationType; @@ -523,4 +534,67 @@ aggregations: .expect_err("an overflowing threshold must not load"); assert!(error.to_string().contains("overflowed"), "{error}"); } + + fn rqe_config(reference: AggregationReference) -> RqeConfig { + RqeConfig { + aggregations: vec![reference], + query_time_aggregations: Vec::new(), + } + } + + #[test] + fn rqe_only_reference_bounds_retention() { + // An aggregation only an RQE leaf reads keeps that leaf's windows, + // not every window ever built. + let yaml = minmax_yaml("MinMax", "max"); + let mut inference = + InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::CircularBuffer); + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::new(1, Some(4))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("an RQE-only aggregation should load"); + + assert_eq!(config[1].num_aggregates_to_retain, Some(4)); + } + + #[test] + fn query_and_rqe_references_keep_the_largest_retention() { + // A shared aggregation keeps the longer lookback, whichever reader has it. + let yaml = minmax_yaml("MinMax", "max"); + for (query_retain, rqe_retain) in [(7, 2), (2, 7)] { + let mut inference = + InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::CircularBuffer); + inference.query_configs = vec![QueryConfig::new("max(metric[1m])".into()) + .add_aggregation(AggregationReference::new(1, Some(query_retain)))]; + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::new(1, Some(rqe_retain))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("a shared aggregation should load"); + + assert_eq!(config[1].num_aggregates_to_retain, Some(7)); + } + } + + #[test] + fn query_and_rqe_references_sum_read_based_thresholds() { + let yaml = minmax_yaml("MinMax", "max"); + let mut inference = InferenceConfig::new(QueryLanguage::promql, CleanupPolicy::ReadBased); + inference.query_configs = vec![QueryConfig::new("max(metric[1m])".into()) + .add_aggregation(AggregationReference::with_read_count_threshold(1, Some(2)))]; + inference.rqe_configs.insert( + "q0".into(), + rqe_config(AggregationReference::with_read_count_threshold(1, Some(3))), + ); + + let config = StreamingConfig::from_yaml_data(&yaml, Some(&inference)) + .expect("a shared aggregation should load"); + + assert_eq!(config[1].read_count_threshold, Some(5)); + } } diff --git a/asap-planner-rs/src/generator.rs b/asap-planner-rs/src/generator.rs index ba045540..e0137e34 100644 --- a/asap-planner-rs/src/generator.rs +++ b/asap-planner-rs/src/generator.rs @@ -1,7 +1,7 @@ use indexmap::IndexMap; use serde_json::Value as JsonValue; use serde_yaml::Value as YamlValue; -use std::collections::HashMap; +use std::collections::{BTreeMap, HashMap}; use asap_types::enums::CleanupPolicy; use asap_types::query_config::QueryTimeAggregation; @@ -31,6 +31,7 @@ pub(crate) const KEY_QUERIES: &str = "queries"; pub(crate) const KEY_QUERY: &str = "query"; pub(crate) const KEY_QUERY_TIME_AGGREGATIONS: &str = "query_time_aggregations"; pub(crate) const KEY_READ_COUNT_THRESHOLD: &str = "read_count_threshold"; +pub(crate) const KEY_RQE_CONFIGS: &str = "rqe_configs"; pub(crate) const KEY_SLIDE_INTERVAL_MS: &str = "slideIntervalMs"; pub(crate) const KEY_SPATIAL_FILTER: &str = "spatialFilter"; pub(crate) const KEY_TABLE_NAME: &str = "table_name"; @@ -49,6 +50,14 @@ pub struct QueryPlanEntry { pub query_time_aggregations: Vec, } +/// How the engine answers one RQE leaf: the aggregations it reads, and the +/// query-time stages that regroup them to the RQE's grouping. +#[derive(Debug, Clone)] +pub struct RqePlanEntry { + pub aggregation_keys: Vec<(String, Option)>, + pub query_time_aggregations: Vec, +} + impl QueryPlanEntry { pub fn fully_planned(query: String, aggregation_keys: Vec<(String, Option)>) -> Self { Self { @@ -144,6 +153,50 @@ pub fn build_aggregation_entry(id: u32, cfg: &IntermediateAggConfig) -> YamlValu YamlValue::Mapping(map) } +/// The `aggregations` list of a query or RQE config: each aggregation's id +/// and, by cleanup policy, how many aggregates it retains or reads. +fn aggregation_refs_yaml( + cleanup_policy: CleanupPolicy, + aggregation_keys: &[(String, Option)], + id_map: &HashMap, +) -> YamlValue { + YamlValue::Sequence( + aggregation_keys + .iter() + .map(|(key, cleanup_param)| { + let agg_id = id_map[key]; + let mut agg_map = serde_yaml::Mapping::new(); + agg_map.insert( + YamlValue::String(KEY_AGGREGATION_ID.to_string()), + YamlValue::Number(agg_id.into()), + ); + if let Some(param) = cleanup_param { + match cleanup_policy { + CleanupPolicy::CircularBuffer => { + agg_map.insert( + YamlValue::String(KEY_NUM_AGG_TO_RETAIN.to_string()), + YamlValue::Number((*param).into()), + ); + } + CleanupPolicy::ReadBased => { + agg_map.insert( + YamlValue::String(KEY_READ_COUNT_THRESHOLD.to_string()), + YamlValue::Number((*param).into()), + ); + } + CleanupPolicy::NoCleanup => {} + } + } + YamlValue::Mapping(agg_map) + }) + .collect(), + ) +} + +fn query_time_aggregations_yaml(stages: &[QueryTimeAggregation]) -> YamlValue { + serde_yaml::to_value(stages).expect("query-time aggregation pipeline should serialize to YAML") +} + pub fn build_queries_yaml( cleanup_policy: CleanupPolicy, query_plan_map: &IndexMap, @@ -152,41 +205,10 @@ pub fn build_queries_yaml( query_plan_map .iter() .map(|(query_str, plan)| { - let aggregations: Vec = plan - .aggregation_keys - .iter() - .map(|(key, cleanup_param)| { - let agg_id = id_map[key]; - let mut agg_map = serde_yaml::Mapping::new(); - agg_map.insert( - YamlValue::String(KEY_AGGREGATION_ID.to_string()), - YamlValue::Number(agg_id.into()), - ); - if let Some(param) = cleanup_param { - match cleanup_policy { - CleanupPolicy::CircularBuffer => { - agg_map.insert( - YamlValue::String(KEY_NUM_AGG_TO_RETAIN.to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::ReadBased => { - agg_map.insert( - YamlValue::String(KEY_READ_COUNT_THRESHOLD.to_string()), - YamlValue::Number((*param).into()), - ); - } - CleanupPolicy::NoCleanup => {} - } - } - YamlValue::Mapping(agg_map) - }) - .collect(); - let mut q_map = serde_yaml::Mapping::new(); q_map.insert( YamlValue::String(KEY_AGGREGATIONS.to_string()), - YamlValue::Sequence(aggregations), + aggregation_refs_yaml(cleanup_policy, &plan.aggregation_keys, id_map), ); q_map.insert( YamlValue::String(KEY_QUERY.to_string()), @@ -198,14 +220,38 @@ pub fn build_queries_yaml( ); q_map.insert( YamlValue::String(KEY_QUERY_TIME_AGGREGATIONS.to_string()), - serde_yaml::to_value(&plan.query_time_aggregations) - .expect("query-time aggregation pipeline should serialize to YAML"), + query_time_aggregations_yaml(&plan.query_time_aggregations), ); YamlValue::Mapping(q_map) }) .collect() } +/// The `rqe_configs` mapping: RQE id -> its aggregations and query-time stages. +pub fn build_rqe_configs_yaml( + cleanup_policy: CleanupPolicy, + rqe_plans: &BTreeMap, + id_map: &HashMap, +) -> YamlValue { + YamlValue::Mapping( + rqe_plans + .iter() + .map(|(id, plan)| { + let mut rqe_map = serde_yaml::Mapping::new(); + rqe_map.insert( + YamlValue::String(KEY_AGGREGATIONS.to_string()), + aggregation_refs_yaml(cleanup_policy, &plan.aggregation_keys, id_map), + ); + rqe_map.insert( + YamlValue::String(KEY_QUERY_TIME_AGGREGATIONS.to_string()), + query_time_aggregations_yaml(&plan.query_time_aggregations), + ); + (YamlValue::String(id.clone()), YamlValue::Mapping(rqe_map)) + }) + .collect(), + ) +} + pub fn params_to_yaml(params: &HashMap) -> YamlValue { if params.is_empty() { return YamlValue::Mapping(serde_yaml::Mapping::new()); diff --git a/asap-planner-rs/src/main.rs b/asap-planner-rs/src/main.rs index 32dda2b6..ae063089 100644 --- a/asap-planner-rs/src/main.rs +++ b/asap-planner-rs/src/main.rs @@ -87,6 +87,13 @@ struct Args { #[arg(long = "sla-ms")] sla_ms: Option, + /// Saved only. Plan only the tree leaves of the e2e queries that have an + /// RQE tree (`expr`): no query text is planned, so no `query_configs` are + /// written, only deployments and `rqe_configs`. Queries without a tree + /// (e.g. `rate`) are skipped. + #[arg(long = "expr-only")] + expr_only: bool, + /// MILP only. YAML workload facts: per metric, positive `value_range` and /// `cardinality` per label set, including all labels (the series count), /// plus the `shape` of each grouping sketches may serve. @@ -155,8 +162,10 @@ fn main() -> anyhow::Result<()> { && args.workload_table.is_none() && args.method.is_none() && args.weights.is_none() - && args.sla_ms.is_none(), - "--plan, --workload-table, --method, --weights and --sla-ms require --planner saved" + && args.sla_ms.is_none() + && !args.expr_only, + "--plan, --workload-table, --method, --weights, --sla-ms and --expr-only require \ + --planner saved" ); } if args.planner != PlannerArg::Milp { @@ -358,6 +367,7 @@ fn run_saved(args: &Args) -> anyhow::Result<()> { }, args.query_language, scrape_interval_ms, + args.expr_only, )? .write_to_dir(&args.output_dir)?; Ok(()) diff --git a/asap-planner-rs/src/optimizer/milp.rs b/asap-planner-rs/src/optimizer/milp.rs index 9d3ae6cb..3fd99894 100644 --- a/asap-planner-rs/src/optimizer/milp.rs +++ b/asap-planner-rs/src/optimizer/milp.rs @@ -327,6 +327,19 @@ pub struct MilpWorkload { pub raqe_items: Vec, } +impl MilpWorkload { + /// Appends `item` as a single Raqe named `id`, returning the Raqe's index. + pub(crate) fn push_item(&mut self, item: OptimizerItem, id: &str) -> Result { + self.raqes.push(Raqe { + id: id.to_string(), + ..item_to_raqe(&item)? + }); + self.raqe_items.push(self.items.len()); + self.items.push(item); + Ok(self.raqes.len() - 1) + } +} + /// One Raqe per query occurrence: a `Raqe` has no frequency weight, so two /// dashboards running the same query at the same cadence become two /// identical Raqes, which the MILP serves from one shared deployment. diff --git a/asap-planner-rs/src/optimizer/milp_output.rs b/asap-planner-rs/src/optimizer/milp_output.rs index 49de8c19..6e2c9640 100644 --- a/asap-planner-rs/src/optimizer/milp_output.rs +++ b/asap-planner-rs/src/optimizer/milp_output.rs @@ -1,9 +1,11 @@ //! Turns a solved MILP plan into the planner's streaming and inference YAML. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use asap_types::enums::{CleanupPolicy, WindowType}; -use asap_types::query_config::QueryTimeAggregationOperator; +use asap_types::query_config::{ + QueryTimeAggregation, QueryTimeAggregationOperator, QueryTimeGrouping, QueryTimeGroupingMode, +}; use indexmap::map::Entry; use indexmap::IndexMap; use promql_parser::label::Labels; @@ -16,7 +18,9 @@ use serde_json::Value; use thiserror::Error; use crate::error::ControllerError; -use crate::generator::{GeneratorOutput, QueryPlanEntry}; +use crate::generator::{ + build_rqe_configs_yaml, GeneratorOutput, QueryPlanEntry, RqePlanEntry, KEY_RQE_CONFIGS, +}; use crate::planner::agg_config::IntermediateAggConfig; use crate::planner::labels::set_subpopulation_labels; use crate::planner::promql::{aggregation_stage, parse_binary_arms, strip_parens}; @@ -146,16 +150,19 @@ pub fn plan_to_planner_output( .map(|planned| planned.deployment.clone()) .collect(); let served_by: Vec = solution.raqes.iter().map(|r| r.deployment).collect(); - deployments_to_planner_output(config, workload, &deployments, &served_by) + deployments_to_planner_output(config, workload, &deployments, &served_by, &BTreeMap::new()) } /// [`plan_to_planner_output`] for a plan given as its deployments and, per -/// Raqe of `workload`, the index of the deployment serving it. +/// Raqe of `workload`, the index of the deployment serving it. Each +/// `rqe_raqes` entry (RQE id -> index of the Raqe that is that RQE leaf) gets +/// an `rqe_configs` entry, written only when there are any. pub fn deployments_to_planner_output( config: MilpConfig, workload: &MilpWorkload, deployments: &[Deployment], served_by: &[usize], + rqe_raqes: &BTreeMap, ) -> Result { assert_eq!( served_by.len(), @@ -232,12 +239,27 @@ pub fn deployments_to_planner_output( } plan_per_series_avgs(config, &binary_leaves, &mut queries)?; + let mut rqe_plans: BTreeMap = BTreeMap::new(); + for (id, &raqe) in rqe_raqes { + let index = served_by[raqe]; + let key = &item_keys[&(index, workload.raqe_items[raqe])]; + rqe_plans.insert( + id.clone(), + rqe_plan_entry( + key, + &workload.raqes[raqe], + &deployments[index], + aggregations[key].aggregation_type, + )?, + ); + } + let id_map: HashMap = aggregations .keys() .enumerate() .map(|(index, key)| (key.clone(), index as u32 + 1)) .collect(); - let (streaming_yaml, inference_yaml) = match config { + let (streaming_yaml, mut inference_yaml) = match config { MilpConfig::Promql(config) => { let schema = config.schema_from_hints(); ( @@ -250,6 +272,15 @@ pub fn deployments_to_planner_output( build_sql_inference_yaml(config, CleanupPolicy::CircularBuffer, &queries, &id_map)?, ), }; + if !rqe_plans.is_empty() { + let serde_yaml::Value::Mapping(root) = &mut inference_yaml else { + unreachable!("the inference YAML is a mapping"); + }; + root.insert( + KEY_RQE_CONFIGS.into(), + build_rqe_configs_yaml(CleanupPolicy::CircularBuffer, &rqe_plans, &id_map), + ); + } Ok(PlannerOutput::from_output(GeneratorOutput { punted_queries: Vec::new(), streaming_yaml, @@ -276,61 +307,96 @@ fn check_grouping(raqe: &Raqe, deployment: &Deployment) -> Result<(), MilpOutput }) } -/// How `query` (an occurrence of `raqe`) reads the aggregation `key`. -/// -/// A Raqe grouped by a strict subset of its deployment's grouping is a -/// roll-up. The engine merges single-subpopulation sketches across the -/// deployment's groups itself, so those name the aggregation as is. Exact -/// keyed accumulators are read at the deployment's grouping and regrouped -/// at query time, which is exact for sum, count, min and max, unless -/// `stage_blocker` says `query` can't run a query-time stage. -fn query_plan_entry( +/// The closed windows of `deployment` that `raqe`'s lookback reads. The +/// store holds closed windows only; open ones live in the precompute engine. +fn retained_windows( query: &str, - key: &str, raqe: &Raqe, deployment: &Deployment, - aggregation_type: AggregationType, - stage_blocker: Option, -) -> Result { - // The store holds closed windows only; open ones live in the precompute - // engine. - let retained = deployment +) -> Result { + deployment .closed_instance_count(raqe.lookback_ms) .ok_or_else(|| MilpOutputError::PartialWindowLookback { query: query.to_string(), lookback_ms: raqe.lookback_ms, window_ms: deployment.window_ms, slide_ms: deployment.slide_ms, - })?; - let aggregation_keys = vec![(key.to_string(), Some(retained))]; + }) +} + +/// Whether `raqe` reads `deployment` through a query-time stage. +/// +/// A Raqe grouped by a strict subset of its deployment's grouping is a +/// roll-up. The engine merges single-subpopulation sketches across the +/// deployment's groups itself, so those need no stage. Exact keyed +/// accumulators are read at the deployment's grouping and regrouped at query +/// time, which is exact for sum, count, min and max. +fn needs_query_time_stage( + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, +) -> Result { let (fine, coarse) = (&deployment.grouping_labels, &raqe.grouping_labels); - if coarse == fine { - return Ok(QueryPlanEntry::fully_planned( - query.to_string(), - aggregation_keys, - )); + if coarse == fine || aggregation_type.merges_across_groups() { + return Ok(false); } - if aggregation_type.merges_across_groups() { - return Ok(QueryPlanEntry::fully_planned( - query.to_string(), - aggregation_keys, - )); + if matches!( + aggregation_type, + AggregationType::MultipleSum | AggregationType::MultipleMinMax + ) { + return Ok(true); } - let unsupported = |reason| MilpOutputError::RollupUnsupported { + Err(rollup_unsupported( + raqe, + deployment, + aggregation_type, + RollupBlocker::Aggregation, + )) +} + +fn rollup_unsupported( + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, + reason: RollupBlocker, +) -> MilpOutputError { + MilpOutputError::RollupUnsupported { query: raqe.id.clone(), aggregation_type, - raqe: label_names(coarse), - deployment: label_names(fine), + raqe: label_names(&raqe.grouping_labels), + deployment: label_names(&deployment.grouping_labels), reason, - }; - if !matches!( - aggregation_type, - AggregationType::MultipleSum | AggregationType::MultipleMinMax - ) { - return Err(unsupported(RollupBlocker::Aggregation)); + } +} + +/// How `query` (an occurrence of `raqe`) reads the aggregation `key`: the +/// query-time stage of a roll-up regroups `query`'s own outer aggregation, +/// unless `stage_blocker` says `query` can't run one. +fn query_plan_entry( + query: &str, + key: &str, + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, + stage_blocker: Option, +) -> Result { + let aggregation_keys = vec![( + key.to_string(), + Some(retained_windows(query, raqe, deployment)?), + )]; + if !needs_query_time_stage(raqe, deployment, aggregation_type)? { + return Ok(QueryPlanEntry::fully_planned( + query.to_string(), + aggregation_keys, + )); } if let Some(reason) = stage_blocker { - return Err(unsupported(reason)); + return Err(rollup_unsupported( + raqe, + deployment, + aggregation_type, + reason, + )); } let unregroupable = || MilpOutputError::UnregroupableQuery(query.to_string()); let aggregate = outer_aggregate(query).ok_or_else(unregroupable)?; @@ -341,11 +407,58 @@ fn query_plan_entry( } Ok(QueryPlanEntry { aggregation_keys, - planned_subquery: regrouped(aggregate, fine), + planned_subquery: regrouped(aggregate, &deployment.grouping_labels), query_time_aggregations: vec![stage], }) } +/// How the engine answers the RQE leaf `raqe` from the aggregation `key`. A +/// roll-up's query-time stage regroups by the RQE's own grouping. +fn rqe_plan_entry( + key: &str, + raqe: &Raqe, + deployment: &Deployment, + aggregation_type: AggregationType, +) -> Result { + let aggregation_keys = vec![( + key.to_string(), + Some(retained_windows(&raqe.id, raqe, deployment)?), + )]; + let query_time_aggregations = if needs_query_time_stage(raqe, deployment, aggregation_type)? { + vec![rqe_roll_up_stage(raqe)] + } else { + Vec::new() + }; + Ok(RqePlanEntry { + aggregation_keys, + query_time_aggregations, + }) +} + +/// The stage that rolls an exact keyed accumulator up to `raqe`'s grouping. +fn rqe_roll_up_stage(raqe: &Raqe) -> QueryTimeAggregation { + let operator = match raqe.capability { + // A coarse count is the sum of the fine counts. + Capability::Sum | Capability::Count => QueryTimeAggregationOperator::Sum, + Capability::Min => QueryTimeAggregationOperator::Min, + Capability::Max => QueryTimeAggregationOperator::Max, + other => { + unreachable!("only exact sum, count, min and max roll up at query time, not {other:?}") + } + }; + let labels = label_names(&raqe.grouping_labels); + let mode = if labels.is_empty() { + QueryTimeGroupingMode::All + } else { + QueryTimeGroupingMode::By + }; + QueryTimeAggregation { + operator, + grouping: QueryTimeGrouping { mode, labels }, + parameter: None, + } +} + /// Plans each per-series avg query (`per_series_avg`) as its leaf's entry /// plus a query-time avg, replacing the leaf's entry unless the leaf is /// itself a query or a binary query's arm. @@ -944,9 +1057,14 @@ mod tests { deployment.config.sketch = "hll".into(); deployment.config.sketch_config["params"] = json!({"lg_k": 12}); - let output = - deployments_to_planner_output(MilpConfig::Sql(&sql), &workload, &[deployment], &[0]) - .unwrap(); + let output = deployments_to_planner_output( + MilpConfig::Sql(&sql), + &workload, + &[deployment], + &[0], + &BTreeMap::new(), + ) + .unwrap(); let a = aggregation_in(&output, query, QueryLanguage::sql); assert_eq!(a.aggregation_type, AggregationType::HLL); assert_eq!(a.grouping_labels, instance_job()); diff --git a/asap-planner-rs/src/optimizer/saved_plan.rs b/asap-planner-rs/src/optimizer/saved_plan.rs index 13588e62..274756b3 100644 --- a/asap-planner-rs/src/optimizer/saved_plan.rs +++ b/asap-planner-rs/src/optimizer/saved_plan.rs @@ -1,14 +1,21 @@ //! Writes a plan solved elsewhere, sketch-bench `autosketch_vs_asap`'s plan //! file, as the planner's streaming and inference YAML, without re-solving. //! The workload comes from the eval table's `e2e` block: its PromQL or SQL -//! queries, the plan RQEs each reads, and the metrics' labels. +//! queries, the plan RQEs each reads, and the metrics' labels. A query's RQE +//! expression tree (`expr`) also gets each of its leaves an `rqe_configs` +//! entry, so the engine can answer the tree without parsing PromQL or SQL. -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::io::Read; use std::path::Path; use anyhow::{bail, ensure, Context}; use asap_types::enums::QueryLanguage; +use asap_types::query_requirements::QueryRequirements; +use asap_types::rqe_expr::{RqeCapability, RqeExpr, RqeLeaf}; +use asap_types::utils::normalize_spatial_filter; +use promql_utilities::data_model::KeyByLabelNames; +use promql_utilities::query_logics::enums::Statistic; use rqe_optimizer::{heap_capacity, AtomicCostEntry, Deployment, LabelSet, Raqe}; use serde::Deserialize; use serde_json::{json, Value}; @@ -18,9 +25,10 @@ use crate::config::input::{ }; use crate::planner_output::PlannerOutput; -use super::aqe_extractor::decompose_to_leaves; +use super::aqe_extractor::{decompose_to_leaves, sql_metric}; use super::milp::{workload_from_config, MilpConfig}; use super::milp_output::{deployments_to_planner_output, reject_unwritable_queries}; +use super::solution::OptimizerItem; #[derive(Deserialize)] struct Table { @@ -44,28 +52,128 @@ const SQL_TIME_COLUMN: &str = "ts"; /// The metric type whose metrics are SQL tables. const RECORDS_METRIC_TYPE: &str = "records"; -/// One query, written in exactly one of PromQL and SQL. +/// One query: written in at most one of PromQL and SQL, and with an RQE +/// expression tree, `expr`, or not. It needs text or a tree. #[derive(Deserialize)] struct E2eQuery { promql: Option, sql: Option, interval_secs: u64, /// Plan RQE ids: for `promql`, one per leaf in order; for `sql`, a single - /// leaf, every RQE it reads. + /// leaf, every RQE it reads. Required with text; a query with only `expr` + /// may omit them, and then reads its leaves' ids. + #[serde(default)] rqes: Vec, + /// Leaves name plan RQE ids. + expr: Option, } impl E2eQuery { - /// The query if it is written in SQL (`sql`) or PromQL (otherwise). - fn text(&self, sql: bool) -> anyhow::Result> { + /// How errors name the query: its text, else its tree's RQEs. + fn name(&self) -> String { + match (self.promql.as_ref().or(self.sql.as_ref()), &self.expr) { + (Some(text), _) => format!("{text:?}"), + (None, Some(expr)) => format!("with RQEs {:?}", leaf_ids(expr)), + (None, None) if self.rqes.is_empty() => "with no text, `expr` or RQEs".to_string(), + (None, None) => format!("reading {:?}", self.rqes), + } + } + + /// The query's PromQL or SQL text, if any. + fn text(&self) -> anyhow::Result> { match (&self.promql, &self.sql) { - (Some(_), Some(_)) | (None, None) => bail!( - "e2e query reading {:?} needs exactly one of `promql` and `sql`", - self.rqes + (Some(_), Some(_)) => bail!( + "e2e query {} needs exactly one of `promql` and `sql`", + self.name() + ), + (promql, sql) => Ok(promql.as_deref().or(sql.as_deref())), + } + } + + /// Whether the query is SQL: written in `sql`, or, with no text, a tree + /// whose leaves read `records` metrics (SQL tables). Its tree, if any, + /// must be valid. + fn is_sql(&self, metrics: &BTreeMap) -> anyhow::Result { + let text_sql = self.text()?.map(|_| self.sql.is_some()); + let tree_sql = match &self.expr { + None => None, + Some(expr) => { + let reads_records = |leaf: &RqeLeaf| { + anyhow::Ok(leaf_metric(leaf, metrics)?.kind == RECORDS_METRIC_TYPE) + }; + let leaves = expr.leaves(); + let (first, rest) = leaves.split_first().expect("a valid tree reads a leaf"); + let sql = reads_records(first)?; + for leaf in rest { + ensure!( + reads_records(leaf)? == sql, + "e2e query {}: its `expr` mixes records and non-records metrics", + self.name() + ); + } + Some(sql) + } + }; + match (text_sql, tree_sql) { + (Some(text), Some(tree)) if text != tree => bail!( + "e2e query {} is written in {} but its `expr` reads {} metrics", + self.name(), + if text { "SQL" } else { "PromQL" }, + if tree { "records" } else { "non-records" } ), - (promql, sql_text) => Ok(if sql { sql_text } else { promql }.as_deref()), + (Some(sql), _) | (None, Some(sql)) => Ok(sql), + (None, None) => bail!("e2e query {} needs `promql`, `sql` or `expr`", self.name()), } } + + /// The plan RQEs the query reads: `rqes`, or, for a query with only a + /// tree that omits them, its leaves. With both, they must name the same + /// RQEs, in any order and however often each appears. + fn rqe_ids(&self) -> anyhow::Result> { + let rqes: Vec<&str> = self.rqes.iter().map(String::as_str).collect(); + // A text query's `rqes` map its text leaves to plan RQEs, which a + // tree's leaf order can't. + ensure!( + self.text()?.is_none() || !rqes.is_empty(), + "e2e query {} has text but no `rqes`", + self.name() + ); + let Some(expr) = &self.expr else { + return Ok(rqes); + }; + let leaves = leaf_ids(expr); + if rqes.is_empty() { + return Ok(leaves); + } + ensure!( + rqes.iter().collect::>() == leaves.iter().collect(), + "e2e query {} names RQEs {rqes:?} but its `expr` reads {leaves:?}", + self.name() + ); + Ok(rqes) + } +} + +/// The ids of a tree's leaves, left to right. +fn leaf_ids(expr: &RqeExpr) -> Vec<&str> { + expr.leaves().iter().map(|l| l.id.as_str()).collect() +} + +/// The table metric an RQE leaf reads. +fn leaf_metric<'a>( + leaf: &RqeLeaf, + metrics: &'a BTreeMap, +) -> anyhow::Result<&'a E2eMetric> { + metrics + .get(&leaf.metric) + .with_context(|| format!("RQE {:?} reads unknown metric {:?}", leaf.id, leaf.metric)) +} + +/// A query planned from its text, and the plan RQEs it reads. +struct TextQuery<'a> { + query: &'a E2eQuery, + text: &'a str, + rqes: Vec<&'a str>, } #[derive(Deserialize)] @@ -122,13 +230,13 @@ pub struct PlanSelector<'a> { /// The selected queries' texts by repeat interval in seconds: one query group /// each. -fn by_interval(selected: &[(&E2eQuery, &str)]) -> BTreeMap> { +fn by_interval(selected: &[TextQuery]) -> BTreeMap> { let mut groups: BTreeMap> = BTreeMap::new(); - for (q, text) in selected { + for q in selected { groups - .entry(q.interval_secs) + .entry(q.query.interval_secs) .or_default() - .push(text.to_string()); + .push(q.text.to_string()); } groups } @@ -136,7 +244,7 @@ fn by_interval(selected: &[(&E2eQuery, &str)]) -> BTreeMap> { /// The PromQL controller config of the selected queries, with every /// non-`records` metric's labels as hints. fn e2e_promql_config( - selected: &[(&E2eQuery, &str)], + selected: &[TextQuery], metrics: &BTreeMap, ) -> anyhow::Result { let config = json!({ @@ -157,7 +265,7 @@ fn e2e_promql_config( /// is a table, its labels the metadata columns and its values the value /// columns. fn e2e_sql_config( - selected: &[(&E2eQuery, &str)], + selected: &[TextQuery], metrics: &BTreeMap, ) -> anyhow::Result { let mut tables = Vec::new(); @@ -207,13 +315,15 @@ fn read_json(path: &Path) -> anyhow::Result { /// Streaming and inference YAML for the selected plan of `plan_path` (JSON, /// gzipped or not) over the workload of `table_path`: its queries written in -/// `language`. +/// `language`. With `expr_only`, only the queries with an RQE tree, planned +/// from their tree leaves alone: no text items and no `query_configs`. pub fn saved_plan_output( table_path: &Path, plan_path: &Path, selector: &PlanSelector, language: QueryLanguage, scrape_interval_ms: u64, + expr_only: bool, ) -> anyhow::Result { let sql = match language { QueryLanguage::sql => true, @@ -263,16 +373,38 @@ pub fn saved_plan_output( c.rqe ); } - let mut selected: Vec<(&E2eQuery, &str)> = Vec::new(); + // Queries in `language`: those with text (unless `expr_only`), and those + // with a tree. + let mut selected: Vec = Vec::new(); + let mut trees: Vec<(&E2eQuery, &RqeExpr)> = Vec::new(); for q in &e2e.queries { - if let Some(text) = q.text(sql)? { - selected.push((q, text)); + if expr_only && q.expr.is_none() { + continue; + } + if let Some(expr) = &q.expr { + expr.validate() + .with_context(|| format!("e2e query {}: invalid `expr`", q.name()))?; + } + if q.is_sql(&e2e.metrics)? != sql { + continue; + } + let rqes = q.rqe_ids()?; + if let (Some(text), false) = (q.text()?, expr_only) { + selected.push(TextQuery { + query: q, + text, + rqes, + }); + } + if let Some(expr) = &q.expr { + trees.push((q, expr)); } } ensure!( - !selected.is_empty(), - "{}: no e2e query is written in {language:?}", - table_path.display() + !selected.is_empty() || !trees.is_empty(), + "{}: no e2e query{} is written in {language:?}", + table_path.display(), + if expr_only { " with an `expr`" } else { "" } ); let (promql_config, sql_config); let config = if sql { @@ -289,25 +421,32 @@ pub fn saved_plan_output( // Leaf query string and repeat interval -> the plan RQEs it reads. let mut rqes_of: HashMap<(String, u64), Vec<&str>> = HashMap::new(); - for &(q, text) in &selected { - let rqes = q.rqes.iter().map(String::as_str); + for TextQuery { + query: q, + text, + rqes: rqe_ids, + } in &selected + { + let rqe_ids = rqe_ids.clone(); let leaves: Vec<(String, Vec<&str>)> = match config { // A SQL query is one leaf, and its RQEs a set. MilpConfig::Sql(_) => { - ensure!(!q.rqes.is_empty(), "query {text:?} names no RQEs"); - let mut rqes: Vec<&str> = rqes.collect(); + let mut rqes = rqe_ids; rqes.sort_unstable(); vec![(text.to_string(), rqes)] } MilpConfig::Promql(_) => { let leaves = decompose_to_leaves(text); ensure!( - leaves.len() == q.rqes.len(), + leaves.len() == rqe_ids.len(), "query {text:?} has {} leaves but names {} RQEs", leaves.len(), - q.rqes.len() + rqe_ids.len() ); - leaves.into_iter().zip(rqes.map(|r| vec![r])).collect() + leaves + .into_iter() + .zip(rqe_ids.into_iter().map(|r| vec![r])) + .collect() } }; for (leaf, rqes) in leaves { @@ -323,27 +462,65 @@ pub fn saved_plan_output( } } - let workload = workload_from_config(config, scrape_interval_ms)?; + // Every tree leaf, by RQE id, and the repeat intervals of its queries. + let mut tree_leaves: BTreeMap<&str, (&RqeLeaf, BTreeSet)> = BTreeMap::new(); + for (q, expr) in &trees { + for leaf in expr.leaves() { + let (other, intervals_ms) = tree_leaves + .entry(&leaf.id) + .or_insert_with(|| (leaf, BTreeSet::new())); + ensure!( + *other == leaf, + "RQE {:?} is two different leaves: {other:?} and {leaf:?}", + leaf.id + ); + intervals_ms.insert(q.interval_secs * 1000); + } + } + + let mut workload = workload_from_config(config, scrape_interval_ms)?; + let text_items = workload.items.len(); + // Each tree leaf at each repeat interval is one more item and Raqe, + // read directly by its RQE id; `leaf_rqes[i]` is item `text_items + i`'s. + let mut rqe_raqes: BTreeMap = BTreeMap::new(); + let mut leaf_rqes: Vec<&str> = Vec::new(); + for (&id, (leaf, intervals_ms)) in &tree_leaves { + for &interval_ms in intervals_ms { + let item = leaf_item(leaf, interval_ms, &e2e.metrics)?; + let raqe = workload.push_item(item, id)?; + // One pick serves every interval, and the interval doesn't reach + // the `rqe_configs` entry: any interval's Raqe writes it. + rqe_raqes.entry(id.to_string()).or_insert(raqe); + leaf_rqes.push(id); + } + } // Plan deployment index -> its position in `deployments`, in order of // the first Raqe each serves. let mut position: HashMap = HashMap::new(); let mut deployments: Vec = Vec::new(); let mut served_by = Vec::with_capacity(workload.raqes.len()); - for (raqe, &item) in workload.raqes.iter().zip(&workload.raqe_items) { - let item = &workload.items[item]; + for (raqe, &item_index) in workload.raqes.iter().zip(&workload.raqe_items) { + let item = &workload.items[item_index]; // An item's leaves (say p90 and p50 over one window), and every RQE // a SQL leaf reads, read one deployment, so their picks must name the - // same sketch. - let mut index = None; - let mut reads: Vec<(&String, &str)> = Vec::new(); - for leaf in &item.query_strings { - let rqes = rqes_of - .get(&(leaf.clone(), item.t_repeat_ms)) - .with_context(|| format!("leaf {leaf:?} has no RQE in the table's e2e block"))?; - reads.extend(rqes.iter().map(|&rqe| (leaf, rqe))); + // same sketch. Each (reader, RQE): a leaf query, or a tree leaf's id. + let mut reads: Vec<(&str, &str)> = Vec::new(); + if item_index < text_items { + for leaf in &item.query_strings { + let rqes = rqes_of + .get(&(leaf.clone(), item.t_repeat_ms)) + .with_context(|| { + format!("leaf {leaf:?} has no RQE in the table's e2e block") + })?; + reads.extend(rqes.iter().map(|&rqe| (leaf.as_str(), rqe))); + } + } else { + let id = leaf_rqes[item_index - text_items]; + reads.push((id, id)); } - for (leaf, rqe) in reads { + let mut index = None; + for &(reader, rqe) in &reads { let pick = chosen .get(rqe) .with_context(|| format!("the plan doesn't serve RQE {rqe:?}"))?; @@ -364,7 +541,7 @@ pub fn saved_plan_output( }; ensure!( can_serve(raqe, &deployment), - "RQE {rqe:?}'s {} {}s window sliding every {}s can't serve {leaf:?} \ + "RQE {rqe:?}'s {} {}s window sliding every {}s can't serve {reader:?} \ ({} ms lookback every {} ms); does --plan match --workload-table?", pick.sketch, pick.window_secs, @@ -386,11 +563,11 @@ pub fn saved_plan_output( ensure!( index.is_none_or(|index| index == at), "query {:?} is served by two deployments", - item.query_strings + reads.iter().map(|&(reader, _)| reader).collect::>() ); index = Some(at); } - let index = index.expect("an item has a query string"); + let index = index.expect("an item reads an RQE"); served_by.push(index); } Ok(deployments_to_planner_output( @@ -398,9 +575,78 @@ pub fn saved_plan_output( &workload, &deployments, &served_by, + &rqe_raqes, )?) } +/// The optimizer item of an RQE leaf, read every `interval_ms`: what a +/// PromQL or SQL leaf with the same capability, metric, lookback, grouping +/// and top-k buckets needs. It has no query string, so it gets no `query_configs` entry. +fn leaf_item( + leaf: &RqeLeaf, + interval_ms: u64, + metrics: &BTreeMap, +) -> anyhow::Result { + let id = &leaf.id; + let m = leaf_metric(leaf, metrics)?; + let metric = match (m.kind == RECORDS_METRIC_TYPE, &leaf.value) { + (true, Some(value)) => { + ensure!( + m.values.contains_key(value), + "RQE {id:?} reads {value:?}, which is not a value of {:?}", + leaf.metric + ); + sql_metric(&leaf.metric, value) + } + (true, None) => bail!( + "RQE {id:?} reads records metric {:?} without a `value`", + leaf.metric + ), + (false, Some(_)) => bail!( + "RQE {id:?} has a `value`, but {:?} is not a records metric", + leaf.metric + ), + (false, None) => leaf.metric.clone(), + }; + for label in &leaf.grouping { + ensure!( + m.labels.iter().any(|l| &l.name == label), + "RQE {id:?} groups by {label:?}, which is not a label of {:?}", + leaf.metric + ); + } + ensure!(interval_ms > 0, "RQE {id:?} repeats every 0 s"); + let (statistic, topk_count_events) = match leaf.capability { + RqeCapability::Sum => (Statistic::Sum, None), + RqeCapability::Quantile => (Statistic::Quantile, None), + RqeCapability::Cardinality => (Statistic::Cardinality, None), + // Value-ranked, as `topk(k, sum_over_time(...))` is. + RqeCapability::TopkByValue => (Statistic::Topk, Some(false)), + }; + // A text query of a saved plan's table has the default options too. + let options = ControllerOptions::default(); + Ok(OptimizerItem { + requirements: QueryRequirements { + metric, + statistics: vec![statistic], + data_range_ms: leaf.lookback_ms, + grouping_labels: KeyByLabelNames::new(leaf.grouping.clone()), + spatial_filter_normalized: normalize_spatial_filter(""), + topk_count_events, + // As a PromQL `topk by`: the Raqe is grouped by these buckets, + // as the plan's top-k RQE is. + topk_by_labels: leaf.topk_by.clone().map(KeyByLabelNames::new), + }, + query_strings: Vec::new(), + query_frequency_hz: 1000.0 / interval_ms as f64, + occurrences: 1, + t_repeat_ms: interval_ms, + accuracy_sla: options.accuracy_sla, + latency_sla_ms: options.latency_sla_ms, + topk_k: leaf.k, + }) +} + /// Largest gap between `secs` in ms and the nearest whole ms that is still /// float error, not a fraction of a ms. const WHOLE_MS_TOLERANCE: f64 = 1e-6; @@ -508,6 +754,16 @@ mod tests { plan: &[u8], sla_ms: Option, language: QueryLanguage, + ) -> anyhow::Result { + run_filtered(table, plan, sla_ms, language, false) + } + + fn run_filtered( + table: &str, + plan: &[u8], + sla_ms: Option, + language: QueryLanguage, + expr_only: bool, ) -> anyhow::Result { let dir = tempfile::tempdir().unwrap(); let (table_path, plan_path) = (dir.path().join("table.json"), dir.path().join("plan.json")); @@ -518,7 +774,14 @@ mod tests { weights: "cpu", sla_ms, }; - saved_plan_output(&table_path, &plan_path, &selector, language, 1000) + saved_plan_output( + &table_path, + &plan_path, + &selector, + language, + 1000, + expr_only, + ) } fn refusal(plan: Value, sla_ms: Option) -> String { @@ -981,7 +1244,12 @@ mod tests { .err() .expect("refused") .to_string(); - assert!(err.contains("exactly one of `promql` and `sql`"), "{err}"); + assert!( + err.contains( + r#"e2e query "sum by (job) (data)" needs exactly one of `promql` and `sql`"# + ), + "{err}" + ); } #[test] @@ -991,4 +1259,612 @@ mod tests { let err = refusal(plan, None); assert!(err.contains(r#"doesn't serve RQE "p50""#), "{err}"); } + + fn leaf(id: &str, capability: &str, lookback_ms: u64, grouping: &[&str]) -> Value { + json!({"rqe": {"id": id, "metric": "data", "capability": capability, + "lookback_ms": lookback_ms, "grouping": grouping}}) + } + + fn quantile_leaf(id: &str, phi: f64, lookback_ms: u64, grouping: &[&str]) -> Value { + let mut l = leaf(id, "quantile", lookback_ms, grouping); + l["rqe"]["phi"] = json!(phi); + l + } + + /// `TABLE` with each query's RQE tree. + fn tree_table() -> Value { + let mut table: Value = serde_json::from_str(TABLE).unwrap(); + let series = ["job", "label_0"]; + let p90 = quantile_leaf("p90", 0.9, 300_000, &series); + let p50 = quantile_leaf("p50", 0.5, 300_000, &series); + let exprs = [ + leaf("sum", "sum", 1000, &["job"]), + json!({"op": "/", "lhs": p90, "rhs": p50.clone()}), + p50, + ]; + for (q, expr) in queries(&mut table).iter_mut().zip(exprs) { + q["expr"] = expr; + } + table + } + + fn queries(table: &mut Value) -> &mut Vec { + table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + } + + fn run_table( + table: &Value, + plan: &Value, + language: QueryLanguage, + ) -> anyhow::Result { + run_in( + &table.to_string(), + plan.to_string().as_bytes(), + None, + language, + ) + } + + fn inference_of( + output: &PlannerOutput, + language: QueryLanguage, + ) -> asap_types::InferenceConfig { + output.to_inference_config(language).unwrap() + } + + fn query_aggregation_ids(inference: &asap_types::InferenceConfig, query: &str) -> Vec { + let config = inference + .query_configs + .iter() + .find(|q| q.query == query) + .unwrap_or_else(|| panic!("no query_config for {query:?}")); + config + .aggregations + .iter() + .map(|a| a.aggregation_id) + .collect() + } + + /// (aggregation id, retained windows) per aggregation the RQE reads. + fn rqe_aggregations( + inference: &asap_types::InferenceConfig, + rqe: &str, + ) -> Vec<(u64, Option)> { + inference.rqe_configs[rqe] + .aggregations + .iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect() + } + + fn stages(inference: &asap_types::InferenceConfig, rqe: &str) -> Value { + serde_json::to_value(&inference.rqe_configs[rqe].query_time_aggregations).unwrap() + } + + /// Each tree leaf reads the aggregation its text query reads. The + /// ratio's two leaves are two entries on one aggregation. + #[test] + fn writes_an_rqe_config_per_tree_leaf() { + let output = run_table(&tree_table(), &plans(shared_p50()), QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["p50", "p90", "sum"]); + let [sum_id] = query_aggregation_ids(&inference, "sum by (job) (data)")[..] else { + panic!("one aggregation"); + }; + assert_eq!(rqe_aggregations(&inference, "sum"), [(sum_id, Some(1))]); + let [dd_id] = query_aggregation_ids(&inference, "quantile_over_time(0.5, data[5m])")[..] + else { + panic!("one aggregation"); + }; + assert_ne!(dd_id, sum_id); + // 5m of a 5m window sliding every minute: the newest closed window. + for rqe in ["p90", "p50"] { + assert_eq!( + rqe_aggregations(&inference, rqe), + [(dd_id, Some(1))], + "{rqe}" + ); + assert_eq!(stages(&inference, rqe), json!([]), "{rqe}"); + } + assert_eq!(stages(&inference, "sum"), json!([])); + } + + /// Trees only add `rqe_configs`: the deployments and `query_configs` + /// are those of the same table without trees, which has no `rqe_configs`. + #[test] + fn trees_leave_the_text_queries_configs_unchanged() { + let plan = plans(shared_p50()); + let without = run_on(TABLE, plan.to_string().as_bytes(), None).unwrap(); + let with = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + + assert_eq!( + with.to_streaming_yaml_string().unwrap(), + without.to_streaming_yaml_string().unwrap() + ); + let without_yaml = without.to_inference_yaml_string().unwrap(); + assert!(!without_yaml.contains("rqe_configs"), "{without_yaml}"); + let mut with_inference: serde_yaml::Value = + serde_yaml::from_str(&with.to_inference_yaml_string().unwrap()).unwrap(); + assert!(with_inference + .as_mapping_mut() + .unwrap() + .remove("rqe_configs") + .is_some()); + assert_eq!( + serde_yaml::to_string(&with_inference).unwrap(), + without_yaml + ); + } + + #[test] + fn writes_an_exact_sum_roll_up_as_an_rqe_query_time_sum() { + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"][0]["deployment_grouping"] = json!(["job", "label_0"]); + let output = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + + assert_eq!( + stages(&inference, "sum"), + json!([{ + "operator": "sum", + "grouping": {"mode": "by", "labels": ["job"]}, + "parameter": null, + }]) + ); + assert_eq!( + rqe_aggregations(&inference, "sum")[0].0, + query_aggregation_ids(&inference, "sum by (job) (data)")[0] + ); + } + + /// The engine merges a sketch across the deployment's groups itself, so + /// the RQE reads the deployment's aggregation with no stage. + #[test] + fn writes_a_sketch_roll_up_rqe_without_a_stage() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"promql": "quantile by (job) (0.5, data)", "interval_secs": 1, "rqes": ["p50"], + "expr": quantile_leaf("p50", 0.5, 1000, &["job"])}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let mut p50 = pick("p50", 0, "dd", json!({"alpha": 0.02}), 1); + p50["deployment_grouping"] = json!(["job", "label_0"]); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [p50]}, + ]}); + + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!(output.streaming_aggregation_count(), 1); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50"), [(1, Some(1))]); + assert_eq!(stages(&inference, "p50"), json!([])); + } + + /// A query with only a tree is planned in the language of the metrics + /// its leaves read: it gets a deployment and an `rqe_configs` entry, but + /// no `query_configs` entry. + #[test] + fn plans_a_tree_only_query() { + let mut table = tree_table(); + let tree_only = + json!({"interval_secs": 1, "expr": quantile_leaf("p50_by_job", 0.5, 1000, &["job"])}); + queries(&mut table).push(tree_only.clone()); + let mut plan = plans(shared_p50()); + plan["plans"][0]["chosen"] + .as_array_mut() + .unwrap() + .push(pick("p50_by_job", 5, "dd", json!({"alpha": 0.05}), 1)); + + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!(output.streaming_aggregation_count(), 3); + assert_eq!(output.inference_query_count(), 3); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50_by_job"), [(3, Some(1))]); + let streaming = output.to_streaming_config(QueryLanguage::promql).unwrap(); + let dd = &streaming.aggregation_configs[&3]; + assert_eq!( + dd.aggregation_type, + promql_utilities::query_logics::enums::AggregationType::DDSketch + ); + assert_eq!(dd.grouping_labels.labels, ["job"]); + + // A table of tree-only queries converts too. + let table = json!({"workloads": [{"e2e": { + "queries": [tree_only], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + assert_eq!(output.streaming_aggregation_count(), 1); + assert_eq!(output.inference_query_count(), 0); + let inference = inference_of(&output, QueryLanguage::promql); + assert_eq!(rqe_aggregations(&inference, "p50_by_job"), [(1, Some(1))]); + let err = run_table(&table, &plan, QueryLanguage::sql) + .err() + .expect("refused") + .to_string(); + assert!(err.contains("no e2e query is written in sql"), "{err}"); + } + + fn http_leaf(id: &str, capability: &str, grouping: &[&str], value: &str) -> Value { + json!({"rqe": {"id": id, "metric": "http", "capability": capability, + "lookback_ms": 300_000, "grouping": grouping, "value": value}}) + } + + /// `MIXED_TABLE`'s p99 query with its tree, and a tree-only SQL sum. + fn mixed_tree_table() -> Value { + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + let mut p99 = http_leaf("p99", "quantile", &["region"], "latency"); + p99["rqe"]["phi"] = json!(0.99); + queries(&mut table)[2]["expr"] = p99; + queries(&mut table).push(json!({"interval_secs": 60, + "expr": http_leaf("latency_sum", "sum", &["region"], "latency")})); + table + } + + /// A SQL query can't run a query-time stage, but its RQE can: an exact + /// roll-up of a tree-only SQL sum is written. + #[test] + fn writes_sql_rqe_configs_with_an_exact_roll_up() { + let mut plan = mixed_plan(1); + let mut sum = pick("latency_sum", 4, "exact-sum", json!({}), 60); + sum["deployment_grouping"] = json!(["region", "service"]); + plan["plans"][0]["chosen"].as_array_mut().unwrap().push(sum); + + let output = run_table(&mixed_tree_table(), &plan, QueryLanguage::sql).unwrap(); + + let inference = inference_of(&output, QueryLanguage::sql); + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["latency_sum", "p99"]); + let p99_query = "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region"; + // 5m of 5m windows sliding every minute: the newest closed window. + assert_eq!( + rqe_aggregations(&inference, "p99"), + [(query_aggregation_ids(&inference, p99_query)[0], Some(1))] + ); + assert_eq!( + stages(&inference, "latency_sum"), + json!([{ + "operator": "sum", + "grouping": {"mode": "by", "labels": ["region"]}, + "parameter": null, + }]) + ); + let streaming = output.to_streaming_config(QueryLanguage::sql).unwrap(); + let (sum_id, _) = rqe_aggregations(&inference, "latency_sum")[0]; + let sum = &streaming.aggregation_configs[&sum_id]; + assert_eq!(sum.table_name.as_deref(), Some("http")); + assert_eq!(sum.value_column.as_deref(), Some("latency")); + // A keyed accumulator keeps its groups as keys. + assert_eq!(sum.aggregated_labels.labels, ["region", "service"]); + + // The PromQL run skips the SQL trees. + let output = run_table(&mixed_tree_table(), &plan, QueryLanguage::promql).unwrap(); + assert!(inference_rqe_configs_absent(&output)); + } + + fn inference_rqe_configs_absent(output: &PlannerOutput) -> bool { + !output + .to_inference_yaml_string() + .unwrap() + .contains("rqe_configs") + } + + fn tree_refusal(table: &Value) -> String { + format!( + "{:#}", + run_table(table, &plans(shared_p50()), QueryLanguage::promql) + .err() + .expect("refused") + ) + } + + #[test] + fn refuses_a_tree_disagreeing_with_its_rqes() { + let mut table = tree_table(); + queries(&mut table)[1]["rqes"] = json!(["p90", "sum"]); + let err = tree_refusal(&table); + assert!( + err.contains(r#"names RQEs ["p90", "sum"] but its `expr` reads ["p90", "p50"]"#), + "{err}" + ); + } + + /// `rqes` and the tree's leaves are compared as sets: a SQL query's + /// `rqes` in another order, or a tree reading one leaf twice, agree. + #[test] + fn reads_a_tree_whose_rqes_are_reordered_or_repeated() { + let users = |id: &str| http_leaf(id, "cardinality", &["service"], "user_id"); + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + // `rqes` is ["users_1pct", "users_5pct"]. + queries(&mut table)[1]["expr"] = + json!({"op": "-", "lhs": users("users_5pct"), "rhs": users("users_1pct")}); + let output = run_table(&table, &mixed_plan(1), QueryLanguage::sql).unwrap(); + assert_eq!( + inference_of(&output, QueryLanguage::sql).rqe_configs.len(), + 2 + ); + + let mut table = tree_table(); + let p50 = quantile_leaf("p50", 0.5, 300_000, &["job", "label_0"]); + queries(&mut table)[2]["expr"] = json!({"op": "-", "lhs": p50.clone(), "rhs": p50}); + assert_eq!(queries(&mut table)[2]["rqes"], json!(["p50"])); + run_table(&table, &plans(shared_p50()), QueryLanguage::promql).unwrap(); + } + + /// The `rqe_configs` entry doesn't depend on the interval, so one leaf + /// run at two intervals is one entry on one aggregation. + #[test] + fn plans_one_rqe_read_at_two_intervals() { + let plan = plans(shared_p50()); + let once = run_table(&tree_table(), &plan, QueryLanguage::promql).unwrap(); + let mut table = tree_table(); + queries(&mut table) + .push(json!({"interval_secs": 10, "expr": leaf("sum", "sum", 1000, &["job"])})); + let twice = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + + assert_eq!( + twice.to_streaming_yaml_string().unwrap(), + once.to_streaming_yaml_string().unwrap() + ); + assert_eq!( + rqe_aggregations(&inference_of(&twice, QueryLanguage::promql), "sum"), + rqe_aggregations(&inference_of(&once, QueryLanguage::promql), "sum") + ); + } + + /// `--expr-only` skips the queries without a tree, and plans the others + /// from their tree leaves alone: only deployments and `rqe_configs`, no + /// `query_configs`. + #[test] + fn expr_only_plans_only_the_queries_with_a_tree() { + let mut table = tree_table(); + queries(&mut table)[0] + .as_object_mut() + .unwrap() + .remove("expr"); + let plan = plans(shared_p50()).to_string(); + let run = |expr_only| { + run_filtered( + &table.to_string(), + plan.as_bytes(), + None, + QueryLanguage::promql, + expr_only, + ) + .unwrap() + }; + + let all = run(false); + assert_eq!(all.streaming_aggregation_count(), 2); + assert_eq!(all.inference_query_count(), 3); + + let expr_only = run(true); + // Only the DDSketch the quantile trees read; no sum. + assert_eq!(expr_only.streaming_aggregation_count(), 1); + assert_eq!(expr_only.inference_query_count(), 0); + let inference = inference_of(&expr_only, QueryLanguage::promql); + assert!(inference.query_configs.is_empty()); + let mut ids: Vec<&str> = inference.rqe_configs.keys().map(String::as_str).collect(); + ids.sort(); + assert_eq!(ids, ["p50", "p90"]); + + // A table with no tree in the language has nothing to plan. + let err = run_filtered(TABLE, plan.as_bytes(), None, QueryLanguage::promql, true) + .err() + .expect("refused") + .to_string(); + assert!(err.contains("no e2e query with an `expr`"), "{err}"); + } + + /// A query with text needs `rqes`, even with a tree: its text leaves + /// map to RQEs by `rqes`, not by the tree's leaf order. Only a query + /// with just a tree may omit them (`plans_a_tree_only_query`). + #[test] + fn refuses_a_query_with_text_but_no_rqes() { + let mut table = tree_table(); + queries(&mut table)[1] + .as_object_mut() + .unwrap() + .remove("rqes"); + let err = tree_refusal(&table); + assert!( + err.contains( + r#"e2e query "quantile_over_time(0.9, data[5m]) / quantile_over_time(0.5, data[5m])" has text but no `rqes`"# + ), + "{err}" + ); + } + + #[test] + fn refuses_an_invalid_tree() { + let mut table = tree_table(); + queries(&mut table)[0]["expr"] = + json!({"agg": "sum", "by": ["region"], "of": leaf("sum", "sum", 1000, &["job"])}); + let err = tree_refusal(&table); + assert!( + err.contains(r#"e2e query "sum by (job) (data)": invalid `expr`"#), + "{err}" + ); + assert!(err.contains("not a subset"), "{err}"); + // A tree-only query is named by its leaves. + let mut table = tree_table(); + queries(&mut table).push(json!({"interval_secs": 1, "expr": + {"agg": "sum", "by": ["region"], "of": leaf("sum", "sum", 1000, &["job"])}})); + let err = tree_refusal(&table); + assert!( + err.contains(r#"e2e query with RQEs ["sum"]: invalid `expr`"#), + "{err}" + ); + } + + #[test] + fn refuses_one_rqe_as_two_different_leaves() { + let mut table = tree_table(); + queries(&mut table)[2]["expr"] = quantile_leaf("p50", 0.5, 300_000, &["job"]); + let err = tree_refusal(&table); + assert!( + err.contains(r#"RQE "p50" is two different leaves"#), + "{err}" + ); + } + + #[test] + fn refuses_a_query_without_text_or_tree_or_with_mismatched_languages() { + let mut mixed: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + let cases = [ + ( + json!({"interval_secs": 1, "rqes": ["sum"]}), + r#"e2e query reading ["sum"] needs `promql`, `sql` or `expr`"#, + ), + ( + json!({"interval_secs": 1}), + "e2e query with no text, `expr` or RQEs needs `promql`, `sql` or `expr`", + ), + ( + json!({"promql": "sum by (job) (data)", "interval_secs": 1, + "expr": http_leaf("sum", "sum", &[], "latency")}), + r#"e2e query "sum by (job) (data)" is written in PromQL but its `expr` reads records metrics"#, + ), + ( + json!({"interval_secs": 1, "expr": {"op": "+", + "lhs": leaf("sum", "sum", 1000, &[]), + "rhs": http_leaf("sum2", "sum", &[], "latency")}}), + r#"e2e query with RQEs ["sum", "sum2"]: its `expr` mixes records and non-records metrics"#, + ), + ( + json!({"interval_secs": 1, "expr": {"scalar": 1.0}}), + "root reads no RQE leaf", + ), + ]; + for (query, reason) in cases { + queries(&mut mixed).push(query); + let err = format!( + "{:#}", + run_table(&mixed, &mixed_plan(1), QueryLanguage::sql) + .err() + .expect("refused") + ); + assert!(err.contains(reason), "{err}"); + queries(&mut mixed).pop(); + } + } + + #[test] + fn refuses_a_leaf_the_table_cannot_describe() { + let records_leaf = |value: Option<&str>, grouping: &[&str]| { + let mut l = http_leaf("p99", "quantile", grouping, "latency"); + l["rqe"]["phi"] = json!(0.99); + match value { + Some(value) => l["rqe"]["value"] = json!(value), + None => { + l["rqe"].as_object_mut().unwrap().remove("value"); + } + } + l + }; + let mut unknown = records_leaf(Some("latency"), &["region"]); + unknown["rqe"]["metric"] = json!("nope"); + for (expr, reason) in [ + (unknown, r#"reads unknown metric "nope""#), + (records_leaf(None, &["region"]), "without a `value`"), + ( + records_leaf(Some("bytes"), &["region"]), + r#"which is not a value of "http""#, + ), + ( + records_leaf(Some("latency"), &["zone"]), + r#"groups by "zone""#, + ), + ] { + let mut table: Value = serde_json::from_str(MIXED_TABLE).unwrap(); + queries(&mut table)[2]["expr"] = expr; + let err = format!( + "{:#}", + run_table(&table, &mixed_plan(1), QueryLanguage::sql) + .err() + .expect("refused") + ); + assert!(err.contains(reason), "{err}"); + } + let mut table = tree_table(); + queries(&mut table)[0]["expr"]["rqe"]["value"] = json!("v"); + let err = tree_refusal(&table); + assert!(err.contains("is not a records metric"), "{err}"); + } + + #[test] + fn refuses_a_tree_leaf_the_plan_does_not_serve() { + let mut table = tree_table(); + queries(&mut table).push(json!({"interval_secs": 1, + "expr": quantile_leaf("unplanned", 0.5, 1000, &["job"])})); + let err = tree_refusal(&table); + assert!(err.contains(r#"doesn't serve RQE "unplanned""#), "{err}"); + } + + /// A `topk_by` leaf is a Raqe grouped by its buckets: one heap per `job`, + /// keyed by the remaining series label. + #[test] + fn writes_a_tree_only_topk_by_leaf_as_one_heap_per_bucket() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"interval_secs": 60, "expr": {"rqe": {"id": "top", "metric": "data", + "capability": "topk_by_value", "k": 5, "lookback_ms": 300_000, + "grouping": ["job", "label_0"], "topk_by": ["job"]}}}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let top = pick( + "top", + 0, + "cms-heap-topk-fastpath-vector2d", + json!({"rows": 3, "cols": 1024, "heap": 50}), + 300, + ); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [top]}, + ]}); + let output = run_table(&table, &plan, QueryLanguage::promql).unwrap(); + let inference = inference_of(&output, QueryLanguage::promql); + let [(id, _)] = rqe_aggregations(&inference, "top")[..] else { + panic!("{:?}", inference.rqe_configs); + }; + assert_eq!(stages(&inference, "top"), json!([])); + let streaming = output.to_streaming_config(QueryLanguage::promql).unwrap(); + let heap = streaming.get_aggregation_config(id).unwrap(); + assert_eq!(heap.grouping_labels.labels, ["job"]); + assert_eq!(heap.aggregated_labels.labels, ["label_0"]); + } + + /// A topk deployment can't roll up, for an RQE as for a query. + #[test] + fn refuses_a_tree_only_topk_roll_up() { + let table = json!({"workloads": [{"e2e": { + "queries": [{"interval_secs": 60, "expr": {"rqe": {"id": "top", "metric": "data", + "capability": "topk_by_value", "k": 5, "lookback_ms": 300_000, + "grouping": ["job", "label_0"]}}}], + "metrics": {"data": {"type": "gauge", "labels": [{"name": "job"}, {"name": "label_0"}]}} + }}]}); + let mut top = pick( + "top", + 0, + "cms-heap-topk-fastpath-vector2d", + json!({"rows": 3, "cols": 1024, "heap": 50}), + 300, + ); + top["deployment_grouping"] = json!(["job"]); + let plan = json!({"plans": [ + {"method": "asap", "weights": "cpu", "kind": "cheapest", "chosen": [top]}, + ]}); + let err = run_table(&table, &plan, QueryLanguage::promql) + .err() + .expect("refused") + .to_string(); + assert!(err.contains(r#"query "top""#), "{err}"); + assert!(err.contains("roll-ups need an aggregation"), "{err}"); + } } diff --git a/asap-planner-rs/tests/saved_plan_e2e_tables.rs b/asap-planner-rs/tests/saved_plan_e2e_tables.rs new file mode 100644 index 00000000..d4abc599 --- /dev/null +++ b/asap-planner-rs/tests/saved_plan_e2e_tables.rs @@ -0,0 +1,369 @@ +//! `--planner saved` on real sketch-bench eval tables and their saved plans: +//! the `all` workload of sketch-bench's `rqe-optimizer/results/e2e-rate10`, +//! whose e2e queries carry RQE trees (all but its `rate` queries), and a +//! pilot random workload, `seed-862/workload-01-pilot`, whose top-k queries +//! are `topk by (region, service)`. + +use std::collections::{BTreeSet, HashMap}; +use std::path::{Path, PathBuf}; + +use asap_planner::optimizer::{saved_plan_output, PlanSelector}; +use asap_planner::PlannerOutput; +use asap_types::enums::QueryLanguage; +use serde_json::Value; + +/// The table's scrape interval: instant RQEs read one 100 ms scrape. +const SCRAPE_INTERVAL_MS: u64 = 100; + +fn data_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/test_data/saved_plan") +} + +fn table_path() -> PathBuf { + data_dir().join("synthetic-templatesall-shared1.json") +} + +fn convert(table: &Path, language: QueryLanguage) -> PlannerOutput { + convert_with_plans( + table, + "synthetic-templatesall-shared1.plans.json.gz", + language, + ) + .unwrap() +} + +fn convert_with_plans( + table: &Path, + plans: &str, + language: QueryLanguage, +) -> anyhow::Result { + convert_filtered(table, plans, language, false) +} + +fn convert_filtered( + table: &Path, + plans: &str, + language: QueryLanguage, + expr_only: bool, +) -> anyhow::Result { + let selector = PlanSelector { + method: "asap", + weights: "cpu", + sla_ms: None, + }; + saved_plan_output( + table, + &data_dir().join(plans), + &selector, + language, + SCRAPE_INTERVAL_MS, + expr_only, + ) +} + +fn table_queries() -> Vec { + let table: Value = + serde_json::from_str(&std::fs::read_to_string(table_path()).unwrap()).unwrap(); + table["workloads"][0]["e2e"]["queries"] + .as_array() + .unwrap() + .clone() +} + +fn language_key(language: QueryLanguage) -> &'static str { + match language { + QueryLanguage::promql => "promql", + QueryLanguage::sql => "sql", + other => panic!("{other:?}"), + } +} + +fn leaf_ids(expr: &Value, out: &mut BTreeSet) { + if let Some(rqe) = expr.get("rqe") { + out.insert(rqe["id"].as_str().unwrap().to_string()); + } + for child in ["of", "lhs", "rhs"] { + if let Some(child) = expr.get(child) { + leaf_ids(child, out); + } + } +} + +/// Every tree leaf of the language's queries gets an `rqe_configs` entry +/// reading what its single-RQE query reads. +#[test] +fn writes_rqe_configs_for_every_tree_leaf() { + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let output = convert(&table_path(), language); + let inference = output.to_inference_config(language).unwrap(); + let queries: Vec = table_queries() + .into_iter() + .filter(|q| q.get(language_key(language)).is_some()) + .collect(); + + let mut expected = BTreeSet::new(); + for q in &queries { + if let Some(expr) = q.get("expr") { + leaf_ids(expr, &mut expected); + } + } + assert!(!expected.is_empty(), "{language:?}"); + let written: BTreeSet = inference.rqe_configs.keys().cloned().collect(); + assert_eq!(written, expected, "{language:?}"); + + let query_configs: HashMap<&str, _> = inference + .query_configs + .iter() + .map(|c| (c.query.as_str(), c)) + .collect(); + for q in &queries { + let [rqe] = q["rqes"].as_array().unwrap().as_slice() else { + continue; + }; + let Some(rqe) = inference.rqe_configs.get(rqe.as_str().unwrap()) else { + continue; + }; + let query = query_configs[q[language_key(language)].as_str().unwrap()]; + let ids = |refs: &[asap_types::AggregationReference]| { + refs.iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect::>() + }; + assert_eq!(ids(&rqe.aggregations), ids(&query.aggregations), "{q}"); + assert_eq!( + rqe.query_time_aggregations.len(), + query.query_time_aggregations.len(), + "{q}" + ); + } + } +} + +/// The trees add only `rqe_configs`: the table without them converts to the +/// same deployments and `query_configs`. +#[test] +fn trees_add_only_rqe_configs() { + let mut table: Value = + serde_json::from_str(&std::fs::read_to_string(table_path()).unwrap()).unwrap(); + for q in table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + { + q.as_object_mut().unwrap().remove("expr"); + } + let dir = tempfile::tempdir().unwrap(); + let stripped = dir.path().join("table.json"); + std::fs::write(&stripped, table.to_string()).unwrap(); + + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let with = convert(&table_path(), language); + let without = convert(&stripped, language); + let yaml = |s: String| serde_yaml::from_str::(&s).unwrap(); + assert_eq!( + yaml(with.to_streaming_yaml_string().unwrap()), + yaml(without.to_streaming_yaml_string().unwrap()), + "{language:?}" + ); + let mut with_inference = yaml(with.to_inference_yaml_string().unwrap()); + let without_inference = yaml(without.to_inference_yaml_string().unwrap()); + assert!(without_inference.get("rqe_configs").is_none()); + assert!(with_inference + .as_mapping_mut() + .unwrap() + .remove("rqe_configs") + .is_some()); + assert_eq!(with_inference, without_inference, "{language:?}"); + } +} + +/// `--expr-only` plans tree leaves alone: no `query_configs`, every tree +/// leaf's `rqe_configs` entry, and only the aggregations those entries read. +/// The `rate` queries have no tree, so nothing reads `data_total`. +#[test] +fn expr_only_skips_the_rate_queries() { + let rate_queries: Vec = table_queries() + .iter() + .filter(|q| q.get("expr").is_none()) + .map(|q| q["promql"].as_str().unwrap().to_string()) + .collect(); + assert!(!rate_queries.is_empty()); + assert!(rate_queries.iter().all(|q| q.starts_with("rate("))); + let reads_rate_metric = |output: &PlannerOutput| { + output + .to_streaming_config(QueryLanguage::promql) + .unwrap() + .aggregation_configs + .values() + .any(|a| a.metric == "data_total") + }; + assert!(reads_rate_metric(&convert( + &table_path(), + QueryLanguage::promql + ))); + + let plans = "synthetic-templatesall-shared1.plans.json.gz"; + for language in [QueryLanguage::promql, QueryLanguage::sql] { + let all = convert(&table_path(), language); + let expr_only = convert_filtered(&table_path(), plans, language, true).unwrap(); + if language == QueryLanguage::promql { + assert!(!reads_rate_metric(&expr_only)); + } + let inference = expr_only.to_inference_config(language).unwrap(); + assert!(inference.query_configs.is_empty(), "{language:?}"); + let rqe_ids = |i: &asap_types::InferenceConfig| { + i.rqe_configs.keys().cloned().collect::>() + }; + assert_eq!( + rqe_ids(&inference), + rqe_ids(&all.to_inference_config(language).unwrap()), + "{language:?}" + ); + let read: BTreeSet = inference + .rqe_configs + .values() + .flat_map(|c| c.aggregations.iter().map(|a| a.aggregation_id)) + .collect(); + let deployed: BTreeSet = expr_only + .to_streaming_config(language) + .unwrap() + .aggregation_configs + .keys() + .copied() + .collect(); + assert_eq!(deployed, read, "{language:?}"); + } +} + +const PILOT_PLANS: &str = "random-seed862-workload01-pilot.plans.json.gz"; + +/// The pilot table without its `avg by (...) (quantile_over_time(...))` +/// queries, which `--planner saved` can't write yet. +fn pilot_table() -> Value { + let path = data_dir().join("random-seed862-workload01-pilot.json"); + let mut table: Value = serde_json::from_str(&std::fs::read_to_string(path).unwrap()).unwrap(); + table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + .retain(|q| !q["promql"].as_str().unwrap().starts_with("avg by")); + table +} + +/// The pilot table with a tree on each `topk by (region, service) (4, +/// sum_over_time(data[w]))` query: a leaf over every series label, ranked +/// within `topk_by` buckets. Returns the table and the top-k RQE ids. +fn pilot_with_topk_trees(topk_by: Option<&[&str]>) -> (Value, Vec) { + let mut table = pilot_table(); + let mut ids = Vec::new(); + for q in table["workloads"][0]["e2e"]["queries"] + .as_array_mut() + .unwrap() + { + let promql = q["promql"].as_str().unwrap(); + if !promql.starts_with("topk by (region, service) (4, sum_over_time(data[") { + continue; + } + let id = q["rqes"][0].as_str().unwrap().to_string(); + let lookback_ms = if promql.contains("[1m]") { + 60_000 + } else { + assert!(promql.contains("[20m]"), "{promql}"); + 1_200_000 + }; + let mut leaf = serde_json::json!({ + "id": id, "metric": "data", "capability": "topk_by_value", "k": 4, + "lookback_ms": lookback_ms, + "grouping": ["region", "service", "job", "label_0"], + }); + if let Some(topk_by) = topk_by { + leaf["topk_by"] = serde_json::json!(topk_by); + } + q["expr"] = serde_json::json!({"rqe": leaf}); + ids.push(id); + } + assert_eq!(ids.len(), 2, "the pilot has two top-k queries"); + (table, ids) +} + +fn write_table(dir: &tempfile::TempDir, table: &Value) -> PathBuf { + let path = dir.path().join("table.json"); + std::fs::write(&path, table.to_string()).unwrap(); + path +} + +/// A `topk by (region, service)` leaf is the plan's top-k RQE, grouped by +/// its buckets: it reads the same per-bucket heap as its PromQL query and +/// adds only its `rqe_configs` entry. +#[test] +fn pilot_topk_by_leaf_reads_its_promql_querys_heap() { + let dir = tempfile::tempdir().unwrap(); + let (with_trees, ids) = pilot_with_topk_trees(Some(&["region", "service"])); + let with = convert_with_plans( + &write_table(&dir, &with_trees), + PILOT_PLANS, + QueryLanguage::promql, + ) + .unwrap(); + let without_dir = tempfile::tempdir().unwrap(); + let without = convert_with_plans( + &write_table(&without_dir, &pilot_table()), + PILOT_PLANS, + QueryLanguage::promql, + ) + .unwrap(); + + let yaml = |s: String| serde_yaml::from_str::(&s).unwrap(); + assert_eq!( + yaml(with.to_streaming_yaml_string().unwrap()), + yaml(without.to_streaming_yaml_string().unwrap()) + ); + let inference = with.to_inference_config(QueryLanguage::promql).unwrap(); + let written: BTreeSet<&String> = inference.rqe_configs.keys().collect(); + assert_eq!(written, ids.iter().collect()); + let streaming = with.to_streaming_config(QueryLanguage::promql).unwrap(); + for q in pilot_with_topk_trees(None).0["workloads"][0]["e2e"]["queries"] + .as_array() + .unwrap() + { + let Some(rqe) = inference.rqe_configs.get(q["rqes"][0].as_str().unwrap()) else { + continue; + }; + let query = inference + .query_configs + .iter() + .find(|c| c.query == q["promql"].as_str().unwrap()) + .unwrap(); + let refs = |refs: &[asap_types::AggregationReference]| { + refs.iter() + .map(|a| (a.aggregation_id, a.num_aggregates_to_retain)) + .collect::>() + }; + assert_eq!(refs(&rqe.aggregations), refs(&query.aggregations), "{q}"); + assert!(rqe.query_time_aggregations.is_empty(), "{q}"); + // One heap per (region, service) bucket, keyed by the other labels. + let heap = streaming + .get_aggregation_config(rqe.aggregations[0].aggregation_id) + .unwrap(); + assert_eq!(heap.grouping_labels.labels, ["region", "service"], "{q}"); + assert_eq!(heap.aggregated_labels.labels, ["job", "label_0"], "{q}"); + } +} + +/// Without `topk_by` the leaf ranks every series as one bucket: not the +/// plan's per-(region, service) RQE, so the deployment its PromQL query +/// shares can't serve it. +#[test] +fn pilot_topk_leaf_without_topk_by_is_refused() { + let dir = tempfile::tempdir().unwrap(); + let (table, _) = pilot_with_topk_trees(None); + let Err(error) = convert_with_plans( + &write_table(&dir, &table), + PILOT_PLANS, + QueryLanguage::promql, + ) else { + panic!("a single-bucket leaf is not the plan's RQE"); + }; + assert!( + error.to_string().contains("need different sketches"), + "{error:#}" + ); +} diff --git a/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json new file mode 100644 index 00000000..38e91a55 --- /dev/null +++ b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.json @@ -0,0 +1,448 @@ +{ + "inputs": { + "workload-01.json": "edacdc8662d75c0cd1c8bcc3136614225fd9f273d98360e73d8c563016586ae4" + }, + "plan": "ProjectASAP/ASAPQuery#879", + "schema_version": 1, + "sketch_bench_revision": "85699c9edef79ed6362ff19020d4c1f486af531d", + "workloads": [ + { + "dataset": "random/seed=862/workload-01", + "e2e": { + "metrics": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ], + "samples_per_sec_per_series": 10, + "type": "gauge", + "values": { + "dist": { + "pareto": 2.0 + }, + "value": { + "keys": 10000, + "zipf": 1.1 + } + } + } + }, + "queries": [ + { + "interval_secs": 20, + "promql": "sum by (region, service, job) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q01-sum-fine/keys/600s" + ] + }, + { + "interval_secs": 20, + "promql": "sum by (region, service) (sum_over_time(data[10m]))", + "rqes": [ + "random/seed=862/workload-01/q02-sum-coarse/keys/600s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.99, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q03-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region) (quantile_over_time(0.95, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q04-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "avg by (region, service, job) (quantile_over_time(0.75, data[100s]))", + "rqes": [ + "random/seed=862/workload-01/q05-quantile/values/100s" + ] + }, + { + "interval_secs": 1, + "promql": "topk by (region, service) (4, sum_over_time(data[1m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s" + ] + }, + { + "interval_secs": 30, + "promql": "topk by (region, service) (4, sum_over_time(data[20m]))", + "rqes": [ + "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s" + ] + } + ] + }, + "rqes": [ + { + "assumptions": [], + "capability": "sum", + "covered_groups": 200, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service", + "job" + ], + "id": "random/seed=862/workload-01/q01-sum-fine/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 200, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 300000.0, + "max_covered_share": 0.005, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.005, + "queries_per_instance": 1, + "query_id": "q01-sum-fine", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "sum", + "covered_groups": 20, + "covered_max_N": 3000000.0, + "covered_min_N": 3000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "exact-sum", + "grid_K": null, + "grid_param": null, + "sketch": "exact-sum", + "target": 0.0 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/q02-sum-coarse/keys/600s", + "instant": false, + "interval_secs": 20, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": null + }, + "lookback_secs": 600.0, + "max_N": 3000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 3000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "q02-sum-coarse", + "range": "600s", + "stream": "data/value" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q03-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q03-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q04-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q04-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [], + "capability": "quantile", + "covered_groups": 10000, + "covered_max_N": 1000.0, + "covered_min_N": 1000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "kll", + "grid_K": null, + "grid_param": 2.0, + "sketch": "kll-percall", + "target": 0.01 + }, + { + "family": "dd", + "grid_K": null, + "grid_param": 2.0, + "sketch": "dd", + "target": 0.01 + } + ], + "grouping": [ + "region", + "service", + "job", + "label_0" + ], + "id": "random/seed=862/workload-01/q05-quantile/values/100s", + "instant": false, + "interval_secs": 1, + "kind": "values", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 10000, + "keys_per_window": null + }, + "lookback_secs": 100.0, + "max_N": 1000.0, + "max_covered_share": 0.0001, + "metric": "data", + "min_N": 1000.0, + "min_covered_share": 0.0001, + "queries_per_instance": 1, + "query_id": "q05-quantile", + "range": "100s", + "stream": "data/dist" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 300000.0, + "covered_min_N": 300000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q06-topk-sum/keys/60s", + "instant": false, + "interval_secs": 1, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 60.0, + "max_N": 300000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 300000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q06-topk-sum", + "range": "60s", + "stream": "data/value" + }, + { + "assumptions": [ + "500 keys per heap read at the curves' 1000" + ], + "capability": "topk", + "covered_groups": 20, + "covered_max_N": 6000000.0, + "covered_min_N": 6000000.0, + "covers_share": null, + "dataset": "random/seed=862/workload-01", + "families": [ + { + "family": "topk", + "grid_K": 1000, + "grid_param": 1.1, + "sketch": "cms-heap-topk-fastpath-vector2d", + "target": 0.95 + } + ], + "grouping": [ + "region", + "service" + ], + "id": "random/seed=862/workload-01/topk4_q07-topk-sum/keys/1200s", + "instant": false, + "interval_secs": 30, + "kind": "keys", + "label_set": { + "arrival_rate_per_sec": 100000, + "groups": 20, + "keys_per_window": 1000 + }, + "lookback_secs": 1200.0, + "max_N": 6000000.0, + "max_covered_share": 0.05, + "metric": "data", + "min_N": 6000000.0, + "min_covered_share": 0.05, + "queries_per_instance": 1, + "query_id": "topk4_q07-topk-sum", + "range": "1200s", + "stream": "data/value" + } + ], + "schemas": { + "data": { + "labels": [ + { + "cardinality": 4, + "name": "region" + }, + { + "cardinality": 5, + "child_of": "region", + "name": "service" + }, + { + "cardinality": 10, + "child_of": "service", + "name": "job" + }, + { + "cardinality": 50, + "child_of": "job", + "name": "label_0" + } + ] + } + } + } + ] +} diff --git a/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz new file mode 100644 index 00000000..dcd2f883 Binary files /dev/null and b/asap-planner-rs/tests/test_data/saved_plan/random-seed862-workload01-pilot.plans.json.gz differ diff --git a/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json new file mode 100644 index 00000000..3102ab21 --- /dev/null +++ b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.json @@ -0,0 +1 @@ +{"inputs": {}, "plan": "ProjectASAP/ASAPQuery#777", "schema_version": 1, "sketch_bench_revision": "acc2db1", "workloads": [{"dataset": "synthetic/templates=all/shared=1", "e2e": {"metrics": {"data": {"labels": [{"cardinality": 10, "name": "job"}, {"cardinality": 1000, "child_of": "job", "name": "label_0"}], "samples_per_sec_per_series": 10, "type": "gauge", "values": {"dist": {"pareto": 2.0}, "value": {"keys": 10000, "zipf": 1.1}}}, "data_total": {"labels": [{"cardinality": 10, "name": "job"}, {"cardinality": 1000, "child_of": "job", "name": "label_0"}], "samples_per_sec_per_series": 10, "type": "counter", "values": {"increment": {"keys": 10000, "zipf": 1.1}}}, "flows": {"labels": [{"cardinality": 1000, "name": "dst_subnet", "skew": 1.1}, {"cardinality": 1000, "name": "dst_port", "skew": 1.2}, {"cardinality": 3, "name": "proto", "skew": 0.0}], "records_per_sec": 200000, "type": "records", "values": {"src_ip": {"burst": {"keys": 10000, "label": "dst_subnet", "share": 0.05, "value": 999}, "keys": 1000000, "zipf": 0.0}}}, "http": {"labels": [{"cardinality": 4, "name": "region", "skew": 0.5}, {"cardinality": 25, "name": "service", "skew": 1.1}, {"cardinality": 25, "child_of": "service", "name": "endpoint", "skew": 1.1}, {"cardinality": 4, "name": "status", "skew": 2.0}], "records_per_sec": 100000, "type": "records", "values": {"latency": {"anomaly": {"factor": 10, "label": "endpoint"}, "pareto": 2.0, "scaled_by": "service"}, "user_id": {"keys": 1000000, "zipf": 0.8}}}}, "queries": [{"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job/keys/1s", "lookback_ms": 100, "metric": "data"}}, "interval_secs": 1, "promql": "sum by (job) (data)", "rqes": ["synthetic/templates=all/shared=1/sum_by_job/keys/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.5}}, "interval_secs": 1, "promql": "quantile by (job) (0.5, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.75}}, "interval_secs": 1, "promql": "quantile by (job) (0.75, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.9}}, "interval_secs": 1, "promql": "quantile by (job) (0.9, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.95}}, "interval_secs": 1, "promql": "quantile by (job) (0.95, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s", "lookback_ms": 100, "metric": "data", "phi": 0.99}}, "interval_secs": 1, "promql": "quantile by (job) (0.99, data)", "rqes": ["synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m", "k": 32, "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[15m]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/15m", "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[15m])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m"]}, {"interval_secs": 60, "promql": "rate(data_total[15m])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/15m"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m", "lookback_ms": 900000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[15m]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "lookback_ms": 900000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[15m]) / quantile_over_time(0.5, data[15m])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h", "k": 32, "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[1h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/1h", "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[1h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h"]}, {"interval_secs": 60, "promql": "rate(data_total[1h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/1h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h", "lookback_ms": 3600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[1h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "lookback_ms": 3600000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[1h]) / quantile_over_time(0.5, data[1h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h", "k": 32, "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[6h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/6h", "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[6h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h"]}, {"interval_secs": 60, "promql": "rate(data_total[6h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/6h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h", "lookback_ms": 21600000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[6h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "lookback_ms": 21600000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[6h]) / quantile_over_time(0.5, data[6h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h"]}, {"expr": {"rqe": {"capability": "topk_by_value", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h", "k": 32, "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "topk(32, sum_over_time(data[24h]))", "rqes": ["synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/24h", "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "sum_over_time(data[24h])", "rqes": ["synthetic/templates=all/shared=1/sum_over_time/keys/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.5}}, "interval_secs": 60, "promql": "quantile_over_time(0.5, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.75}}, "interval_secs": 60, "promql": "quantile_over_time(0.75, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.9}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.95}}, "interval_secs": 60, "promql": "quantile_over_time(0.95, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h"]}, {"expr": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.99}}, "interval_secs": 60, "promql": "quantile_over_time(0.99, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h"]}, {"interval_secs": 60, "promql": "rate(data_total[24h])", "rqes": ["synthetic/templates=all/shared=1/rate/keys/24h"]}, {"expr": {"rqe": {"capability": "sum", "grouping": ["job"], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h", "lookback_ms": 86400000, "metric": "data"}}, "interval_secs": 60, "promql": "sum by (job) (sum_over_time(data[24h]))", "rqes": ["synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h"]}, {"expr": {"lhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.9}}, "op": "/", "rhs": {"rqe": {"capability": "quantile", "grouping": ["job", "label_0"], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "lookback_ms": 86400000, "metric": "data", "phi": 0.5}}}, "interval_secs": 60, "promql": "quantile_over_time(0.9, data[24h]) / quantile_over_time(0.5, data[24h])", "rqes": ["synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h"]}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m"], "sql": "SELECT dst_subnet, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_subnet", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m"], "sql": "SELECT dst_subnet, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_subnet", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m"], "sql": "SELECT dst_port, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_port", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m"], "sql": "SELECT dst_port, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_port", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m"], "sql": "SELECT dst_subnet, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_subnet, proto", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_subnet", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m"], "sql": "SELECT dst_subnet, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_subnet, proto", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m", "lookback_ms": 300000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m"], "sql": "SELECT dst_port, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY dst_port, proto", "table": "flows", "value": "src_ip", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m", "lookback_ms": 900000, "metric": "flows", "value": "src_ip"}}, "grouping": ["dst_port", "proto"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m"], "sql": "SELECT dst_port, proto, COUNT(DISTINCT src_ip) FROM flows WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY dst_port, proto", "table": "flows", "value": "src_ip", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m"], "sql": "SELECT region, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m"], "sql": "SELECT region, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m"], "sql": "SELECT service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m"], "sql": "SELECT service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m"], "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m"], "sql": "SELECT region, service, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m"], "sql": "SELECT endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m"], "sql": "SELECT endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m"], "sql": "SELECT status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m"], "sql": "SELECT status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m"], "sql": "SELECT region, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m"], "sql": "SELECT region, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m"], "sql": "SELECT region, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m"], "sql": "SELECT region, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m"], "sql": "SELECT service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m"], "sql": "SELECT service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m"], "sql": "SELECT endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m"], "sql": "SELECT endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m"], "sql": "SELECT region, service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m"], "sql": "SELECT region, service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m"], "sql": "SELECT region, service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m"], "sql": "SELECT region, service, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m"], "sql": "SELECT region, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m"], "sql": "SELECT region, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m"], "sql": "SELECT service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m"], "sql": "SELECT service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m"], "sql": "SELECT region, service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["region", "service", "endpoint", "status"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m"], "sql": "SELECT region, service, endpoint, status, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint, status", "table": "http", "value": "user_id", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m"], "sql": "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m"], "sql": "SELECT region, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m"], "sql": "SELECT service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m"], "sql": "SELECT service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m"], "sql": "SELECT endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m"], "sql": "SELECT endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m"], "sql": "SELECT region, service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m"], "sql": "SELECT region, service, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m"], "sql": "SELECT region, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m", "lookback_ms": 300000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m"], "sql": "SELECT service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "latency", "window_secs": 300}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m"], "sql": "SELECT service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "quantile", "expr": {"rqe": {"capability": "quantile", "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m", "lookback_ms": 900000, "metric": "http", "phi": 0.99, "value": "latency"}}, "grouping": ["region", "service", "endpoint"], "interval_secs": 60, "quantile": 0.99, "rqes": ["synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m"], "sql": "SELECT region, service, endpoint, QUANTILE(0.99, latency) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY region, service, endpoint", "table": "http", "value": "latency", "window_secs": 900}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m", "lookback_ms": 300000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m"], "sql": "SELECT service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -300, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "user_id", "window_secs": 300}, {"capability": "cardinality", "expr": {"rqe": {"capability": "cardinality", "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m", "lookback_ms": 900000, "metric": "http", "value": "user_id"}}, "grouping": ["service", "endpoint"], "interval_secs": 60, "rqes": ["synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m"], "sql": "SELECT service, endpoint, COUNT(DISTINCT user_id) FROM http WHERE ts BETWEEN DATEADD(s, -900, NOW()) AND NOW() GROUP BY service, endpoint", "table": "http", "value": "user_id", "window_secs": 900}]}, "rqes": [{"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job/keys/1s", "instant": false, "interval_secs": 1, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "sum_by_job", "range": "1s", "stream": "job/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.5/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.5", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.75/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.75", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.9/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.9", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.95/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.95", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_by_job_p0.99/values/1s", "instant": false, "interval_secs": 1, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 0.1, "max_N": 1000.0, "metric": "data", "min_N": 1000.0, "queries_per_instance": 1, "query_id": "quantile_by_job_p0.99", "range": "1s", "stream": "job/dist"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 900, "max_N": 90000000.0, "metric": "data", "min_N": 90000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "15m", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "15m", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "15m", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000.0, "metric": "data", "min_N": 9000.0, "queries_per_instance": 1, "query_id": "rate", "range": "15m", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 900, "max_N": 9000000.0, "metric": "data", "min_N": 9000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "15m", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 3600, "max_N": 360000000.0, "metric": "data", "min_N": 360000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "1h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "1h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/1h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "1h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000.0, "metric": "data", "min_N": 36000.0, "queries_per_instance": 1, "query_id": "rate", "range": "1h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/1h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 3600, "max_N": 36000000.0, "metric": "data", "min_N": 36000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "1h", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 21600, "max_N": 2160000000.0, "metric": "data", "min_N": 2160000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "6h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "6h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/6h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "6h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000.0, "metric": "data", "min_N": 216000.0, "queries_per_instance": 1, "query_id": "rate", "range": "6h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/6h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 21600, "max_N": 216000000.0, "metric": "data", "min_N": 216000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "6h", "stream": "job/value"}, {"assumptions": [], "capability": "topk", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "topk", "grid_K": 10000, "grid_param": 1.1, "sketch": "cms-heap-topk-fastpath-vector2d", "target": 0.95}], "id": "synthetic/templates=all/shared=1/topk32_sum_by_label0_sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 1, "keys_per_window": 10000}, "lookback_secs": 86400, "max_N": 8640000000.0, "metric": "data", "min_N": 8640000000.0, "queries_per_instance": 1, "query_id": "topk32_sum_by_label0_sum_over_time", "range": "24h", "stream": "label_0/value"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "sum_over_time", "range": "24h", "stream": "series/value"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.5/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.5", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.75/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.75", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.9/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.9", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.95/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.95", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "quantile", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "id": "synthetic/templates=all/shared=1/quantile_over_time_p0.99/values/24h", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "quantile_over_time_p0.99", "range": "24h", "stream": "series/dist"}, {"assumptions": [], "capability": "rate", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-increase", "grid_K": null, "grid_param": null, "sketch": "exact-increase", "target": 0.0}], "id": "synthetic/templates=all/shared=1/rate/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000.0, "metric": "data", "min_N": 864000.0, "queries_per_instance": 1, "query_id": "rate", "range": "24h", "stream": "series/increment"}, {"assumptions": [], "capability": "sum", "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "exact-sum", "grid_K": null, "grid_param": null, "sketch": "exact-sum", "target": 0.0}], "id": "synthetic/templates=all/shared=1/sum_by_job_sum_over_time/keys/24h", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10, "keys_per_window": null}, "lookback_secs": 86400, "max_N": 864000000.0, "metric": "data", "min_N": 864000000.0, "queries_per_instance": 1, "query_id": "sum_by_job_sum_over_time", "range": "24h", "stream": "job/value"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 10228202.546091888, "covered_min_N": 3005126.2445390555, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 60000.0, "max_covered_share": 0.1704700424348648, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.05008543740898426, "queries_per_instance": 1, "query_id": "t11_distinct_src_by_dst_subnet", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 30684607.638275664, "covered_min_N": 9015378.733617166, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet"], "id": "synthetic/templates=all/shared=1/t11_distinct_src_by_dst_subnet/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 180000.0, "max_covered_share": 0.1704700424348648, "metric": "flows", "min_N": 180000.0, "min_covered_share": 0.05008543740898426, "queries_per_instance": 1, "query_id": "t11_distinct_src_by_dst_subnet", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 13838389.036778929, "covered_min_N": 3702883.342543191, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 60000.0, "max_covered_share": 0.2306398172796488, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.061714722375719844, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 41515167.11033678, "covered_min_N": 11108650.027629571, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 1000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 180000.0, "max_covered_share": 0.2306398172796488, "metric": "flows", "min_N": 180000.0, "min_covered_share": 0.061714722375719844, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 3409400.848697296, "covered_min_N": 3409400.848697296, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 20000.0, "max_covered_share": 0.056823347478288266, "metric": "flows", "min_N": 20000.0, "min_covered_share": 0.056823347478288266, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_subnet,proto", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 10228202.546091888, "covered_min_N": 10228202.546091888, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_subnet", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_subnet,proto/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 60000.0, "max_covered_share": 0.056823347478288266, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.056823347478288266, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_subnet,proto", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 4612796.345592976, "covered_min_N": 4612796.345592976, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 20000.0, "max_covered_share": 0.07687993909321626, "metric": "flows", "min_N": 20000.0, "min_covered_share": 0.07687993909321626, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port,proto", "range": "5m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 13838389.036778927, "covered_min_N": 13838389.036778927, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.0, "sketch": "hll", "target": 0.02}], "grouping": ["dst_port", "proto"], "id": "synthetic/templates=all/shared=1/t12_distinct_src_by_dst_port,proto/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 200000, "groups": 3000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 60000.0, "max_covered_share": 0.07687993909321626, "metric": "flows", "min_N": 60000.0, "min_covered_share": 0.07687993909321626, "queries_per_instance": 1, "query_id": "t12_distinct_src_by_dst_port,proto", "range": "15m", "stream": "flows/src_ip"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 10774093.281829244, "covered_min_N": 5387046.640914622, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 32322279.84548773, "covered_min_N": 16161139.922743864, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 21, "covered_max_N": 8952441.571673146, "covered_min_N": 314412.8382575546, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 1200000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 1200000.0, "min_covered_share": 0.010480427941918488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_service", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 21, "covered_max_N": 26857324.715019435, "covered_min_N": 943238.5147726639, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_service/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 3600000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 3600000.0, "min_covered_share": 0.010480427941918488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_service", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 23, "covered_max_N": 3215148.0197777487, "covered_min_N": 316062.76837714465, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.010535425612571488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region,service", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 23, "covered_max_N": 9645444.059333246, "covered_min_N": 948188.3051314339, "covers_share": 0.01, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t14_distinct_users_by_region,service/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.010535425612571488, "queries_per_instance": 1, "query_id": "t14_distinct_users_by_region,service", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2671540.3364740512, "covered_min_N": 2671540.3364740512, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 0.08905134454913503, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 8014621.009422153, "covered_min_N": 8014621.009422153, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 0.08905134454913503, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 21073170.731707316, "covered_min_N": 2341463.414634146, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.7024390243902439, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.07804878048780486, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 3, "covered_max_N": 63219512.195121944, "covered_min_N": 7024390.243902437, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.7024390243902439, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.07804878048780486, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 959447.493044697, "covered_min_N": 959447.493044697, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2878342.479134091, "covered_min_N": 2878342.479134091, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 5, "covered_max_N": 7568143.5735776145, "covered_min_N": 1892035.8933944036, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 16, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 1875000.0, "max_covered_share": 0.25227145245258714, "metric": "http", "min_N": 1875000.0, "min_covered_share": 0.06306786311314679, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 5, "covered_max_N": 22704430.72073284, "covered_min_N": 5676107.68018321, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 16, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 5625000.0, "max_covered_share": 0.25227145245258714, "metric": "http", "min_N": 5625000.0, "min_covered_share": 0.06306786311314679, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 6288544.323516745, "covered_min_N": 1572136.0808791863, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.20961814411722485, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.05240453602930621, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 4, "covered_max_N": 18865632.970550235, "covered_min_N": 4716408.242637559, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.20961814411722485, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.05240453602930621, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 1876594.1875720161, "covered_min_N": 1876594.1875720161, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 5629782.562716048, "covered_min_N": 5629782.562716048, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 959447.493044697, "covered_min_N": 959447.493044697, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2878342.479134091, "covered_min_N": 2878342.479134091, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.0319815831014899, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 2, "covered_max_N": 2258445.438282906, "covered_min_N": 1596962.084349668, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 400, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 75000.0, "max_covered_share": 0.07528151460943021, "metric": "http", "min_N": 75000.0, "min_covered_share": 0.053232069478322264, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 2, "covered_max_N": 6775336.314848719, "covered_min_N": 4790886.253049004, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 400, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 225000.0, "max_covered_share": 0.07528151460943021, "metric": "http", "min_N": 225000.0, "min_covered_share": 0.053232069478322264, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 673953.3609679822, "covered_min_N": 673953.3609679822, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 3000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 3000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2021860.0829039467, "covered_min_N": 2021860.0829039467, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 9000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 9000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 1876594.1875720161, "covered_min_N": 1876594.1875720161, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 12000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 12000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 5629782.562716048, "covered_min_N": 5629782.562716048, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_service,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.06255313958573387, "metric": "http", "min_N": 36000.0, "min_covered_share": 0.06255313958573387, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_service,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 673953.3609679822, "covered_min_N": 673953.3609679822, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 3000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 3000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint,status", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 1, "covered_max_N": 2021860.0829039467, "covered_min_N": 2021860.0829039467, "covers_share": 0.05, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["region", "service", "endpoint", "status"], "id": "synthetic/templates=all/shared=1/t15_distinct_users_by_region,service,endpoint,status/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 10000, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 9000.0, "max_covered_share": 0.022465112032266074, "metric": "http", "min_N": 9000.0, "min_covered_share": 0.022465112032266074, "queries_per_instance": 1, "query_id": "t15_distinct_users_by_region,service,endpoint,status", "range": "15m", "stream": "http/user_id"}, {"assumptions": [], "capability": "quantile", "covered_groups": 4, "covered_max_N": 10774093.281829244, "covered_min_N": 5387046.640914622, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": null}, "lookback_secs": 300, "max_N": 7500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 7500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 4, "covered_max_N": 32322279.84548773, "covered_min_N": 16161139.922743864, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 4, "keys_per_window": null}, "lookback_secs": 900, "max_N": 22500000.0, "max_covered_share": 0.35913644272764145, "metric": "http", "min_N": 22500000.0, "min_covered_share": 0.17956822136382072, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 25, "covered_max_N": 8952441.571673146, "covered_min_N": 259541.9036564592, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": null}, "lookback_secs": 300, "max_N": 1200000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 1200000.0, "min_covered_share": 0.00865139678854864, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 25, "covered_max_N": 26857324.715019435, "covered_min_N": 778625.7109693777, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 25, "keys_per_window": null}, "lookback_secs": 900, "max_N": 3600000.0, "max_covered_share": 0.2984147190557715, "metric": "http", "min_N": 3600000.0, "min_covered_share": 0.00865139678854864, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_endpoint", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 100, "covered_max_N": 3215148.0197777487, "covered_min_N": 46605.478008970495, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": null}, "lookback_secs": 300, "max_N": 300000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 300000.0, "min_covered_share": 0.00155351593363235, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 100, "covered_max_N": 9645444.059333246, "covered_min_N": 139816.4340269115, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 100, "keys_per_window": null}, "lookback_secs": 900, "max_N": 900000.0, "max_covered_share": 0.10717160065925829, "metric": "http", "min_N": 900000.0, "min_covered_share": 0.00155351593363235, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 2500, "covered_max_N": 2878342.479134091, "covered_min_N": 1209.6074483267448, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": null}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 1.3440082759186053e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/5m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service,endpoint", "range": "5m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_service,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": null}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_service,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "quantile", "covered_groups": 2500, "covered_max_N": 2878342.479134091, "covered_min_N": 1209.6074483267448, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "kll", "grid_K": null, "grid_param": 2.0, "sketch": "kll-percall", "target": 0.01}, {"family": "dd", "grid_K": null, "grid_param": 2.0, "sketch": "dd", "target": 0.01}], "grouping": ["region", "service", "endpoint"], "id": "synthetic/templates=all/shared=1/t16_p99_latency_by_region,service,endpoint/values/15m", "instant": false, "interval_secs": 60, "kind": "values", "label_set": {"arrival_rate_per_sec": 100000, "groups": 2500, "keys_per_window": null}, "lookback_secs": 900, "max_N": 36000.0, "max_covered_share": 0.0319815831014899, "metric": "http", "min_N": 36000.0, "min_covered_share": 1.3440082759186053e-05, "queries_per_instance": 1, "query_id": "t16_p99_latency_by_region,service,endpoint", "range": "15m", "stream": "http/latency"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 625, "covered_max_N": 2671540.3364740512, "covered_min_N": 2245.3999917872916, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/5m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 300, "max_N": 48000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 48000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t17_distinct_users_by_service,endpoint", "range": "5m", "stream": "http/user_id"}, {"assumptions": [], "capability": "cardinality", "covered_groups": 625, "covered_max_N": 8014621.009422153, "covered_min_N": 6736.199975361875, "covers_share": null, "dataset": "synthetic/templates=all/shared=1", "families": [{"family": "hll", "grid_K": 1000000, "grid_param": 0.8, "sketch": "hll", "target": 0.02}], "grouping": ["service", "endpoint"], "id": "synthetic/templates=all/shared=1/t17_distinct_users_by_service,endpoint/keys/15m", "instant": false, "interval_secs": 60, "kind": "keys", "label_set": {"arrival_rate_per_sec": 100000, "groups": 625, "keys_per_window": 1000000}, "lookback_secs": 900, "max_N": 144000.0, "max_covered_share": 0.08905134454913503, "metric": "http", "min_N": 144000.0, "min_covered_share": 7.484666639290972e-05, "queries_per_instance": 1, "query_id": "t17_distinct_users_by_service,endpoint", "range": "15m", "stream": "http/user_id"}], "schemas": {"flows": {"labels": [{"cardinality": 1000, "name": "dst_subnet", "skew": 1.1}, {"cardinality": 1000, "name": "dst_port", "skew": 1.2}, {"cardinality": 3, "name": "proto", "skew": 0.0}], "rate": 200000, "values": {"src_ip": {"burst": {"keys": 10000, "label": "dst_subnet", "share": 0.05, "value": 999}, "keys": 1000000, "zipf": 0.0}}}, "http": {"labels": [{"cardinality": 4, "name": "region", "skew": 0.5}, {"cardinality": 25, "name": "service", "skew": 1.1}, {"cardinality": 25, "child_of": "service", "name": "endpoint", "skew": 1.1}, {"cardinality": 4, "name": "status", "skew": 2.0}], "rate": 100000, "values": {"latency": {"anomaly": {"factor": 10, "label": "endpoint"}, "pareto": 2.0, "scaled_by": "service"}, "user_id": {"keys": 1000000, "zipf": 0.8}}}}}]} diff --git a/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz new file mode 100644 index 00000000..2d3a7bf2 Binary files /dev/null and b/asap-planner-rs/tests/test_data/saved_plan/synthetic-templatesall-shared1.plans.json.gz differ diff --git a/asap-query-engine/src/engines/simple_engine/mod.rs b/asap-query-engine/src/engines/simple_engine/mod.rs index 8514abfc..e2f8990f 100644 --- a/asap-query-engine/src/engines/simple_engine/mod.rs +++ b/asap-query-engine/src/engines/simple_engine/mod.rs @@ -5180,6 +5180,7 @@ mod stage_e4_instant_wrapper_equivalence_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -5438,6 +5439,7 @@ mod stage_e4_instant_wrapper_equivalence_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -5586,6 +5588,7 @@ mod stage_e4_instant_wrapper_equivalence_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -5668,6 +5671,7 @@ mod stage_e4_instant_wrapper_equivalence_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -5792,6 +5796,7 @@ mod stage_e4_instant_wrapper_equivalence_tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/src/engines/simple_engine/promql.rs b/asap-query-engine/src/engines/simple_engine/promql.rs index 22ddc621..0cdabfd3 100644 --- a/asap-query-engine/src/engines/simple_engine/promql.rs +++ b/asap-query-engine/src/engines/simple_engine/promql.rs @@ -1666,6 +1666,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { @@ -1882,6 +1883,7 @@ mod topk_pipeline_tests { .add_aggregation(AggregationReference::new(AGG_ID, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let context = engine @@ -1951,6 +1953,7 @@ mod topk_pipeline_tests { ) .add_aggregation(AggregationReference::new(AGG_ID, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); assert!(matches!( @@ -2042,6 +2045,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { diff --git a/asap-query-engine/src/engines/simple_engine/sql.rs b/asap-query-engine/src/engines/simple_engine/sql.rs index 55616788..953bf533 100644 --- a/asap-query-engine/src/engines/simple_engine/sql.rs +++ b/asap-query-engine/src/engines/simple_engine/sql.rs @@ -1052,6 +1052,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { @@ -1143,6 +1144,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { @@ -1221,6 +1223,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { @@ -1307,6 +1310,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // sub_type "sum" ⇒ the heap is weighted by the summed value rather @@ -1439,6 +1443,7 @@ mod topk_pipeline_tests { schema: SchemaConfig::SQL(netflow_sql_schema()), query_configs: vec![], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( store, @@ -1815,6 +1820,7 @@ mod spatiotemporal_timestamp_alignment_tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let agg_config = AggregationConfig { diff --git a/asap-query-engine/src/tests/capability_matching_tests.rs b/asap-query-engine/src/tests/capability_matching_tests.rs index 9311b45e..c8964f09 100644 --- a/asap-query-engine/src/tests/capability_matching_tests.rs +++ b/asap-query-engine/src/tests/capability_matching_tests.rs @@ -96,6 +96,7 @@ fn engine_no_query_configs( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![], // intentionally empty — forces capability matching cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -143,6 +144,7 @@ fn engine_with_query_config( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/exact_window_grid_adversarial_tests.rs b/asap-query-engine/src/tests/exact_window_grid_adversarial_tests.rs index 6b4ef30f..e077bd66 100644 --- a/asap-query-engine/src/tests/exact_window_grid_adversarial_tests.rs +++ b/asap-query-engine/src/tests/exact_window_grid_adversarial_tests.rs @@ -862,6 +862,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/native_range_query_tests.rs b/asap-query-engine/src/tests/native_range_query_tests.rs index 209426b3..cb54f332 100644 --- a/asap-query-engine/src/tests/native_range_query_tests.rs +++ b/asap-query-engine/src/tests/native_range_query_tests.rs @@ -461,6 +461,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -586,6 +587,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -2262,6 +2264,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -2577,6 +2580,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs b/asap-query-engine/src/tests/prometheus_forwarding_tests.rs index 605bf2d3..d5c6d637 100644 --- a/asap-query-engine/src/tests/prometheus_forwarding_tests.rs +++ b/asap-query-engine/src/tests/prometheus_forwarding_tests.rs @@ -207,6 +207,7 @@ fn native_range_error_configs() -> (InferenceConfig, Arc) { query_configs: vec![QueryConfig::new("sum(native_metric)".to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; (inference_config, streaming_config) } diff --git a/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs b/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs index 5035f1ba..f5b58515 100644 --- a/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs +++ b/asap-query-engine/src/tests/query_time_aggregation_public_tests.rs @@ -90,6 +90,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((_, result)) = engine .handle_query_promql(query, 1_000.0) @@ -317,6 +318,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); assert!(matches!( @@ -374,6 +376,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); for result in [ @@ -414,6 +417,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); assert!(matches!( @@ -455,6 +459,7 @@ mod tests { ) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); // Prometheus omits an absent grouping label; this native representation diff --git a/asap-query-engine/src/tests/range_query_arithmetic_tests.rs b/asap-query-engine/src/tests/range_query_arithmetic_tests.rs index a18c6497..0fe3337e 100644 --- a/asap-query-engine/src/tests/range_query_arithmetic_tests.rs +++ b/asap-query-engine/src/tests/range_query_arithmetic_tests.rs @@ -108,6 +108,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -408,6 +409,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -531,6 +533,7 @@ mod tests { .add_aggregation(AggregationReference::new(2, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/rollup_tests.rs b/asap-query-engine/src/tests/rollup_tests.rs index 62d3cf11..4b96eeaf 100644 --- a/asap-query-engine/src/tests/rollup_tests.rs +++ b/asap-query-engine/src/tests/rollup_tests.rs @@ -322,6 +322,7 @@ mod tests { query_configs: vec![QueryConfig::new(sql.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -417,6 +418,7 @@ mod tests { query_configs: vec![QueryConfig::new(sql.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/sql_nested_aggregation_tests.rs b/asap-query-engine/src/tests/sql_nested_aggregation_tests.rs index f2e04861..7db6121f 100644 --- a/asap-query-engine/src/tests/sql_nested_aggregation_tests.rs +++ b/asap-query-engine/src/tests/sql_nested_aggregation_tests.rs @@ -106,6 +106,7 @@ mod tests { )), query_configs: vec![config.add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs index 8f8d135c..514838bf 100644 --- a/asap-query-engine/src/tests/sql_pattern_matching_tests.rs +++ b/asap-query-engine/src/tests/sql_pattern_matching_tests.rs @@ -53,6 +53,7 @@ mod tests { schema: SchemaConfig::SQL(sql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Streaming config diff --git a/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs b/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs index 510ba17c..6815e7c6 100644 --- a/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs +++ b/asap-query-engine/src/tests/stage_e_instant_range_equivalence_tests.rs @@ -254,6 +254,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -466,6 +467,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( store, diff --git a/asap-query-engine/src/tests/structural_matching_tests.rs b/asap-query-engine/src/tests/structural_matching_tests.rs index 10ca99eb..dcf10629 100644 --- a/asap-query-engine/src/tests/structural_matching_tests.rs +++ b/asap-query-engine/src/tests/structural_matching_tests.rs @@ -82,6 +82,7 @@ mod tests { )), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((labels, result)) = engine diff --git a/asap-query-engine/src/tests/test_utilities/config_builders.rs b/asap-query-engine/src/tests/test_utilities/config_builders.rs index d432c204..7b358a21 100644 --- a/asap-query-engine/src/tests/test_utilities/config_builders.rs +++ b/asap-query-engine/src/tests/test_utilities/config_builders.rs @@ -206,6 +206,7 @@ impl TestConfigBuilder { schema: SchemaConfig::PromQL(promql_schema), query_configs: self.query_configs, cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create StreamingConfig @@ -231,6 +232,7 @@ impl TestConfigBuilder { schema: SchemaConfig::PromQL(promql_schema), query_configs: self.query_configs.clone(), cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create SQL InferenceConfig @@ -247,6 +249,7 @@ impl TestConfigBuilder { schema: SchemaConfig::SQL(sql_schema), query_configs: self.query_configs, cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; // Create StreamingConfig diff --git a/asap-query-engine/src/tests/test_utilities/engine_factories.rs b/asap-query-engine/src/tests/test_utilities/engine_factories.rs index 6433dc03..85f9bb13 100644 --- a/asap-query-engine/src/tests/test_utilities/engine_factories.rs +++ b/asap-query-engine/src/tests/test_utilities/engine_factories.rs @@ -128,6 +128,7 @@ pub fn create_engine_single_pop_with_aggregated( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -258,6 +259,7 @@ pub fn create_engine_dual_input( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -374,6 +376,7 @@ pub fn create_engine_two_metrics( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config_a, query_config_b], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -482,6 +485,7 @@ pub fn create_engine_three_metrics( .add_aggregation(AggregationReference::new(3, None)), ], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -556,6 +560,7 @@ pub fn create_engine_multi_timestamp( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -689,6 +694,7 @@ pub fn create_engine_multi_timestamp_with_window( schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( diff --git a/asap-query-engine/src/tests/window_semantics_consistency_tests.rs b/asap-query-engine/src/tests/window_semantics_consistency_tests.rs index 8df32eb3..03bdd871 100644 --- a/asap-query-engine/src/tests/window_semantics_consistency_tests.rs +++ b/asap-query-engine/src/tests/window_semantics_consistency_tests.rs @@ -391,6 +391,7 @@ mod tests { query_configs: vec![QueryConfig::new(query.to_string()) .add_aggregation(AggregationReference::new(1, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -492,6 +493,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -698,6 +700,7 @@ mod tests { schema: SchemaConfig::PromQL(promql_schema), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; SimpleEngine::new( @@ -814,6 +817,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, @@ -922,6 +926,7 @@ mod tests { .add_aggregation(AggregationReference::new(1, None)) .add_aggregation(AggregationReference::new(2, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let engine = SimpleEngine::new( store, diff --git a/asap-query-engine/tests/e2e_precompute_equivalence.rs b/asap-query-engine/tests/e2e_precompute_equivalence.rs index 1a595dc7..e6801c12 100644 --- a/asap-query-engine/tests/e2e_precompute_equivalence.rs +++ b/asap-query-engine/tests/e2e_precompute_equivalence.rs @@ -334,6 +334,7 @@ impl NativeDagScenario<'_> { )), query_configs: vec![query_config], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let query_engine = SimpleEngine::new( store, @@ -929,6 +930,7 @@ async fn e2e_nested_aggregation_operator_matrix_executes_instant_and_range() { ) .add_aggregation(AggregationReference::new(18, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }); let Some((_, instant)) = engine .handle_query_promql(query.clone(), 1.0) @@ -1320,6 +1322,7 @@ async fn e2e_sliding_precompute_outputs_compose_a_wider_query() { query_configs: vec![QueryConfig::new(query.to_string()) .add_aggregation(AggregationReference::new(agg_id, None))], cleanup_policy: CleanupPolicy::NoCleanup, + rqe_configs: Default::default(), }; let query_engine = SimpleEngine::new( store,