From 61fdb613dccc8e27ad469561fce0803227df884a Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 21:47:04 -0600 Subject: [PATCH 01/16] feat(promql): bind canonical queries to native exact kernels --- Cargo.lock | 10 +- Cargo.toml | 8 +- control_plane/src/physical/mod.rs | 2 + control_plane/src/physical/promql_exact.rs | 382 ++++++++++++ .../query_engines/canonical/exact_promql.rs | 579 ++++++++++++++++++ data_plane/src/query_engines/canonical/mod.rs | 2 + 6 files changed, 974 insertions(+), 9 deletions(-) create mode 100644 control_plane/src/physical/promql_exact.rs create mode 100644 data_plane/src/query_engines/canonical/exact_promql.rs diff --git a/Cargo.lock b/Cargo.lock index e173aa68..7e1a22ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,7 +364,7 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=029ff2fe041172c94c2d32c90b185bc83c5e8a57#029ff2fe041172c94c2d32c90b185bc83c5e8a57" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" dependencies = [ "asap-types", "serde", @@ -375,7 +375,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=029ff2fe041172c94c2d32c90b185bc83c5e8a57#029ff2fe041172c94c2d32c90b185bc83c5e8a57" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -384,7 +384,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=029ff2fe041172c94c2d32c90b185bc83c5e8a57#029ff2fe041172c94c2d32c90b185bc83c5e8a57" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -407,12 +407,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=029ff2fe041172c94c2d32c90b185bc83c5e8a57#029ff2fe041172c94c2d32c90b185bc83c5e8a57" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=029ff2fe041172c94c2d32c90b185bc83c5e8a57#029ff2fe041172c94c2d32c90b185bc83c5e8a57" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" dependencies = [ "serde", "serde_json", diff --git a/Cargo.toml b/Cargo.toml index ae7e90f8..0165c059 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,10 +20,10 @@ asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch [workspace.dependencies] # Keep Planner frontends, selection, and IR on the same immutable revision (current-series Planner PR). # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "029ff2fe041172c94c2d32c90b185bc83c5e8a57" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "029ff2fe041172c94c2d32c90b185bc83c5e8a57" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "029ff2fe041172c94c2d32c90b185bc83c5e8a57" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "029ff2fe041172c94c2d32c90b185bc83c5e8a57" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index a31de4e5..8ae2ca9b 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -16,3 +16,5 @@ pub mod workload_cost; pub mod publication; pub(crate) mod maintained_population; + +pub mod promql_exact; diff --git a/control_plane/src/physical/promql_exact.rs b/control_plane/src/physical/promql_exact.rs new file mode 100644 index 00000000..92ae2e56 --- /dev/null +++ b/control_plane/src/physical/promql_exact.rs @@ -0,0 +1,382 @@ +//! Executable exact kernels for the PromQL float-sample surface. +//! +//! Summary binding cannot implement ordered-window reducers or label-producing +//! operators with the existing five accumulator families. This plan binds each +//! operator to a backend kernel and explicitly requires raw timestamped samples. +use std::rc::Rc; + +use planner_types::pre_asap::{ + AggIntent, CompareOpKind, GroupKeys, QueryExpr, Reduction, SampleKind, ScalarValue, Source, + VectorMatchKind, +}; +use planner_types::types::AccuracyTarget; +use promql_parser::label::Matcher; +use promql_parser::parser::token; + +macro_rules! kernels { + ($name:ident { $($variant:ident => $text:literal),+ $(,)? }) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum $name { $($variant),+ } + impl std::str::FromStr for $name { + type Err = anyhow::Error; + fn from_str(name: &str) -> anyhow::Result { + match name { $($text => Ok(Self::$variant),)+ _ => anyhow::bail!("no exact kernel for {name}") } + } + } + } +} +kernels!(AggregateKernel { + Sum => "sum", Avg => "avg", Count => "count", Min => "min", Max => "max", + Group => "group", Stddev => "stddev", Stdvar => "stdvar", TopK => "topk", + BottomK => "bottomk", CountValues => "count_values", Quantile => "quantile", + LimitK => "limitk", LimitRatio => "limit_ratio", +}); +kernels!(RangeKernel { + Avg => "avg_over_time", Min => "min_over_time", Max => "max_over_time", + Sum => "sum_over_time", Count => "count_over_time", Quantile => "quantile_over_time", + Stddev => "stddev_over_time", Stdvar => "stdvar_over_time", Last => "last_over_time", + Present => "present_over_time", Absent => "absent_over_time", Changes => "changes", + Delta => "delta", Deriv => "deriv", IDelta => "idelta", Increase => "increase", + IRate => "irate", PredictLinear => "predict_linear", Rate => "rate", Resets => "resets", + Smoothing => "double_exponential_smoothing", Mad => "mad_over_time", + TsMin => "ts_of_min_over_time", TsMax => "ts_of_max_over_time", TsLast => "ts_of_last_over_time", +}); +kernels!(BinaryKernel { + Add => "+", Sub => "-", Mul => "*", Div => "/", Mod => "%", Pow => "^", + And => "and", Or => "or", Unless => "unless", +}); + +#[derive(Debug, Clone)] +pub struct Grouping { + pub labels: Vec, + pub without: bool, +} + +#[derive(Debug, Clone)] +pub struct ExactSelector { + pub metric: String, + pub matchers: Vec, +} + +#[derive(Debug, Clone)] +pub enum ExactExpr { + Scalar(f64), + Select { + selector: ExactSelector, + range_seconds: Option, + }, + Aggregate { + kernel: AggregateKernel, + parameter: Option, + label: Option, + grouping: Grouping, + input: Box, + }, + Range { + kernel: RangeKernel, + parameters: Vec, + input: Box, + }, + Binary { + kernel: BinaryKernel, + lhs: Box, + rhs: Box, + }, +} + +#[derive(Debug, Clone)] +pub struct ExactPromqlPlan { + /// The canonical frontend result is retained for review and semantic regression checks. + pub canonical: Rc, + root: ExactExpr, +} + +impl ExactPromqlPlan { + pub fn bind(query: &str) -> anyhow::Result { + let canonical = Rc::new(crate::query_parser::parse_query_expr_canonical( + query, + AccuracyTarget::Exact, + )?); + Self::from_canonical(canonical) + } + + /// Bind the planner's canonical tree; execution never reparses the query text. + pub fn from_canonical(canonical: Rc) -> anyhow::Result { + let root = bind_expr(&canonical)?; + anyhow::ensure!( + !matches!( + &root, + ExactExpr::Scalar(_) + | ExactExpr::Select { + range_seconds: Some(_), + .. + } + ), + "exact endpoint requires an instant-vector result" + ); + Ok(Self { canonical, root }) + } + + pub fn root(&self) -> &ExactExpr { + &self.root + } +} + +fn grouping(keys: &GroupKeys, child: &QueryExpr) -> anyhow::Result { + let schema = child.output_schema()?; + let labels = keys + .keys() + .iter() + .map(|index| { + let column = schema + .columns + .get(*index) + .ok_or_else(|| anyhow::anyhow!("invalid grouping column {index}"))?; + anyhow::ensure!( + column.name != "ts" && column.name != "value", + "grouping requires label columns" + ); + Ok(column.name.clone()) + }) + .collect::>>()?; + Ok(Grouping { + labels, + without: keys.is_without(), + }) +} + +fn bind_expr(expr: &QueryExpr) -> anyhow::Result { + match expr { + QueryExpr::PromqlScalarBridge(child) => bind_expr(child), + QueryExpr::Literal(ScalarValue::Float64(value)) => Ok(ExactExpr::Scalar(*value)), + QueryExpr::Literal(ScalarValue::Int64(value)) => Ok(ExactExpr::Scalar(*value as f64)), + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + predicates, + schema, + } => { + let mut matchers = Vec::new(); + for predicate in predicates { + let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { + anyhow::bail!("unsupported exact scan predicate") + }; + let (QueryExpr::Column(index), QueryExpr::Literal(ScalarValue::Utf8(value))) = + (left.as_ref(), right.as_ref()) + else { + anyhow::bail!("exact scan requires literal label matchers") + }; + let column = schema + .columns + .get(*index) + .ok_or_else(|| anyhow::anyhow!("invalid matcher column"))?; + let token = match op { + CompareOpKind::Eq => token::T_EQL, + CompareOpKind::Ne => token::T_NEQ, + CompareOpKind::Regex => token::T_EQL_REGEX, + CompareOpKind::NotRegex => token::T_NEQ_REGEX, + _ => anyhow::bail!("unsupported label comparison"), + }; + matchers.push( + Matcher::new_matcher(token, column.name.clone(), value.clone()) + .map_err(anyhow::Error::msg)?, + ); + } + Ok(ExactExpr::Select { + selector: ExactSelector { + metric: metric.clone(), + matchers, + }, + range_seconds: None, + }) + } + QueryExpr::TimeRange { range, child } => { + let mut input = bind_expr(child)?; + let ExactExpr::Select { range_seconds, .. } = &mut input else { + anyhow::bail!("exact range currently requires a raw selector") + }; + anyhow::ensure!( + range_seconds.is_none(), + "nested raw ranges are not supported" + ); + *range_seconds = Some(range.as_secs_f64()); + Ok(input) + } + QueryExpr::Aggregate { + reduction, + measures, + having: None, + child, + .. + } if measures.len() == 1 => { + let intent = &measures[0]; + let input = Box::new(bind_expr(child)?); + match reduction { + Reduction::PerEntity => { + anyhow::ensure!( + matches!( + input.as_ref(), + ExactExpr::Select { + range_seconds: Some(_), + .. + } + ), + "exact rollup needs a raw range input" + ); + let (kernel, parameters) = range_kernel(intent)?; + Ok(ExactExpr::Range { + kernel, + parameters, + input, + }) + } + Reduction::Reduce(keys) => { + let (kernel, parameter, label) = aggregate_kernel(intent)?; + Ok(ExactExpr::Aggregate { + kernel, + parameter, + label, + grouping: grouping(keys, child)?, + input, + }) + } + } + } + QueryExpr::Limit { + n, + offset: 0, + child, + } => { + let QueryExpr::Sort { + keys, + partition_by, + child: input, + } = child.as_ref() + else { + anyhow::bail!("limit requires a bound value sort") + }; + anyhow::ensure!(keys.len() == 1, "exact topk requires one value sort key"); + let QueryExpr::Column(index) = keys[0].expr else { + anyhow::bail!("topk must sort sample values") + }; + anyhow::ensure!( + input + .output_schema()? + .columns + .get(index) + .is_some_and(|c| c.name == "value"), + "topk must sort sample values" + ); + Ok(ExactExpr::Aggregate { + kernel: if keys[0].ascending { + AggregateKernel::BottomK + } else { + AggregateKernel::TopK + }, + parameter: Some(*n as f64), + label: None, + grouping: grouping(partition_by, input)?, + input: Box::new(bind_expr(input)?), + }) + } + QueryExpr::PromqlSeriesSample { by, kind, child } => { + let (kernel, parameter) = match kind { + SampleKind::LimitK(k) => (AggregateKernel::LimitK, *k as f64), + SampleKind::LimitRatio(r) => (AggregateKernel::LimitRatio, *r), + }; + Ok(ExactExpr::Aggregate { + kernel, + parameter: Some(parameter), + label: None, + grouping: grouping(by, child)?, + input: Box::new(bind_expr(child)?), + }) + } + QueryExpr::BinaryOp { + op, + lhs, + rhs, + vector_match, + } => { + if let Some(m) = vector_match { + anyhow::ensure!( + m.kind == VectorMatchKind::Ignoring + && m.labels.is_empty() + && m.grouping.is_none(), + "exact kernel does not implement explicit vector matching" + ); + } + Ok(ExactExpr::Binary { + kernel: op.to_string().to_lowercase().parse()?, + lhs: Box::new(bind_expr(lhs)?), + rhs: Box::new(bind_expr(rhs)?), + }) + } + _ => anyhow::bail!("no executable exact kernel for canonical node {expr:?}"), + } +} + +fn aggregate_kernel( + intent: &AggIntent, +) -> anyhow::Result<(AggregateKernel, Option, Option)> { + use AggregateKernel as K; + let (kernel, parameter, label) = match intent { + AggIntent::Sum { col: None } => (K::Sum, None, None), + AggIntent::Avg { col: None } => (K::Avg, None, None), + AggIntent::Count { .. } => (K::Count, None, None), + AggIntent::Min { col: None } => (K::Min, None, None), + AggIntent::Max { col: None } => (K::Max, None, None), + AggIntent::Group => (K::Group, None, None), + AggIntent::StdDev { + population: true, + col: None, + } => (K::Stddev, None, None), + AggIntent::Variance { + population: true, + col: None, + } => (K::Stdvar, None, None), + AggIntent::Quantile { q, col: None, .. } => (K::Quantile, Some(*q), None), + AggIntent::CountValues { label } => (K::CountValues, None, Some(label.clone())), + _ => anyhow::bail!("no exact aggregation kernel for {intent:?}"), + }; + Ok((kernel, parameter, label)) +} + +fn range_kernel(intent: &AggIntent) -> anyhow::Result<(RangeKernel, Vec)> { + use RangeKernel as K; + Ok(match intent { + AggIntent::Sum { col: None } => (K::Sum, vec![]), + AggIntent::Avg { col: None } => (K::Avg, vec![]), + AggIntent::Count { .. } => (K::Count, vec![]), + AggIntent::Min { col: None } => (K::Min, vec![]), + AggIntent::Max { col: None } => (K::Max, vec![]), + AggIntent::StdDev { + population: true, + col: None, + } => (K::Stddev, vec![]), + AggIntent::Variance { + population: true, + col: None, + } => (K::Stdvar, vec![]), + AggIntent::Quantile { q, col: None, .. } => (K::Quantile, vec![*q]), + AggIntent::LastOverTime => (K::Last, vec![]), + AggIntent::PresentOverTime => (K::Present, vec![]), + AggIntent::AbsentOverTime => (K::Absent, vec![]), + AggIntent::Changes => (K::Changes, vec![]), + AggIntent::Delta => (K::Delta, vec![]), + AggIntent::Deriv => (K::Deriv, vec![]), + AggIntent::IDelta => (K::IDelta, vec![]), + AggIntent::Increase => (K::Increase, vec![]), + AggIntent::IRate => (K::IRate, vec![]), + AggIntent::Rate => (K::Rate, vec![]), + AggIntent::Resets => (K::Resets, vec![]), + AggIntent::PredictLinear { seconds } => (K::PredictLinear, vec![*seconds]), + AggIntent::DoubleExpSmoothing { smoothing, trend } => { + (K::Smoothing, vec![*smoothing, *trend]) + } + AggIntent::MadOverTime => (K::Mad, vec![]), + AggIntent::TsOfMinOverTime => (K::TsMin, vec![]), + AggIntent::TsOfMaxOverTime => (K::TsMax, vec![]), + AggIntent::TsOfLastOverTime => (K::TsLast, vec![]), + _ => anyhow::bail!("no exact range kernel for {intent:?}"), + }) +} diff --git a/data_plane/src/query_engines/canonical/exact_promql.rs b/data_plane/src/query_engines/canonical/exact_promql.rs new file mode 100644 index 00000000..4e81f446 --- /dev/null +++ b/data_plane/src/query_engines/canonical/exact_promql.rs @@ -0,0 +1,579 @@ +//! Execute the control plane's bound exact PromQL kernels over raw float samples. +//! No query is forwarded and no sketch value is substituted for a raw observation. +use std::collections::{BTreeMap, BTreeSet}; + +use control_plane::physical::promql_exact::{ + AggregateKernel as A, BinaryKernel as B, ExactExpr, ExactPromqlPlan, Grouping, RangeKernel as R, +}; +use serde::{Deserialize, Serialize}; + +pub type Labels = BTreeMap; + +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct RawSeries { + pub labels: Labels, + /// Unix seconds, strictly increasing; missing samples are absent from this list. + pub samples: Vec<(f64, f64)>, +} + +#[derive(Debug, Clone)] +pub struct ExactSample { + pub labels: Labels, + pub value: f64, +} + +enum Value { + Scalar(f64), + Vector(Vec), + Matrix { + series: Vec, + seconds: f64, + }, +} +impl Value { + fn vector(self) -> anyhow::Result> { + match self { + Self::Vector(v) => Ok(v), + _ => anyhow::bail!("expected an instant vector"), + } + } +} + +/// Evaluate only the bound program. Raw data must already be a consistent snapshot. +pub fn execute( + plan: &ExactPromqlPlan, + data: &[RawSeries], + evaluation: f64, + lookback: f64, +) -> anyhow::Result> { + anyhow::ensure!( + evaluation.is_finite() && lookback.is_finite() && lookback > 0.0, + "invalid evaluation time/lookback" + ); + let mut seen = BTreeSet::new(); + for series in data { + anyhow::ensure!(seen.insert(&series.labels), "duplicate input label set"); + anyhow::ensure!( + series.samples.iter().all(|(t, _)| t.is_finite()) + && series.samples.windows(2).all(|p| p[0].0 < p[1].0), + "raw samples must have unique increasing finite timestamps" + ); + } + let out = eval(plan.root(), data, evaluation, lookback)?.vector()?; + let mut seen = BTreeSet::new(); + anyhow::ensure!( + out.iter().all(|s| seen.insert(&s.labels)), + "duplicate output label set" + ); + Ok(out) +} + +fn eval(expr: &ExactExpr, data: &[RawSeries], time: f64, lookback: f64) -> anyhow::Result { + match expr { + ExactExpr::Scalar(value) => Ok(Value::Scalar(*value)), + ExactExpr::Select { + selector, + range_seconds, + } => { + let mut selected = Vec::new(); + for series in data { + if series.labels.get("__name__") != Some(&selector.metric) { + continue; + } + if !selector.matchers.iter().all(|m| { + m.is_match(series.labels.get(&m.name).map(String::as_str).unwrap_or("")) + }) { + continue; + } + let seconds = range_seconds.unwrap_or(lookback); + let mut samples: Vec<_> = series + .samples + .iter() + .copied() + .filter(|(t, _)| *t > time - seconds && *t <= time) + .collect(); + if range_seconds.is_none() && !samples.is_empty() { + samples = vec![*samples.last().unwrap()]; + } + if !samples.is_empty() { + selected.push(RawSeries { + labels: series.labels.clone(), + samples, + }); + } + } + selected.sort_by(|a, b| a.labels.cmp(&b.labels)); + Ok(match range_seconds { + Some(seconds) => Value::Matrix { + series: selected, + seconds: *seconds, + }, + None => Value::Vector( + selected + .into_iter() + .map(|s| ExactSample { + labels: s.labels, + value: s.samples[0].1, + }) + .collect(), + ), + }) + } + ExactExpr::Aggregate { + kernel, + parameter, + label, + grouping, + input, + } => { + let rows = eval(input, data, time, lookback)?.vector()?; + Ok(Value::Vector(aggregate( + *kernel, + *parameter, + label.as_deref(), + grouping, + rows, + )?)) + } + ExactExpr::Range { + kernel, + parameters, + input, + } => { + let Value::Matrix { series, seconds } = eval(input, data, time, lookback)? else { + anyhow::bail!("range kernel needs raw range samples") + }; + if *kernel == R::Absent { + let labels = absent_labels(input); + return Ok(Value::Vector(if series.is_empty() { + vec![ExactSample { labels, value: 1.0 }] + } else { + vec![] + })); + } + let mut out = Vec::new(); + for series in series { + if let Some(value) = rollup(*kernel, parameters, &series.samples, time, seconds)? { + let mut labels = series.labels; + if *kernel != R::Last { + labels.remove("__name__"); + } + out.push(ExactSample { labels, value }); + } + } + Ok(Value::Vector(out)) + } + ExactExpr::Binary { kernel, lhs, rhs } => binary( + *kernel, + eval(lhs, data, time, lookback)?, + eval(rhs, data, time, lookback)?, + ), + } +} + +fn absent_labels(input: &ExactExpr) -> Labels { + let mut labels = Labels::new(); + if let ExactExpr::Select { selector, .. } = input { + // Derive a label only when it has a single equality matcher. + let mut counts = BTreeMap::new(); + for m in &selector.matchers { + *counts.entry(&m.name).or_insert(0) += 1; + } + for m in &selector.matchers { + if m.name != "__name__" + && m.op.to_string() == "=" + && counts[&m.name] == 1 + && !m.value.is_empty() + { + labels.insert(m.name.clone(), m.value.clone()); + } + } + } + labels +} + +fn group_key(labels: &Labels, grouping: &Grouping) -> Labels { + labels + .iter() + .filter(|(key, _)| { + let included = grouping.labels.contains(key); + if grouping.without { + key.as_str() != "__name__" && !included + } else { + included + } + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect() +} + +fn aggregate( + kernel: A, + parameter: Option, + label: Option<&str>, + grouping: &Grouping, + rows: Vec, +) -> anyhow::Result> { + let mut groups: BTreeMap> = BTreeMap::new(); + for mut row in rows { + let mut key = group_key(&row.labels, grouping); + if kernel == A::CountValues { + let label = label.ok_or_else(|| anyhow::anyhow!("count_values requires a label"))?; + // The output value label participates in grouping even with no 'by'. + row.labels.insert(label.into(), float_label(row.value)); + key.insert(label.into(), float_label(row.value)); + } + groups.entry(key).or_default().push(row); + } + let mut out = Vec::new(); + for (labels, mut rows) in groups { + match kernel { + A::TopK | A::BottomK | A::LimitK => { + let k = parameter.unwrap_or(0.0); + anyhow::ensure!(k.is_finite() && k >= 0.0, "invalid selection count"); + if kernel != A::LimitK { + rows.sort_by(|a, b| { + if a.value.is_nan() { + return if b.value.is_nan() { + std::cmp::Ordering::Equal + } else { + std::cmp::Ordering::Greater + }; + } + if b.value.is_nan() { + return std::cmp::Ordering::Less; + } + if kernel == A::TopK { + b.value.total_cmp(&a.value) + } else { + a.value.total_cmp(&b.value) + } + }); + } + out.extend(rows.into_iter().take(k as usize)); + } + A::LimitRatio => { + let ratio = parameter.unwrap_or(0.0).clamp(-1.0, 1.0); + anyhow::ensure!(ratio.is_finite(), "invalid sampling ratio"); + out.extend(rows.into_iter().filter(|row| { + let mut bytes = Vec::new(); + for (key, value) in &row.labels { + bytes.extend(key.as_bytes()); + bytes.push(255); + bytes.extend(value.as_bytes()); + bytes.push(255); + } + let offset = xxhash_rust::xxh64::xxh64(&bytes, 0) as f64 / u64::MAX as f64; + if ratio < 0.0 { + offset >= 1.0 + ratio + } else { + offset < ratio + } + })); + } + _ => { + let values: Vec<_> = rows.iter().map(|s| s.value).collect(); + let value = match kernel { + A::Sum => values.iter().sum(), + A::Avg => mean(&values), + A::Count | A::CountValues => values.len() as f64, + A::Min => minimum(&values), + A::Max => maximum(&values), + A::Group => 1.0, + A::Stdvar => variance(&values), + A::Stddev => variance(&values).sqrt(), + A::Quantile => quantile(&values, parameter.unwrap_or(f64::NAN)), + _ => unreachable!(), + }; + out.push(ExactSample { labels, value }); + } + } + } + Ok(out) +} + +fn float_label(value: f64) -> String { + if value.is_nan() { + "NaN".into() + } else if value == f64::INFINITY { + "+Inf".into() + } else if value == f64::NEG_INFINITY { + "-Inf".into() + } else { + value.to_string() + } +} +fn minimum(values: &[f64]) -> f64 { + values.iter().copied().reduce(f64::min).unwrap_or(f64::NAN) +} +fn maximum(values: &[f64]) -> f64 { + values.iter().copied().reduce(f64::max).unwrap_or(f64::NAN) +} +fn mean(values: &[f64]) -> f64 { + values.iter().sum::() / values.len() as f64 +} +fn variance(values: &[f64]) -> f64 { + let mut mean = 0.0; + let mut m2 = 0.0; + for (i, value) in values.iter().enumerate() { + let d = value - mean; + mean += d / (i + 1) as f64; + m2 += d * (value - mean); + } + m2 / values.len() as f64 +} +fn quantile(values: &[f64], phi: f64) -> f64 { + if phi.is_nan() || values.is_empty() { + return f64::NAN; + } + if phi < 0.0 { + return f64::NEG_INFINITY; + } + if phi > 1.0 { + return f64::INFINITY; + } + let mut sorted = values.to_vec(); + sorted.sort_by(|a, b| match (a.is_nan(), b.is_nan()) { + (true, false) => std::cmp::Ordering::Less, + (false, true) => std::cmp::Ordering::Greater, + _ => a.total_cmp(b), + }); + let rank = phi * (sorted.len() - 1) as f64; + let lower = rank.floor() as usize; + let upper = rank.ceil() as usize; + let weight = rank - lower as f64; + sorted[lower] * (1.0 - weight) + sorted[upper] * weight +} + +fn rollup( + kernel: R, + parameters: &[f64], + samples: &[(f64, f64)], + time: f64, + seconds: f64, +) -> anyhow::Result> { + if samples.is_empty() { + return Ok(None); + } + let values: Vec<_> = samples.iter().map(|s| s.1).collect(); + let first = samples[0]; + let last = *samples.last().unwrap(); + let needs_two = matches!( + kernel, + R::Delta + | R::Deriv + | R::IDelta + | R::Increase + | R::IRate + | R::PredictLinear + | R::Rate + | R::Smoothing + ); + if needs_two && samples.len() < 2 { + return Ok(None); + } + Ok(Some(match kernel { + R::Avg => mean(&values), + R::Min => minimum(&values), + R::Max => maximum(&values), + R::Sum => values.iter().sum(), + R::Count => values.len() as f64, + R::Quantile => quantile(&values, parameters[0]), + R::Stddev => variance(&values).sqrt(), + R::Stdvar => variance(&values), + R::Last => last.1, + R::Present => 1.0, + R::Changes => values + .windows(2) + .filter(|p| p[0] != p[1] && !(p[0].is_nan() && p[1].is_nan())) + .count() as f64, + R::Resets => values.windows(2).filter(|p| p[1] < p[0]).count() as f64, + R::IDelta => last.1 - samples[samples.len() - 2].1, + R::IRate => { + let previous = samples[samples.len() - 2]; + let delta = if last.1 < previous.1 { + last.1 + } else { + last.1 - previous.1 + }; + delta / (last.0 - previous.0) + } + R::Rate | R::Increase | R::Delta => { + let counter = kernel != R::Delta; + let mut difference = last.1 - first.1; + if counter { + for pair in values.windows(2) { + if pair[1] < pair[0] { + difference += pair[0]; + } + } + } + let observed = last.0 - first.0; + let interval = observed / (samples.len() - 1) as f64; + let mut before = first.0 - (time - seconds); + let mut after = time - last.0; + if before >= interval * 1.1 { + before = interval / 2.0; + } + if after >= interval * 1.1 { + after = interval / 2.0; + } + if counter && difference > 0.0 && first.1 >= 0.0 { + before = before.min(observed * first.1 / difference); + } + difference * ((observed + before + after) / observed) + / if kernel == R::Rate { seconds } else { 1.0 } + } + R::Deriv | R::PredictLinear => { + // Center timestamps near the window to avoid losing precision on Unix time. + let xs: Vec<_> = samples.iter().map(|s| s.0 - time).collect(); + let mx = mean(&xs); + let my = mean(&values); + let slope = xs + .iter() + .zip(&values) + .map(|(x, y)| (x - mx) * (y - my)) + .sum::() + / xs.iter().map(|x| (x - mx).powi(2)).sum::(); + if kernel == R::Deriv { + slope + } else { + my + slope * (parameters[0] - mx) + } + } + R::Smoothing => { + let (sf, tf) = (parameters[0], parameters[1]); + anyhow::ensure!( + sf > 0.0 && sf < 1.0 && tf > 0.0 && tf < 1.0, + "invalid smoothing/trend factor" + ); + let mut level = first.1; + let mut previous = 0.0; + let mut trend = values[1] - values[0]; + for (i, value) in values.iter().enumerate().skip(1) { + if i > 1 { + trend = tf * (level - previous) + (1.0 - tf) * trend; + } + previous = level; + level = sf * value + (1.0 - sf) * (level + trend); + } + level + } + R::Mad => { + let median = quantile(&values, 0.5); + let deviations: Vec<_> = values.iter().map(|v| (v - median).abs()).collect(); + quantile(&deviations, 0.5) + } + R::TsLast => last.0, + R::TsMin | R::TsMax => { + let mut selected = first; + for sample in samples.iter().copied().skip(1) { + if selected.1.is_nan() + || (kernel == R::TsMin && sample.1 <= selected.1) + || (kernel == R::TsMax && sample.1 >= selected.1) + { + selected = sample; + } + } + selected.0 + } + R::Absent => unreachable!(), + })) +} + +fn arithmetic(kernel: B, a: f64, b: f64) -> anyhow::Result { + Ok(match kernel { + B::Add => a + b, + B::Sub => a - b, + B::Mul => a * b, + B::Div => a / b, + B::Mod => a % b, + B::Pow => a.powf(b), + _ => anyhow::bail!("set operator needs two vectors"), + }) +} +fn matching_key(labels: &Labels) -> Labels { + let mut key = labels.clone(); + key.remove("__name__"); + key +} +fn binary(kernel: B, lhs: Value, rhs: Value) -> anyhow::Result { + let vector = match (lhs, rhs) { + (Value::Scalar(a), Value::Scalar(b)) => { + return Ok(Value::Scalar(arithmetic(kernel, a, b)?)) + } + (Value::Vector(rows), Value::Scalar(scalar)) + | (Value::Scalar(scalar), Value::Vector(rows)) + if matches!(kernel, B::Add | B::Mul) => + { + rows.into_iter() + .map(|mut row| { + row.value = arithmetic(kernel, row.value, scalar)?; + row.labels.remove("__name__"); + Ok(row) + }) + .collect::>>()? + } + (Value::Vector(rows), Value::Scalar(scalar)) => rows + .into_iter() + .map(|mut row| { + row.value = arithmetic(kernel, row.value, scalar)?; + row.labels.remove("__name__"); + Ok(row) + }) + .collect::>>()?, + (Value::Scalar(scalar), Value::Vector(rows)) => rows + .into_iter() + .map(|mut row| { + row.value = arithmetic(kernel, scalar, row.value)?; + row.labels.remove("__name__"); + Ok(row) + }) + .collect::>>()?, + (Value::Vector(left), Value::Vector(right)) => { + let left_keys: BTreeSet<_> = left.iter().map(|s| matching_key(&s.labels)).collect(); + let right_keys: BTreeSet<_> = right.iter().map(|s| matching_key(&s.labels)).collect(); + match kernel { + B::Or => left + .into_iter() + .chain( + right + .into_iter() + .filter(|r| !left_keys.contains(&matching_key(&r.labels))), + ) + .collect(), + B::And => left + .into_iter() + .filter(|s| right_keys.contains(&matching_key(&s.labels))) + .collect(), + B::Unless => left + .into_iter() + .filter(|s| !right_keys.contains(&matching_key(&s.labels))) + .collect(), + _ => { + anyhow::ensure!( + left_keys.len() == left.len() && right_keys.len() == right.len(), + "non-unique vector match" + ); + let right: BTreeMap<_, _> = right + .into_iter() + .map(|s| (matching_key(&s.labels), s.value)) + .collect(); + let mut out = Vec::new(); + for row in left { + let key = matching_key(&row.labels); + if let Some(value) = right.get(&key) { + out.push(ExactSample { + labels: key, + value: arithmetic(kernel, row.value, *value)?, + }); + } + } + out + } + } + } + _ => anyhow::bail!("binary operator cannot consume a range vector"), + }; + Ok(Value::Vector(vector)) +} diff --git a/data_plane/src/query_engines/canonical/mod.rs b/data_plane/src/query_engines/canonical/mod.rs index c170e5d0..67fba694 100644 --- a/data_plane/src/query_engines/canonical/mod.rs +++ b/data_plane/src/query_engines/canonical/mod.rs @@ -20,3 +20,5 @@ pub mod result { pub mod sds_resolver { pub use asap_types::sds::{DataDescriptorId, SummaryDescriptorId}; } + +pub mod exact_promql; From 269143dea9c414d0f7cb6e94b14071a38aba329d Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 21:50:57 -0600 Subject: [PATCH 02/16] test(promql): cover every aggregation and range function with exact smoke cases --- control_plane/examples/promql_smoke.rs | 117 ++ data_plane/examples/promql_exact_smoke.rs | 164 ++ data_plane/tests/promql_exact_execution.rs | 155 ++ tools/promql-smoke/README.md | 157 ++ tools/promql-smoke/RESULTS.md | 57 + tools/promql-smoke/cases.json | 1849 ++++++++++++++++++++ tools/promql-smoke/catalog.json | 174 ++ tools/promql-smoke/run.py | 273 +++ tools/promql-smoke/test_runner.py | 90 + 9 files changed, 3036 insertions(+) create mode 100644 control_plane/examples/promql_smoke.rs create mode 100644 data_plane/examples/promql_exact_smoke.rs create mode 100644 data_plane/tests/promql_exact_execution.rs create mode 100644 tools/promql-smoke/README.md create mode 100644 tools/promql-smoke/RESULTS.md create mode 100644 tools/promql-smoke/cases.json create mode 100644 tools/promql-smoke/catalog.json create mode 100644 tools/promql-smoke/run.py create mode 100644 tools/promql-smoke/test_runner.py diff --git a/control_plane/examples/promql_smoke.rs b/control_plane/examples/promql_smoke.rs new file mode 100644 index 00000000..2976f4ff --- /dev/null +++ b/control_plane/examples/promql_smoke.rs @@ -0,0 +1,117 @@ +//! Run the smoke corpus through the backend's pinned PromQL planner bridge. +//! Binding is a capability diagnostic, not proof of execution or value correctness. +use control_plane::physical::post_asap::{bind_query_expr, PhysicalExpr, PostAsapPlan}; +use control_plane::physical::promql_exact::ExactPromqlPlan; +use control_plane::query_parser::parse_query_expr_canonical; +use planner_types::post_asap::SummaryExpr; +use planner_types::types::AccuracyTarget; +use serde_json::{json, Value}; +use std::collections::BTreeMap; + +fn inspect(query: &Value) -> Value { + let expr = query["expr"].as_str().expect("query expr must be a string"); + let mut row = json!({ + "id": query["id"], "category": query["category"], + "function": query["function"], "experimental": query["experimental"], + "expr": expr, + }); + match parse_query_expr_canonical(expr, AccuracyTarget::Exact) { + Err(error) => { + row["status"] = json!("PARSE_REJECTED"); + row["error"] = json!(error.to_string()); + } + Ok(tree) => { + row["canonical"] = json!(format!("{tree:#?}")); + match bind_query_expr(&tree, AccuracyTarget::Exact) { + Ok(plan) => { + let logical_only = matches!( + &plan, + PhysicalExpr::Committed(PostAsapPlan::Summary(node)) + if matches!(&node.expr, SummaryExpr::KeepPreAsap(_)) + ); + row["status"] = json!(if logical_only { + "LOGICAL_ONLY" + } else { + "BOUND" + }); + row["plan"] = json!(format!("{plan:#?}")); + } + Err(error) => { + row["status"] = json!("BIND_REJECTED"); + row["error"] = json!(error.to_string()); + } + } + } + } + row["summary_status"] = row["status"].take(); + row["summary_plan"] = row["plan"].take(); + row["summary_error"] = row["error"].take(); + match ExactPromqlPlan::bind(expr) { + Ok(plan) => { + row["status"] = json!("BOUND_EXACT"); + row["executor"] = json!("data_plane::query_engines::canonical::exact_promql"); + row["requires"] = json!("raw timestamped float samples"); + row["plan"] = json!(format!("{:#?}", plan.root())); + } + Err(error) => { + row["status"] = json!("EXACT_BIND_REJECTED"); + row["error"] = json!(error.to_string()); + } + } + row +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + let json_output = args.iter().any(|arg| arg == "--json"); + let require_bound = args.iter().any(|arg| arg == "--require-bound"); + for arg in &args { + if arg.starts_with("--") && arg != "--json" && arg != "--require-bound" { + return Err(format!("unknown option: {arg}").into()); + } + } + let path = args + .iter() + .find(|arg| !arg.starts_with("--")) + .map(String::as_str) + .unwrap_or("tools/promql-smoke/cases.json"); + let cases: Value = serde_json::from_slice(&std::fs::read(path)?)?; + let mut results = Vec::new(); + let mut counts = BTreeMap::::new(); + for query in cases["queries"].as_array().ok_or("missing queries")? { + // A panic is a failed case; continue to expose the other unsupported queries. + let row = std::panic::catch_unwind(|| inspect(query)).unwrap_or_else( + |_| json!({"id": query["id"], "expr": query["expr"], "status": "PANIC"}), + ); + let status = row["status"].as_str().expect("case status"); + *counts.entry(status.to_owned()).or_default() += 1; + if !json_output { + println!( + "{status} {}: {}{}", + query["id"].as_str().unwrap_or("?"), + query["expr"].as_str().unwrap_or("?"), + row["error"] + .as_str() + .map(|e| format!(" — {e}")) + .unwrap_or_default(), + ); + } + results.push(row); + } + let has_unbound = results.iter().any(|row| row["status"] != "BOUND_EXACT"); + if json_output { + println!( + "{}", + serde_json::to_string_pretty(&json!({ + "accuracy": "Exact", "scope": "Canonical tree to native exact kernels; run promql_exact_execution for values", + "summary": counts, "queries": results, + }))? + ); + } else { + println!("Summary: {counts:?}; binding does not prove execution or correct values."); + } + if require_bound && has_unbound { + std::process::exit(1); + } + Ok(()) +} diff --git a/data_plane/examples/promql_exact_smoke.rs b/data_plane/examples/promql_exact_smoke.rs new file mode 100644 index 00000000..0ec221db --- /dev/null +++ b/data_plane/examples/promql_exact_smoke.rs @@ -0,0 +1,164 @@ +//! Serve the small raw fixture using real canonical binding and native exact execution. +//! This is a local test server; it does not alter the production store or routing profile. +use axum::{ + extract::{Query, State}, + http::StatusCode, + routing::get, + Json, Router, +}; +use control_plane::physical::promql_exact::ExactPromqlPlan; +use data_plane::query_engines::canonical::exact_promql::{execute, Labels, RawSeries}; +use serde_json::{json, Value}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +struct Snapshot { + data: Vec, + time: f64, +} +type Reply = (StatusCode, Json); +fn error(message: impl ToString) -> Reply { + ( + StatusCode::BAD_REQUEST, + Json(json!({"status":"error","errorType":"bad_data","error":message.to_string()})), + ) +} +fn timestamp( + params: &HashMap, + key: &str, + default: Option, +) -> anyhow::Result { + let value = match params.get(key) { + Some(value) => value.parse::()?, + None => default.ok_or_else(|| anyhow::anyhow!("missing {key}"))?, + }; + anyhow::ensure!(value.is_finite(), "nonfinite {key}"); + Ok(value) +} +fn sample_value(value: f64) -> String { + if value == f64::INFINITY { + "+Inf".into() + } else if value == f64::NEG_INFINITY { + "-Inf".into() + } else { + value.to_string() + } +} +fn result(data: Value) -> Reply { + ( + StatusCode::OK, + Json( + json!({"status":"success","data":data,"infos":["data_source: asap_exact","accuracy: exact","plan: bound canonical kernels"]}), + ), + ) +} +async fn instant( + State(snapshot): State>, + Query(params): Query>, +) -> Reply { + let run = (|| -> anyhow::Result { + let text = params + .get("query") + .ok_or_else(|| anyhow::anyhow!("missing query"))?; + let plan = ExactPromqlPlan::bind(text)?; + let time = timestamp(¶ms, "time", Some(snapshot.time))?; + let rows = execute(&plan, &snapshot.data, time, 300.0)?; + Ok( + json!({"resultType":"vector","result":rows.into_iter().map(|s|json!({"metric":s.labels,"value":[time,sample_value(s.value)]})).collect::>()}), + ) + })(); + match run { + Ok(data) => result(data), + Err(e) => error(e), + } +} +async fn range( + State(snapshot): State>, + Query(params): Query>, +) -> Reply { + let run = (|| -> anyhow::Result { + let text = params + .get("query") + .ok_or_else(|| anyhow::anyhow!("missing query"))?; + let plan = ExactPromqlPlan::bind(text)?; + let start = timestamp(¶ms, "start", None)?; + let end = timestamp(¶ms, "end", None)?; + let step = timestamp(¶ms, "step", None)?; + anyhow::ensure!(end >= start && step > 0.0, "invalid range bounds or step"); + let steps = ((end - start) / step).floor(); + anyhow::ensure!(steps < 11000.0, "too many evaluation steps"); + let count = steps as usize + 1; + let mut rows: BTreeMap> = BTreeMap::new(); + for i in 0..count { + let time = start + i as f64 * step; + for sample in execute(&plan, &snapshot.data, time, 300.0)? { + rows.entry(sample.labels) + .or_default() + .push(json!([time, sample_value(sample.value)])); + } + } + Ok( + json!({"resultType":"matrix","result":rows.into_iter().map(|(labels,values)|json!({"metric":labels,"values":values})).collect::>()}), + ) + })(); + match run { + Ok(data) => result(data), + Err(e) => error(e), + } +} +#[tokio::main] +async fn main() -> anyhow::Result<()> { + let mut args = std::env::args().skip(1); + let fixture = args + .next() + .unwrap_or_else(|| "tools/promql-smoke/cases.json".into()); + let listen = args.next().unwrap_or_else(|| "127.0.0.1:18081".into()); + anyhow::ensure!( + args.next().is_none(), + "usage: promql_exact_smoke [cases.json] [listen-address]" + ); + let cases: Value = serde_json::from_slice(&std::fs::read(fixture)?)?; + let start = cases["start"] + .as_f64() + .ok_or_else(|| anyhow::anyhow!("missing start"))?; + let interval = cases["interval"] + .as_f64() + .ok_or_else(|| anyhow::anyhow!("missing interval"))?; + let time = start + + cases["eval_offset"] + .as_f64() + .ok_or_else(|| anyhow::anyhow!("missing eval_offset"))?; + let mut data = Vec::new(); + for series in cases["series"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("missing series"))? + { + let labels = serde_json::from_value(series["labels"].clone())?; + let samples = series["values"] + .as_array() + .ok_or_else(|| anyhow::anyhow!("missing values"))? + .iter() + .enumerate() + .filter_map(|(i, v)| v.as_f64().map(|v| (start + i as f64 * interval, v))) + .collect(); + data.push(RawSeries { labels, samples }); + } + let app = Router::new() + .route("/api/v1/query", get(instant)) + .route("/api/v1/query_range", get(range)) + .route("/api/v1/health", get(|| async { "ok" })) + .with_state(Arc::new(Snapshot { data, time })); + let listener = tokio::net::TcpListener::bind(&listen).await?; + eprintln!( + "Native exact PromQL smoke server at http://{}", + listener.local_addr()? + ); + axum::serve(listener, app) + .with_graceful_shutdown(async { + let _ = tokio::signal::ctrl_c().await; + }) + .await?; + Ok(()) +} diff --git a/data_plane/tests/promql_exact_execution.rs b/data_plane/tests/promql_exact_execution.rs new file mode 100644 index 00000000..41a89b08 --- /dev/null +++ b/data_plane/tests/promql_exact_execution.rs @@ -0,0 +1,155 @@ +//! The same hand-checked fixture used by official promtool must execute locally. +use control_plane::physical::promql_exact::ExactPromqlPlan; +use data_plane::query_engines::canonical::exact_promql::{execute, Labels, RawSeries}; +use serde_json::Value; + +fn number(value: &Value) -> f64 { + value + .as_f64() + .unwrap_or_else(|| value.as_str().unwrap().parse().unwrap()) +} + +/// Every smoke query must bind to a real kernel and produce the official labels and values. +#[test] +fn all_smoke_queries_bind_and_execute_exactly() { + let cases: Value = + serde_json::from_str(include_str!("../../tools/promql-smoke/cases.json")).unwrap(); + let start = cases["start"].as_f64().unwrap(); + let interval = cases["interval"].as_f64().unwrap(); + let evaluation = start + cases["eval_offset"].as_f64().unwrap(); + let data: Vec<_> = cases["series"] + .as_array() + .unwrap() + .iter() + .map(|series| RawSeries { + labels: serde_json::from_value(series["labels"].clone()).unwrap(), + samples: series["values"] + .as_array() + .unwrap() + .iter() + .enumerate() + .filter_map(|(i, v)| v.as_f64().map(|value| (start + i as f64 * interval, value))) + .collect(), + }) + .collect(); + let mut failures = Vec::new(); + for query in cases["queries"].as_array().unwrap() { + let id = query["id"].as_str().unwrap(); + let expr = query["expr"].as_str().unwrap(); + let result = (|| -> anyhow::Result<()> { + let plan = ExactPromqlPlan::bind(expr)?; + let actual = execute(&plan, &data, evaluation, 300.0)?; + let expected = query["expected"].as_array().unwrap(); + anyhow::ensure!( + actual.len() == expected.len(), + "series count {} != {}", + actual.len(), + expected.len() + ); + for (index, sample) in expected.iter().enumerate() { + let labels: Labels = serde_json::from_value(sample["labels"].clone())?; + let value = number(&sample["value"]) + + if query["value_is_timestamp"] == true { + start + } else { + 0.0 + }; + let got = actual + .iter() + .find(|s| s.labels == labels) + .ok_or_else(|| anyhow::anyhow!("missing labels {labels:?}; got {actual:?}"))?; + let equal = if value.is_nan() { + got.value.is_nan() + } else if value.is_infinite() { + got.value == value + } else { + (got.value - value).abs() <= 1e-12 + 1e-12 * value.abs() + }; + anyhow::ensure!(equal, "{labels:?}: {} != {value}", got.value); + if query["ordered"] == true { + anyhow::ensure!(actual[index].labels == labels, "wrong series order"); + } + } + Ok(()) + })(); + match result { + Ok(()) => println!("PASS {id}"), + Err(error) => failures.push(format!("{id}: {expr}: {error}")), + } + } + assert!( + failures.is_empty(), + "{} queries failed:\n{}", + failures.len(), + failures.join("\n") + ); +} + +/// Unsupported shape modifiers must be rejected during binding, never ignored by execution. +#[test] +fn unsupported_exact_shapes_are_not_bound() { + for query in [ + "sum(smoke_gauge offset 1m)", + "sum(smoke_gauge @ 100)", + "sum_over_time(smoke_gauge[5m:1m])", + "smoke_gauge + on(job) smoke_gauge", + ] { + assert!(ExactPromqlPlan::bind(query).is_err(), "must reject {query}"); + } +} + +/// Duplicate timestamps and duplicate label identities cannot enter the evaluator silently. +#[test] +fn invalid_raw_snapshots_are_rejected() { + let plan = ExactPromqlPlan::bind("sum(smoke_gauge)").unwrap(); + let series = RawSeries { + labels: [("__name__".into(), "smoke_gauge".into())] + .into_iter() + .collect(), + samples: vec![(1.0, 2.0), (1.0, 3.0)], + }; + assert!(execute(&plan, &[series.clone()], 2.0, 300.0).is_err()); + let unique = RawSeries { + samples: vec![(1.0, 2.0)], + ..series + }; + assert!(execute(&plan, &[unique.clone(), unique], 2.0, 300.0).is_err()); +} + +/// Counting two series with equal numeric values must return two, not one distinct value. +#[test] +fn count_preserves_equal_valued_series_multiplicity() { + let plan = ExactPromqlPlan::bind("count(smoke_gauge)").unwrap(); + let data: Vec<_> = ["a", "b"] + .into_iter() + .map(|job| RawSeries { + labels: [ + ("__name__".into(), "smoke_gauge".into()), + ("job".into(), job.into()), + ] + .into_iter() + .collect(), + samples: vec![(240.0, 5.0)], + }) + .collect(); + let result = execute(&plan, &data, 240.0, 300.0).unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0].value, 2.0); +} + +/// Negative ratios select from the upper end: -1 must retain the entire input. +#[test] +fn negative_full_ratio_keeps_every_series() { + let plan = ExactPromqlPlan::bind("limit_ratio(-1, smoke_gauge)").unwrap(); + let data = vec![RawSeries { + labels: [ + ("__name__".into(), "smoke_gauge".into()), + ("job".into(), "a".into()), + ] + .into_iter() + .collect(), + samples: vec![(240.0, 5.0)], + }]; + let result = execute(&plan, &data, 240.0, 300.0).unwrap(); + assert_eq!(result.len(), 1); +} diff --git a/tools/promql-smoke/README.md b/tools/promql-smoke/README.md new file mode 100644 index 00000000..fedd2b9d --- /dev/null +++ b/tools/promql-smoke/README.md @@ -0,0 +1,157 @@ +# PromQL aggregation and rollup smoke tests + +Small fixtures for **Prometheus 3.5.0**: **14 aggregation operators**, **25 range-vector +functions**, **105 queries**, **14 series**, **64 samples**. Every query has explicit +expected labels and values in [cases.json](cases.json), plus a note explaining the +case. `null` means a missing sample, not zero. + +“Rollup” here means every registered Prometheus function accepting a +`ValueTypeMatrix` argument. This is function-name coverage using float samples, +not full PromQL conformance: native histograms, mixed sample types, staleness, +and all combinations of modifiers are not covered. Instant-vector histogram +helpers and scalar/math/label functions are outside this scope. + +The catalog is extracted from the **v3.5.0** official parser registry, with source +URLs and SHA-256 hashes in [catalog.json](catalog.json). Generation fails if any +catalog entry has no case. `--verify-catalog` also downloads the pinned sources +and verifies both their hashes and their registered names against the catalog. + +| Category | Functions | +| --- | --- | +| Aggregations | `sum`, `avg`, `count`, `min`, `max`, `group`, `stddev`, `stdvar`, `topk`, `bottomk`, `count_values`, `quantile` | +| Experimental aggregations | `limitk`, `limit_ratio` | +| Time-window aggregations | `avg_over_time`, `min_over_time`, `max_over_time`, `sum_over_time`, `count_over_time`, `quantile_over_time`, `stddev_over_time`, `stdvar_over_time`, `last_over_time`, `present_over_time` | +| Other range-vector functions | `absent_over_time`, `changes`, `delta`, `deriv`, `idelta`, `increase`, `irate`, `predict_linear`, `rate`, `resets` | +| Experimental range-vector functions | `double_exponential_smoothing`, `mad_over_time`, `ts_of_min_over_time`, `ts_of_max_over_time`, `ts_of_last_over_time` | + +The extra cases cover empty inputs, grouping, repeated values, sparse sampling, +single-sample ranges, counter resets and zero-point extrapolation, left-open +window boundaries, interpolated p99, tied timestamp extrema, and nonfinite values. +The two original gauges remain `1,2,3,4,5` and `10,20,30,40,50`. + +## Run official reference tests + +From the repository root, using the official **3.5.0** promtool binary: + +```bash +python3 tools/promql-smoke/run.py --promtool /path/to/promtool --verify-catalog +``` + +This checks the binary version and runs both suites, enabling +`promql-experimental-functions` only for the experimental suite. Ordinary runs +can omit `--verify-catalog` to work offline. Outputs default to +`/tmp/asap-promql-smoke`; change that with `--output-dir`. + +If using Docker, generate once and run both suites: + +```bash +python3 tools/promql-smoke/run.py --verify-catalog + +docker run --rm -v /tmp/asap-promql-smoke:/tests:ro --entrypoint promtool \ + prom/prometheus:v3.5.0 test rules /tests/rules.test.yml + +docker run --rm -v /tmp/asap-promql-smoke:/tests:ro --entrypoint promtool \ + prom/prometheus:v3.5.0 --enable-feature=promql-experimental-functions \ + test rules /tests/experimental.test.yml +``` + +The Python `--promtool` runner additionally saves per-case JUnit results, logs, +and `reference-results.json`. It returns nonzero on failure. Expected finite +values use promtool's one-bit floating-point tolerance. + +Promtool 3.5 does not consider NaN equal to NaN. That one reference case uses +`x != bool x` to prove the result is NaN; its original expression and raw NaN +expectations are retained for planner and HTTP tests. The override is explicit +in `cases.json`. The infinity cases compare raw values directly. + +## Bind and execute through ASAPPlanner and the backend + +The workspace pins the published planner commit +`f27b16a747e5d7fcd70a5510075c0cd062f0dcea` +([ASAPPlanner PR #413](https://github.com/ProjectASAP/ASAPPlanner/pull/413)). +This commit applies PR #413 on top of the backend’s existing planner pin +`029ff2fe041172c94c2d32c90b185bc83c5e8a57`, preserving its interfaces. +No adjacent planner checkout or local Cargo patch is required. The planner +preserves `irate`, series-count semantics, and special quantile parameters in +the canonical tree. + +```bash +cargo +1.98.0 test --locked -p data_plane --test promql_exact_execution +cargo +1.98.0 run --locked -p control_plane --example promql_smoke -- --require-bound + +# Save canonical trees, exact plans, and the original summary binder diagnostics. +cargo +1.98.0 run --locked -p control_plane --example promql_smoke -- \ + --json --require-bound > /tmp/asap-promql-smoke/planner-results.json +``` + +`ExactPromqlPlan::bind` calls the backend's `parse_query_expr_canonical` with +`AccuracyTarget::Exact`, then compiles that canonical tree into typed native +kernels. `BOUND_EXACT` means an executable exact plan; unsupported shapes are +`EXACT_BIND_REJECTED`. `--require-bound` fails on any rejection. Original sketch +binder diagnostics remain under `summary_status`; they do not determine exact +execution support. + +The Rust execution test evaluates all 105 queries against raw timestamped float +samples and compares full labels, values, and requested ordering with the same +fixture expectations verified by official Prometheus. Additional regression tests +cover invalid snapshots, unsupported modifiers, equal-valued series counts, and +negative sampling ratios. + +## Run the local native HTTP smoke server + +In one terminal: + +```bash +cargo +1.98.0 run --locked -p data_plane --example promql_exact_smoke +``` + +In another: + +```bash +python3 tools/promql-smoke/run.py --backend-url http://127.0.0.1:18081 +``` + +This example loads `cases.json` directly and serves `/api/v1/query` and +`/api/v1/query_range` using the canonical binder and backend exact executor. +It performs no Prometheus forwarding. Responses identify `data_source: asap_exact`. +Optional positional arguments are the fixture path and listening address. +This is a local test entry point; production storage and routing are not wired +into this new raw-sample execution path. Exact plans require raw samples, which +cannot in general be reconstructed from sketches. + +## Compare backend HTTP results + +After loading `samples.openmetrics` through the deployment's ingest path: + +```bash +python3 tools/promql-smoke/run.py --backend-url http://127.0.0.1:8080 +``` + +The runner does **not** load data or install a plan into the backend. It queries +all cases, including experimental ones, and reports unsupported queries as +failures. Fixture timestamps are in seconds, beginning at `1788825600`, sampled +every 60 seconds, and evaluated at `1788825840`. The official promtool suite uses +relative times starting at zero; `ts_of_*` expected values are shifted to epoch +time for HTTP comparisons. + +The comparator checks all labels including metric names, result type, the full +series set, evaluation timestamps, finite values (`rtol=atol=1e-12`), NaN/Inf, +and explicitly requested topk/bottomk ordering. Missing series are not replaced +with zero. Full responses, including provenance annotations, are saved in +`backend-results.json`. A matching response can still be a fallback; inspect its +provenance separately. Approximate sketch answers can fail these exact checks. + +## Validation + +```bash +python3 -m unittest discover -s tools/promql-smoke -p 'test_*.py' -v +``` + +[RESULTS.md](RESULTS.md) records the observed official and planner results from +2026-09-13. It is a snapshot, not a substitute for rerunning after changes. + +Official sources: +[operators](https://prometheus.io/docs/prometheus/3.5/querying/operators/), +[functions](https://prometheus.io/docs/prometheus/3.5/querying/functions/), +[function registry](https://github.com/prometheus/prometheus/blob/v3.5.0/promql/parser/functions.go), +[aggregation registry](https://github.com/prometheus/prometheus/blob/v3.5.0/promql/parser/lex.go). diff --git a/tools/promql-smoke/RESULTS.md b/tools/promql-smoke/RESULTS.md new file mode 100644 index 00000000..3acc44a3 --- /dev/null +++ b/tools/promql-smoke/RESULTS.md @@ -0,0 +1,57 @@ +# Recorded smoke results + +Run date: 2026-09-13. + +- Fixture SHA-256: `7df76b148e7b1d4c11260f23679d5138d1bb6c14ededf9ff8d1d9415f8146eeb`. +- Reference: official Prometheus/promtool 3.5.0 (`8be3a9560fbdd18a94dedec4b747c35178177202`). +- Backend PR base: `8cf1890b` on `origin/main`. +- Planner dependency: published commit `f27b16a747e5d7fcd70a5510075c0cd062f0dcea` + (the fix from [PR #413](https://github.com/ProjectASAP/ASAPPlanner/pull/413) + applied to the existing backend planner pin); no local path patch. + +| Check | Result | +| --- | --- | +| Official promtool | 84 stable + 21 experimental cases passed | +| Official Prometheus HTTP against isolated fixture TSDB | 105/105 passed | +| Canonical tree → exact kernel binding | 105/105 `BOUND_EXACT` | +| Backend native exact execution against official-verified expectations | 105/105 passed | +| Local native backend HTTP `/api/v1/query` | 105/105 passed; every response identifies `asap_exact` | +| Local native HTTP range consistency | 105/105 range queries match per-step instant results | +| Native executor regression tests | 5/5 passed, including the 105-case corpus | +| Planner frontend regression/conformance/lowering/equivalence tests | 162/162 passed | +| Python comparator/coverage tests | 8/8 passed | + +All 14 aggregation operators and 25 range-vector functions in the pinned catalog +have an executable exact binding. This measures float-sample function coverage, +not full PromQL conformance. Native histograms, mixed types, staleness, offsets, +`@`, subqueries, and explicit vector matching are outside this exact smoke path; +unsupported query shapes are rejected rather than silently stripped. + +The exact plan consumes the real ASAPPlanner canonical tree. Execution uses native +backend kernels and timestamped raw samples; it does not forward queries to +Prometheus or read expected results from the fixture. The HTTP check uses the +`promql_exact_smoke` example, not the production ingestion/storage/router path. +Production use still needs a raw-sample source and routing integration. + +## Semantic behavior exercised + +- Existing upstream `irate` retains a distinct canonical intent from `rate`. +- Existing upstream `count` counts series, including equal-valued series, instead of distinct numbers. +- Quantile phi outside [0,1] and NaN survives lowering and produces the defined + `-Inf`, `+Inf`, or `NaN` result in the smoke cases. +- Negative `limit_ratio` uses the upper hash interval; `-1` keeps every series. + +The original summary/sketch binder diagnostics remain in `summary_status` in the +planner JSON report. Exact bindings do not imply these functions can execute from +existing sketches alone. + +Original smoke logs are under `/tmp/asap-promql-smoke/`, including +`reference-results.json`, `planner-results.json`, `backend-results.json`, +`native-exact-results.log`, `planner-regressions-after.log`, +`planner-types-mapping.log`, and `http-reference/reference-http-results.json`. +These temporary artifacts may be removed; reproduce the checks with [README.md](README.md). + +PR-branch reruns of promtool, canonical binding, native execution, native HTTP +instant/range checks, planner frontend tests, and Python tests are recorded under +`/tmp/promql-pr-smoke/` and `/tmp/promql-pr-*.log`. The official HTTP reference +check was recorded with the identical fixture before the rebase. diff --git a/tools/promql-smoke/cases.json b/tools/promql-smoke/cases.json new file mode 100644 index 00000000..3b13f6ee --- /dev/null +++ b/tools/promql-smoke/cases.json @@ -0,0 +1,1849 @@ +{ + "prometheus_version": "3.5.0", + "start": 1788825600, + "interval": 60, + "eval_offset": 240, + "series": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "a" + }, + "values": [ + 1, + 2, + 3, + 4, + 5 + ] + }, + { + "labels": { + "__name__": "smoke_gauge", + "job": "b" + }, + "values": [ + 10, + 20, + 30, + 40, + 50 + ] + }, + { + "labels": { + "__name__": "smoke_counter_total", + "job": "steady" + }, + "values": [ + 60, + 120, + 180, + 240, + 300 + ] + }, + { + "labels": { + "__name__": "smoke_counter_total", + "job": "reset" + }, + "values": [ + 60, + 120, + 180, + 30, + 90 + ] + }, + { + "labels": { + "__name__": "smoke_counter_total", + "job": "last_reset" + }, + "values": [ + 60, + 120, + 180, + 240, + 30 + ] + }, + { + "labels": { + "__name__": "smoke_zero_counter_total", + "job": "zero" + }, + "values": [ + 0, + 60, + 120, + 180, + 240 + ] + }, + { + "labels": { + "__name__": "smoke_constant", + "job": "constant" + }, + "values": [ + 7, + 7, + 7, + 7, + 7 + ] + }, + { + "labels": { + "__name__": "smoke_repeat", + "job": "repeat" + }, + "values": [ + 3, + 1, + 3, + 1, + 2 + ] + }, + { + "labels": { + "__name__": "smoke_sparse", + "job": "sparse" + }, + "values": [ + 1, + null, + 3, + null, + 5 + ] + }, + { + "labels": { + "__name__": "smoke_single", + "job": "single" + }, + "values": [ + null, + null, + null, + null, + 5 + ] + }, + { + "labels": { + "__name__": "smoke_group", + "job": "a", + "instance": "x" + }, + "values": [ + 1, + 1, + 1, + 1, + 1 + ] + }, + { + "labels": { + "__name__": "smoke_group", + "job": "a", + "instance": "y" + }, + "values": [ + 3, + 3, + 3, + 3, + 3 + ] + }, + { + "labels": { + "__name__": "smoke_group", + "job": "b", + "instance": "x" + }, + "values": [ + 2, + 2, + 2, + 2, + 2 + ] + }, + { + "labels": { + "__name__": "smoke_group", + "job": "b", + "instance": "y" + }, + "values": [ + 6, + 6, + 6, + 6, + 6 + ] + } + ], + "queries": [ + { + "id": "selector", + "category": "selector", + "function": "selector", + "experimental": false, + "expr": "smoke_gauge{job=\"a\"}", + "expected": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "a" + }, + "value": 5 + } + ], + "note": "Baseline label filter." + }, + { + "id": "agg_sum", + "category": "aggregation", + "function": "sum", + "experimental": false, + "expr": "sum(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 55 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_avg", + "category": "aggregation", + "function": "avg", + "experimental": false, + "expr": "avg(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 27.5 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_count", + "category": "aggregation", + "function": "count", + "experimental": false, + "expr": "count(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 2 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_min", + "category": "aggregation", + "function": "min", + "experimental": false, + "expr": "min(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 5 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_max", + "category": "aggregation", + "function": "max", + "experimental": false, + "expr": "max(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 50 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_group", + "category": "aggregation", + "function": "group", + "experimental": false, + "expr": "group(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 1 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_stddev", + "category": "aggregation", + "function": "stddev", + "experimental": false, + "expr": "stddev(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 22.5 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_stdvar", + "category": "aggregation", + "function": "stdvar", + "experimental": false, + "expr": "stdvar(smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 506.25 + } + ], + "note": "Aggregate the final values 5 and 50." + }, + { + "id": "agg_topk", + "category": "aggregation", + "function": "topk", + "experimental": false, + "expr": "topk(1, smoke_gauge)", + "expected": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "b" + }, + "value": 50 + } + ], + "note": "Largest series; preserve its labels and metric name." + }, + { + "id": "agg_bottomk", + "category": "aggregation", + "function": "bottomk", + "experimental": false, + "expr": "bottomk(1, smoke_gauge)", + "expected": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "a" + }, + "value": 5 + } + ], + "note": "Smallest series; preserve its labels and metric name." + }, + { + "id": "agg_count_values", + "category": "aggregation", + "function": "count_values", + "experimental": false, + "expr": "count_values(\"sample\", smoke_gauge % 5)", + "expected": [ + { + "labels": { + "sample": "0" + }, + "value": 2 + } + ], + "note": "Both final values modulo 5 are zero; count repeated values." + }, + { + "id": "agg_quantile", + "category": "aggregation", + "function": "quantile", + "experimental": false, + "expr": "quantile(0.5, smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 27.5 + } + ], + "note": "Median of 5 and 50 is linearly interpolated." + }, + { + "id": "agg_limitk", + "category": "aggregation", + "function": "limitk", + "experimental": true, + "expr": "limitk(1, smoke_gauge{job=\"a\"})", + "expected": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "a" + }, + "value": 5 + } + ], + "note": "Singleton selection has an unambiguous identity; separate case checks sampling two series." + }, + { + "id": "agg_limit_ratio", + "category": "aggregation", + "function": "limit_ratio", + "experimental": true, + "expr": "limit_ratio(1, smoke_gauge)", + "expected": [ + { + "labels": { + "job": "a", + "__name__": "smoke_gauge" + }, + "value": 5 + }, + { + "labels": { + "job": "b", + "__name__": "smoke_gauge" + }, + "value": 50 + } + ], + "note": "Ratio 1 selects the entire input without changing labels." + }, + { + "id": "rollup_avg_over_time", + "category": "rollup", + "function": "avg_over_time", + "experimental": false, + "expr": "avg_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 3 + }, + { + "labels": { + "job": "b" + }, + "value": 30 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_count_over_time", + "category": "rollup", + "function": "count_over_time", + "experimental": false, + "expr": "count_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 5 + }, + { + "labels": { + "job": "b" + }, + "value": 5 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_last_over_time", + "category": "rollup", + "function": "last_over_time", + "experimental": false, + "expr": "last_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a", + "__name__": "smoke_gauge" + }, + "value": 5 + }, + { + "labels": { + "job": "b", + "__name__": "smoke_gauge" + }, + "value": 50 + } + ], + "note": "last_over_time preserves the metric name, unlike the other numeric rollups." + }, + { + "id": "rollup_max_over_time", + "category": "rollup", + "function": "max_over_time", + "experimental": false, + "expr": "max_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 5 + }, + { + "labels": { + "job": "b" + }, + "value": 50 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_min_over_time", + "category": "rollup", + "function": "min_over_time", + "experimental": false, + "expr": "min_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 10 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_present_over_time", + "category": "rollup", + "function": "present_over_time", + "experimental": false, + "expr": "present_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 1 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_stddev_over_time", + "category": "rollup", + "function": "stddev_over_time", + "experimental": false, + "expr": "stddev_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1.4142135623730951 + }, + { + "labels": { + "job": "b" + }, + "value": 14.142135623730951 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_stdvar_over_time", + "category": "rollup", + "function": "stdvar_over_time", + "experimental": false, + "expr": "stdvar_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 2 + }, + { + "labels": { + "job": "b" + }, + "value": 200 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_sum_over_time", + "category": "rollup", + "function": "sum_over_time", + "experimental": false, + "expr": "sum_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 15 + }, + { + "labels": { + "job": "b" + }, + "value": 150 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_mad_over_time", + "category": "rollup", + "function": "mad_over_time", + "experimental": true, + "expr": "mad_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 10 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_delta", + "category": "rollup", + "function": "delta", + "experimental": false, + "expr": "delta(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 5 + }, + { + "labels": { + "job": "b" + }, + "value": 50 + } + ], + "note": "Observed difference 4/40 extrapolated from 240s to the 300s window: 5/50." + }, + { + "id": "rollup_idelta", + "category": "rollup", + "function": "idelta", + "experimental": false, + "expr": "idelta(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 10 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_deriv", + "category": "rollup", + "function": "deriv", + "experimental": false, + "expr": "deriv(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 0.016666666666666666 + }, + { + "labels": { + "job": "b" + }, + "value": 0.16666666666666666 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_changes", + "category": "rollup", + "function": "changes", + "experimental": false, + "expr": "changes(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 4 + }, + { + "labels": { + "job": "b" + }, + "value": 4 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_quantile_over_time", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(0.5, smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 3 + }, + { + "labels": { + "job": "b" + }, + "value": 30 + } + ], + "note": "Evaluate the five samples at t=240s; [5m] includes t=0." + }, + { + "id": "rollup_predict_linear", + "category": "rollup", + "function": "predict_linear", + "experimental": false, + "expr": "predict_linear(smoke_gauge[5m], 60)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 6 + }, + { + "labels": { + "job": "b" + }, + "value": 60 + } + ], + "note": "Linear trend projected 60 seconds beyond evaluation time." + }, + { + "id": "rollup_double_exponential_smoothing", + "category": "rollup", + "function": "double_exponential_smoothing", + "experimental": true, + "expr": "double_exponential_smoothing(smoke_gauge[5m], 0.5, 0.5)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 5 + }, + { + "labels": { + "job": "b" + }, + "value": 50 + } + ], + "note": "The input is a perfect linear trend; smoothing follows it exactly." + }, + { + "id": "rollup_rate", + "category": "rollup", + "function": "rate", + "experimental": false, + "expr": "rate(smoke_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "steady" + }, + "value": 1 + }, + { + "labels": { + "job": "reset" + }, + "value": 0.875 + }, + { + "labels": { + "job": "last_reset" + }, + "value": 0.875 + } + ], + "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." + }, + { + "id": "rollup_increase", + "category": "rollup", + "function": "increase", + "experimental": false, + "expr": "increase(smoke_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "steady" + }, + "value": 300 + }, + { + "labels": { + "job": "reset" + }, + "value": 262.5 + }, + { + "labels": { + "job": "last_reset" + }, + "value": 262.5 + } + ], + "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." + }, + { + "id": "rollup_irate", + "category": "rollup", + "function": "irate", + "experimental": false, + "expr": "irate(smoke_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "steady" + }, + "value": 1 + }, + { + "labels": { + "job": "reset" + }, + "value": 1 + }, + { + "labels": { + "job": "last_reset" + }, + "value": 0.5 + } + ], + "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." + }, + { + "id": "rollup_resets", + "category": "rollup", + "function": "resets", + "experimental": false, + "expr": "resets(smoke_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "steady" + }, + "value": 0 + }, + { + "labels": { + "job": "reset" + }, + "value": 1 + }, + { + "labels": { + "job": "last_reset" + }, + "value": 1 + } + ], + "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." + }, + { + "id": "rollup_absent_over_time", + "category": "rollup", + "function": "absent_over_time", + "experimental": false, + "expr": "absent_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [ + { + "labels": { + "job": "missing" + }, + "value": 1 + } + ], + "note": "Missing range yields 1 and derives equality-matcher labels." + }, + { + "id": "rollup_ts_of_max_over_time", + "category": "rollup", + "function": "ts_of_max_over_time", + "experimental": true, + "expr": "ts_of_max_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 240 + }, + { + "labels": { + "job": "b" + }, + "value": 240 + } + ], + "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", + "value_is_timestamp": true + }, + { + "id": "rollup_ts_of_min_over_time", + "category": "rollup", + "function": "ts_of_min_over_time", + "experimental": true, + "expr": "ts_of_min_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 0 + }, + { + "labels": { + "job": "b" + }, + "value": 0 + } + ], + "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", + "value_is_timestamp": true + }, + { + "id": "rollup_ts_of_last_over_time", + "category": "rollup", + "function": "ts_of_last_over_time", + "experimental": true, + "expr": "ts_of_last_over_time(smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 240 + }, + { + "labels": { + "job": "b" + }, + "value": 240 + } + ], + "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", + "value_is_timestamp": true + }, + { + "id": "empty_avg_over_time", + "category": "rollup", + "function": "avg_over_time", + "experimental": false, + "expr": "avg_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_count_over_time", + "category": "rollup", + "function": "count_over_time", + "experimental": false, + "expr": "count_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_last_over_time", + "category": "rollup", + "function": "last_over_time", + "experimental": false, + "expr": "last_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_max_over_time", + "category": "rollup", + "function": "max_over_time", + "experimental": false, + "expr": "max_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_min_over_time", + "category": "rollup", + "function": "min_over_time", + "experimental": false, + "expr": "min_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_present_over_time", + "category": "rollup", + "function": "present_over_time", + "experimental": false, + "expr": "present_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_stddev_over_time", + "category": "rollup", + "function": "stddev_over_time", + "experimental": false, + "expr": "stddev_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_stdvar_over_time", + "category": "rollup", + "function": "stdvar_over_time", + "experimental": false, + "expr": "stdvar_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_sum_over_time", + "category": "rollup", + "function": "sum_over_time", + "experimental": false, + "expr": "sum_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_mad_over_time", + "category": "rollup", + "function": "mad_over_time", + "experimental": true, + "expr": "mad_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_delta", + "category": "rollup", + "function": "delta", + "experimental": false, + "expr": "delta(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_idelta", + "category": "rollup", + "function": "idelta", + "experimental": false, + "expr": "idelta(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_deriv", + "category": "rollup", + "function": "deriv", + "experimental": false, + "expr": "deriv(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_changes", + "category": "rollup", + "function": "changes", + "experimental": false, + "expr": "changes(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_quantile_over_time", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(0.5, smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_predict_linear", + "category": "rollup", + "function": "predict_linear", + "experimental": false, + "expr": "predict_linear(smoke_missing{job=\"missing\"}[5m], 60)", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_double_exponential_smoothing", + "category": "rollup", + "function": "double_exponential_smoothing", + "experimental": true, + "expr": "double_exponential_smoothing(smoke_missing{job=\"missing\"}[5m], 0.5, 0.5)", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_rate", + "category": "rollup", + "function": "rate", + "experimental": false, + "expr": "rate(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_increase", + "category": "rollup", + "function": "increase", + "experimental": false, + "expr": "increase(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_irate", + "category": "rollup", + "function": "irate", + "experimental": false, + "expr": "irate(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_resets", + "category": "rollup", + "function": "resets", + "experimental": false, + "expr": "resets(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero." + }, + { + "id": "empty_absent_over_time", + "category": "rollup", + "function": "absent_over_time", + "experimental": false, + "expr": "absent_over_time(smoke_gauge[5m])", + "expected": [], + "note": "An existing range must not trigger absence." + }, + { + "id": "empty_ts_of_max_over_time", + "category": "rollup", + "function": "ts_of_max_over_time", + "experimental": true, + "expr": "ts_of_max_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero.", + "value_is_timestamp": true + }, + { + "id": "empty_ts_of_min_over_time", + "category": "rollup", + "function": "ts_of_min_over_time", + "experimental": true, + "expr": "ts_of_min_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero.", + "value_is_timestamp": true + }, + { + "id": "empty_ts_of_last_over_time", + "category": "rollup", + "function": "ts_of_last_over_time", + "experimental": true, + "expr": "ts_of_last_over_time(smoke_missing{job=\"missing\"}[5m])", + "expected": [], + "note": "An empty input must produce no series, not a fabricated zero.", + "value_is_timestamp": true + }, + { + "id": "empty_sum", + "category": "aggregation", + "function": "sum", + "experimental": false, + "expr": "sum(smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_count", + "category": "aggregation", + "function": "count", + "experimental": false, + "expr": "count(smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_group", + "category": "aggregation", + "function": "group", + "experimental": false, + "expr": "group(smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_topk", + "category": "aggregation", + "function": "topk", + "experimental": false, + "expr": "topk(1, smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_quantile", + "category": "aggregation", + "function": "quantile", + "experimental": false, + "expr": "quantile(0.5, smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_limitk", + "category": "aggregation", + "function": "limitk", + "experimental": true, + "expr": "limitk(1, smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "empty_limit_ratio", + "category": "aggregation", + "function": "limit_ratio", + "experimental": true, + "expr": "limit_ratio(1, smoke_missing)", + "expected": [], + "note": "An aggregate over no series yields an empty vector." + }, + { + "id": "sum_by", + "category": "aggregation", + "function": "sum", + "experimental": false, + "expr": "sum by (job) (smoke_group)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 4 + }, + { + "labels": { + "job": "b" + }, + "value": 8 + } + ], + "note": "Group four series into two jobs." + }, + { + "id": "sum_without", + "category": "aggregation", + "function": "sum", + "experimental": false, + "expr": "sum without (instance) (smoke_group)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 4 + }, + { + "labels": { + "job": "b" + }, + "value": 8 + } + ], + "note": "Drop instance and metric name from output labels." + }, + { + "id": "avg_by", + "category": "aggregation", + "function": "avg", + "experimental": false, + "expr": "avg by (job) (smoke_group)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 2 + }, + { + "labels": { + "job": "b" + }, + "value": 4 + } + ], + "note": "Average separately within each job." + }, + { + "id": "count_by", + "category": "aggregation", + "function": "count", + "experimental": false, + "expr": "count by (job) (smoke_group)", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 2 + }, + { + "labels": { + "job": "b" + }, + "value": 2 + } + ], + "note": "Count series separately within each job." + }, + { + "id": "quantile_p99", + "category": "aggregation", + "function": "quantile", + "experimental": false, + "expr": "quantile(0.99, smoke_gauge)", + "expected": [ + { + "labels": {}, + "value": 49.55 + } + ], + "note": "Interpolate between 5 and 50." + }, + { + "id": "limitk_count", + "category": "aggregation", + "function": "limitk", + "experimental": true, + "expr": "count(limitk(1, smoke_gauge))", + "expected": [ + { + "labels": {}, + "value": 1 + } + ], + "note": "Select exactly one of two series without depending on a hash-selected identity." + }, + { + "id": "limit_ratio_complement", + "category": "aggregation", + "function": "limit_ratio", + "experimental": true, + "expr": "sum(limit_ratio(0.5, smoke_gauge) or limit_ratio(-0.5, smoke_gauge))", + "expected": [ + { + "labels": {}, + "value": 55 + } + ], + "note": "Positive and negative ratios cover the full input." + }, + { + "id": "limit_ratio_disjoint", + "category": "aggregation", + "function": "limit_ratio", + "experimental": true, + "expr": "limit_ratio(0.5, smoke_gauge) and limit_ratio(-0.5, smoke_gauge)", + "expected": [], + "note": "Complementary subsets must not overlap." + }, + { + "id": "p99", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(0.99, smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 4.96 + }, + { + "labels": { + "job": "b" + }, + "value": 49.6 + } + ], + "note": "Small-sample p99 requires interpolation." + }, + { + "id": "window_left_open", + "category": "rollup", + "function": "sum_over_time", + "experimental": false, + "expr": "sum_over_time(smoke_gauge[4m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 14 + }, + { + "labels": { + "job": "b" + }, + "value": 140 + } + ], + "note": "The sample at t=0 is exactly on the left boundary and is excluded." + }, + { + "id": "window_single", + "category": "rollup", + "function": "count_over_time", + "experimental": false, + "expr": "count_over_time(smoke_gauge[1m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 1 + } + ], + "note": "Only t=240 is included; t=180 is excluded." + }, + { + "id": "sparse_average", + "category": "rollup", + "function": "avg_over_time", + "experimental": false, + "expr": "avg_over_time(smoke_sparse[5m])", + "expected": [ + { + "labels": { + "job": "sparse" + }, + "value": 3 + } + ], + "note": "Missing samples are omitted, not zero-filled." + }, + { + "id": "sparse_count", + "category": "rollup", + "function": "count_over_time", + "experimental": false, + "expr": "count_over_time(smoke_sparse[5m])", + "expected": [ + { + "labels": { + "job": "sparse" + }, + "value": 3 + } + ], + "note": "Count only the three present samples." + }, + { + "id": "constant_changes", + "category": "rollup", + "function": "changes", + "experimental": false, + "expr": "changes(smoke_constant[5m])", + "expected": [ + { + "labels": { + "job": "constant" + }, + "value": 0 + } + ], + "note": "Equal adjacent values do not count as changes." + }, + { + "id": "constant_stddev", + "category": "rollup", + "function": "stddev_over_time", + "experimental": false, + "expr": "stddev_over_time(smoke_constant[5m])", + "expected": [ + { + "labels": { + "job": "constant" + }, + "value": 0 + } + ], + "note": "A constant series has zero standard deviation." + }, + { + "id": "zero_rate", + "category": "rollup", + "function": "rate", + "experimental": false, + "expr": "rate(smoke_zero_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "zero" + }, + "value": 0.8 + } + ], + "note": "Counter starts at zero: extrapolation must not invent a negative prior counter." + }, + { + "id": "zero_increase", + "category": "rollup", + "function": "increase", + "experimental": false, + "expr": "increase(smoke_zero_counter_total[5m])", + "expected": [ + { + "labels": { + "job": "zero" + }, + "value": 240 + } + ], + "note": "Zero-point clamping limits the extrapolated increase to 240." + }, + { + "id": "single_delta", + "category": "rollup", + "function": "delta", + "experimental": false, + "expr": "delta(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_deriv", + "category": "rollup", + "function": "deriv", + "experimental": false, + "expr": "deriv(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_idelta", + "category": "rollup", + "function": "idelta", + "experimental": false, + "expr": "idelta(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_rate", + "category": "rollup", + "function": "rate", + "experimental": false, + "expr": "rate(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_irate", + "category": "rollup", + "function": "irate", + "experimental": false, + "expr": "irate(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_increase", + "category": "rollup", + "function": "increase", + "experimental": false, + "expr": "increase(smoke_single[5m])", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_predict_linear", + "category": "rollup", + "function": "predict_linear", + "experimental": false, + "expr": "predict_linear(smoke_single[5m], 60)", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "single_double_exponential_smoothing", + "category": "rollup", + "function": "double_exponential_smoothing", + "experimental": true, + "expr": "double_exponential_smoothing(smoke_single[5m], 0.5, 0.5)", + "expected": [], + "note": "A single sample is insufficient; return no series." + }, + { + "id": "ties_ts_of_max_over_time", + "category": "rollup", + "function": "ts_of_max_over_time", + "experimental": true, + "expr": "ts_of_max_over_time(smoke_repeat[5m])", + "expected": [ + { + "labels": { + "job": "repeat" + }, + "value": 120 + } + ], + "note": "For tied extrema choose the latest sample timestamp.", + "value_is_timestamp": true + }, + { + "id": "ties_ts_of_min_over_time", + "category": "rollup", + "function": "ts_of_min_over_time", + "experimental": true, + "expr": "ts_of_min_over_time(smoke_repeat[5m])", + "expected": [ + { + "labels": { + "job": "repeat" + }, + "value": 180 + } + ], + "note": "For tied extrema choose the latest sample timestamp.", + "value_is_timestamp": true + }, + { + "id": "ties_ts_of_last_over_time", + "category": "rollup", + "function": "ts_of_last_over_time", + "experimental": true, + "expr": "ts_of_last_over_time(smoke_repeat[5m])", + "expected": [ + { + "labels": { + "job": "repeat" + }, + "value": 240 + } + ], + "note": "For tied extrema choose the latest sample timestamp.", + "value_is_timestamp": true + }, + { + "id": "phi_-0.1", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(-0.1, smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": "-Inf" + }, + { + "labels": { + "job": "b" + }, + "value": "-Inf" + } + ], + "note": "Out-of-range quantiles return the corresponding infinity." + }, + { + "id": "phi_1.1", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(1.1, smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": "+Inf" + }, + { + "labels": { + "job": "b" + }, + "value": "+Inf" + } + ], + "note": "Out-of-range quantiles return the corresponding infinity." + }, + { + "id": "phi_nan", + "category": "rollup", + "function": "quantile_over_time", + "experimental": false, + "expr": "quantile_over_time(NaN, smoke_gauge[5m])", + "expected": [ + { + "labels": { + "job": "a" + }, + "value": "NaN" + }, + { + "labels": { + "job": "b" + }, + "value": "NaN" + } + ], + "note": "NaN quantile parameter returns NaN. promtool 3.5 cannot compare NaN expectations; the reference expression checks x != x with bool, which is true only for NaN. HTTP comparison checks raw NaN directly.", + "promtool_expr": "(quantile_over_time(NaN, smoke_gauge[5m])) != bool (quantile_over_time(NaN, smoke_gauge[5m]))", + "promtool_expected": [ + { + "labels": { + "job": "a" + }, + "value": 1 + }, + { + "labels": { + "job": "b" + }, + "value": 1 + } + ] + }, + { + "id": "topk_order", + "category": "aggregation", + "function": "topk", + "experimental": false, + "expr": "topk(2, smoke_gauge)", + "expected": [ + { + "labels": { + "__name__": "smoke_gauge", + "job": "b" + }, + "value": 50 + }, + { + "labels": { + "__name__": "smoke_gauge", + "job": "a" + }, + "value": 5 + } + ], + "note": "Instant topk returns descending values; HTTP comparator also verifies order.", + "ordered": true + }, + { + "id": "bottomk_order", + "category": "aggregation", + "function": "bottomk", + "experimental": false, + "expr": "bottomk(2, smoke_gauge)", + "expected": [ + { + "labels": { + "job": "a", + "__name__": "smoke_gauge" + }, + "value": 5 + }, + { + "labels": { + "job": "b", + "__name__": "smoke_gauge" + }, + "value": 50 + } + ], + "note": "Instant bottomk returns ascending values; HTTP comparator also verifies order.", + "ordered": true + } + ] +} diff --git a/tools/promql-smoke/catalog.json b/tools/promql-smoke/catalog.json new file mode 100644 index 00000000..2437d66d --- /dev/null +++ b/tools/promql-smoke/catalog.json @@ -0,0 +1,174 @@ +{ + "prometheus_version": "3.5.0", + "scope": "All aggregation operators and all functions with a ValueTypeMatrix argument; float-sample smoke coverage, not full type/edge-case conformance.", + "sources": { + "functions.go": { + "url": "https://raw.githubusercontent.com/prometheus/prometheus/v3.5.0/promql/parser/functions.go", + "sha256": "11881bfeb3093bea274f16f55a681bb1510a81c9cdc795756ad7256b2ef514cf" + }, + "lex.go": { + "url": "https://raw.githubusercontent.com/prometheus/prometheus/v3.5.0/promql/parser/lex.go", + "sha256": "a8d67c46cf53a10b2b3721d58a7734868f67f0467bbe6469e59bd97c5b1ac5fc" + } + }, + "aggregation": [ + { + "name": "sum", + "experimental": false + }, + { + "name": "avg", + "experimental": false + }, + { + "name": "count", + "experimental": false + }, + { + "name": "min", + "experimental": false + }, + { + "name": "max", + "experimental": false + }, + { + "name": "group", + "experimental": false + }, + { + "name": "stddev", + "experimental": false + }, + { + "name": "stdvar", + "experimental": false + }, + { + "name": "topk", + "experimental": false + }, + { + "name": "bottomk", + "experimental": false + }, + { + "name": "count_values", + "experimental": false + }, + { + "name": "quantile", + "experimental": false + }, + { + "name": "limitk", + "experimental": true + }, + { + "name": "limit_ratio", + "experimental": true + } + ], + "rollup": [ + { + "name": "absent_over_time", + "experimental": false + }, + { + "name": "avg_over_time", + "experimental": false + }, + { + "name": "changes", + "experimental": false + }, + { + "name": "count_over_time", + "experimental": false + }, + { + "name": "delta", + "experimental": false + }, + { + "name": "deriv", + "experimental": false + }, + { + "name": "double_exponential_smoothing", + "experimental": true + }, + { + "name": "idelta", + "experimental": false + }, + { + "name": "increase", + "experimental": false + }, + { + "name": "irate", + "experimental": false + }, + { + "name": "last_over_time", + "experimental": false + }, + { + "name": "mad_over_time", + "experimental": true + }, + { + "name": "max_over_time", + "experimental": false + }, + { + "name": "min_over_time", + "experimental": false + }, + { + "name": "ts_of_max_over_time", + "experimental": true + }, + { + "name": "ts_of_min_over_time", + "experimental": true + }, + { + "name": "ts_of_last_over_time", + "experimental": true + }, + { + "name": "predict_linear", + "experimental": false + }, + { + "name": "present_over_time", + "experimental": false + }, + { + "name": "quantile_over_time", + "experimental": false + }, + { + "name": "rate", + "experimental": false + }, + { + "name": "resets", + "experimental": false + }, + { + "name": "stddev_over_time", + "experimental": false + }, + { + "name": "stdvar_over_time", + "experimental": false + }, + { + "name": "sum_over_time", + "experimental": false + } + ] +} diff --git a/tools/promql-smoke/run.py b/tools/promql-smoke/run.py new file mode 100644 index 00000000..6a952fb9 --- /dev/null +++ b/tools/promql-smoke/run.py @@ -0,0 +1,273 @@ +#!/usr/bin/env python3 +"""Run small, explicit fixtures for every Prometheus 3.5 aggregation and rollup.""" +import argparse +import hashlib +import json +import math +from pathlib import Path +import re +import subprocess +import urllib.error +import urllib.parse +import urllib.request +import xml.etree.ElementTree as ET + +HERE = Path(__file__).resolve().parent +FEATURE = "promql-experimental-functions" +SPECIAL_YAML = {"NaN": ".nan", "+Inf": ".inf", "-Inf": "-.inf"} + + +def selector(labels): + name = labels.get("__name__", "") + rest = ",".join( + f"{key}={json.dumps(value)}" + for key, value in sorted(labels.items()) if key != "__name__" + ) + return name + "{" + rest + "}" + + +def save_json(path, value): + path.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") + + +def check_coverage(cases, catalog, verify_source=False): + """Fail if any registered function is missing, even if all present tests pass.""" + if cases["prometheus_version"] != catalog["prometheus_version"]: + raise ValueError("Fixture and catalog versions differ") + ids = [query["id"] for query in cases["queries"]] + if len(ids) != len(set(ids)): + raise ValueError("Duplicate case IDs") + report = {"prometheus_version": catalog["prometheus_version"], "scope": catalog["scope"]} + for category in ("aggregation", "rollup"): + entries = {entry["name"]: entry for entry in catalog[category]} + covered = {} + for query in cases["queries"]: + if query["category"] != category: + continue + name = query["function"] + if name not in entries: + raise ValueError(f"Unregistered {category}: {name}") + if query["experimental"] != entries[name]["experimental"]: + raise ValueError(f"Incorrect experimental flag: {query['id']}") + if not re.search(rf"\b{re.escape(name)}\s*(?:\(|by\b|without\b)", query["expr"]): + raise ValueError(f"Case does not exercise its declared function: {query['id']}") + covered.setdefault(name, []).append(query["id"]) + missing = sorted(entries.keys() - covered.keys()) + if missing: + raise ValueError(f"Missing {category} coverage: {missing}") + report[category] = {"covered": len(covered), "total": len(entries), "cases": covered} + report["source_verified"] = False + if verify_source: + texts = {} + for name, source in catalog["sources"].items(): + with urllib.request.urlopen(source["url"], timeout=30) as response: + raw = response.read() + actual = hashlib.sha256(raw).hexdigest() + if actual != source["sha256"]: + raise ValueError(f"Upstream source hash changed: {source['url']}") + texts[name] = raw.decode() + blocks = re.findall(r'\n\t"([^"]+)": \{(.*?)\n\t\},', texts["functions.go"], re.S) + rollups = { + name: bool(re.search(r"Experimental:\s*true", body)) + for name, body in blocks if "ValueTypeMatrix" in body + } + block = texts["lex.go"].split("// Aggregators.")[1].split("// Keywords.")[0] + aggregates = { + name: name in ("limitk", "limit_ratio") + for name in re.findall(r'"([^"]+)":', block) + } + for category, actual in (("aggregation", aggregates), ("rollup", rollups)): + expected = {entry["name"]: entry["experimental"] for entry in catalog[category]} + if actual != expected: + raise ValueError(f"Catalog does not match official {category} registry") + report["source_verified"] = True + report["case_count"] = len(cases["queries"]) + report["experimental_case_count"] = sum(q["experimental"] for q in cases["queries"]) + return report + + +def expected_samples(query, start=0): + return [ + {"labels": sample["labels"], "value": ( + sample["value"] + start if query.get("value_is_timestamp") else sample["value"] + )} + for sample in query["expected"] + ] + + +def generate(cases, out): + lines = [] + previous_metric = None + count = 0 + for series in cases["series"]: + metric = series["labels"]["__name__"] + if metric != previous_metric: + family, kind = (metric[:-6], "counter") if metric.endswith("_total") else (metric, "gauge") + lines.append(f"# TYPE {family} {kind}") + previous_metric = metric + for index, value in enumerate(series["values"]): + if value is None: + continue # A missing sample is not a zero-valued observation. + timestamp = cases["start"] + index * cases["interval"] + lines.append(f'{selector(series["labels"])} {value} {timestamp}') + count += 1 + (out / "samples.openmetrics").write_text("\n".join(lines + ["# EOF", ""])) + inputs = [ + {"series": selector(series["labels"]), "values": " ".join( + "_" if value is None else str(value) for value in series["values"] + )} + for series in cases["series"] + ] + suites = [] + for experimental in (False, True): + groups = [] + for query in cases["queries"]: + if query["experimental"] != experimental: + continue + groups.append({ + "name": query["id"], "interval": f'{cases["interval"]}s', + "input_series": inputs, + "promql_expr_test": [{ + "expr": query.get("promtool_expr", query["expr"]), + "eval_time": f'{cases["eval_offset"]}s', + "exp_samples": [ + {"labels": selector(s["labels"]), "value": s["value"]} + for s in query.get("promtool_expected", expected_samples(query)) + ], + }], + }) + path = out / ("experimental.test.yml" if experimental else "rules.test.yml") + # JSON is YAML; special float expectations require YAML numeric scalars. + rendered = json.dumps({"fuzzy_compare": True, "tests": groups}, indent=2) + for value, yaml in SPECIAL_YAML.items(): + rendered = rendered.replace(f'"value": "{value}"', f'"value": {yaml}') + path.write_text(rendered + "\n") + suites.append((path, experimental, len(groups))) + print(f"Generated {len(cases['series'])} series / {count} samples / {len(cases['queries'])} queries in {out}", flush=True) + return suites + + +def reference_checks(promtool, suites, out, version): + version_run = subprocess.run([promtool, "--version"], capture_output=True, text=True, check=True) + version_text = version_run.stdout + version_run.stderr + if not re.search(rf"version {re.escape(version)}(?:\s|\(|,|$)", version_text): + raise ValueError(f"Expected promtool {version}, got: {version_text.strip()}") + results = [] + for path, experimental, count in suites: + command = [promtool] + if experimental: + command.append(f"--enable-feature={FEATURE}") + command += ["test", "rules", f"--junit={path.with_suffix('.xml')}", str(path)] + run = subprocess.run(command, capture_output=True, text=True) + log = run.stdout + run.stderr + path.with_suffix(".log").write_text(log) + status = "PASS" if run.returncode == 0 else "FAIL" + if run.returncode: + print(log, flush=True) + case_results = [] + junit = path.with_suffix(".xml") + if junit.exists(): + for case in ET.parse(junit).iter("testcase"): + passed = not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) + case_results.append({"id": case.attrib["name"], "status": "PASS" if passed else "FAIL"}) + if len(case_results) != count or any(row["status"] != "PASS" for row in case_results): + status = "FAIL" + print(f"Prometheus {path.name}: {status} ({count} cases)", flush=True) + results.append({"suite": path.name, "case_count": count, "status": status, "cases": case_results, + "command": command, "exit_code": run.returncode, "log": log}) + report = {"version": version_text.strip(), "suites": results} + save_json(out / "reference-results.json", report) + return all(row["status"] == "PASS" for row in results) + + +def compare_vector(body, expected, evaluation, ordered=False): + if body.get("status") != "success": + raise ValueError(f"Query error: {body.get('errorType')}: {body.get('error')}") + if body["data"]["resultType"] != "vector": + raise ValueError(f"Expected vector, got {body['data']['resultType']}") + actual = body["data"]["result"] + key = lambda labels: tuple(sorted(labels.items())) + wanted = {key(sample["labels"]): sample["value"] for sample in expected} + if len(actual) != len(wanted): + raise ValueError(f"Expected {len(wanted)} series, got {len(actual)}") + seen = set() + for series in actual: + labels = key(series["metric"]) + if labels in seen or labels not in wanted: + raise ValueError(f"Unexpected or duplicate labels: {labels}") + seen.add(labels) + timestamp, value = series["value"] + if float(timestamp) != evaluation: + raise ValueError(f"Expected timestamp {evaluation}, got {timestamp}") + actual_value, reference_value = float(value), float(wanted[labels]) + equal = ( + math.isnan(actual_value) and math.isnan(reference_value) + if math.isnan(reference_value) + else math.isclose(actual_value, reference_value, rel_tol=1e-12, abs_tol=1e-12) + ) + if not equal: + raise ValueError(f"{labels}: expected {reference_value}, got {actual_value}") + if ordered and [key(s["metric"]) for s in actual] != [key(s["labels"]) for s in expected]: + raise ValueError("Series order differs") + + +def backend_checks(base_url, cases, out): + evaluation = cases["start"] + cases["eval_offset"] + results = [] + for query in cases["queries"]: + expected = expected_samples(query, cases["start"]) + record = {"id": query["id"], "query": query["expr"], "expected": expected} + url = base_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode( + {"query": query["expr"], "time": evaluation} + ) + try: + try: + response = urllib.request.urlopen(url, timeout=15) + except urllib.error.HTTPError as error: + response = error # Retain the actual error body in the report. + with response: + record["http_status"] = response.code + body = json.load(response) + record["response"] = body + if record["http_status"] != 200: + raise ValueError(f"HTTP {record['http_status']}: {body}") + compare_vector(body, expected, evaluation, query.get("ordered", False)) + record["status"] = "PASS" + except (ValueError, KeyError, TypeError, OSError) as error: + record.update(status="FAIL", error=str(error)) + results.append(record) + print(record["status"], query["id"], record.get("error", ""), flush=True) + save_json(out / "backend-results.json", results) + print("Exact HTTP result checks only; inspect saved responses for execution/fallback provenance.") + return all(row["status"] == "PASS" for row in results) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, default=Path("/tmp/asap-promql-smoke")) + parser.add_argument("--promtool", help="Run both stable and experimental official expression suites") + parser.add_argument("--backend-url", help="Query a backend already loaded with these samples") + parser.add_argument("--verify-catalog", action="store_true", help="Verify pinned official registry source hashes online") + args = parser.parse_args() + cases = json.loads((HERE / "cases.json").read_text()) + catalog = json.loads((HERE / "catalog.json").read_text()) + out = args.output_dir.resolve() + out.mkdir(parents=True, exist_ok=True) + coverage = check_coverage(cases, catalog, args.verify_catalog) + save_json(out / "coverage.json", coverage) + print("Coverage: " + ", ".join( + f"{coverage[category]['covered']}/{coverage[category]['total']} {category}" + for category in ("aggregation", "rollup") + )) + suites = generate(cases, out) + success = True + if args.promtool: + success = reference_checks(args.promtool, suites, out, catalog["prometheus_version"]) + if args.backend_url: + success = backend_checks(args.backend_url, cases, out) and success + if not success: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tools/promql-smoke/test_runner.py b/tools/promql-smoke/test_runner.py new file mode 100644 index 00000000..07c6e37c --- /dev/null +++ b/tools/promql-smoke/test_runner.py @@ -0,0 +1,90 @@ +"""Regression checks for failures that could otherwise turn a mismatch into a pass.""" +import copy +import json +from pathlib import Path +import unittest + +from run import check_coverage, compare_vector, expected_samples + +HERE = Path(__file__).resolve().parent + + +def vector(*samples): + return {"status": "success", "data": {"resultType": "vector", "result": [ + {"metric": labels, "value": [1788825840, str(value)]} + for labels, value in samples + ]}} + + +class ComparatorTests(unittest.TestCase): + def test_nonfinite_values(self): + for value in ("NaN", "+Inf", "-Inf"): + compare_vector(vector(({}, value)), [{"labels": {}, "value": value}], 1788825840) + for actual, wanted in (("NaN", 0), (0, "NaN"), ("-Inf", "+Inf"), (0, "+Inf")): + with self.subTest(actual=actual, wanted=wanted), self.assertRaises(ValueError): + compare_vector(vector(({}, actual)), [{"labels": {}, "value": wanted}], 1788825840) + + def test_empty_is_not_zero(self): + compare_vector(vector(), [], 1788825840) + with self.assertRaises(ValueError): + compare_vector(vector(({}, 0)), [], 1788825840) + + def test_checks_every_series_and_full_labels(self): + expected = [{"labels": {"job": "a"}, "value": 5}, {"labels": {"job": "b"}, "value": 50}] + bad_results = [ + vector(({"job": "a"}, 5), ({"job": "b"}, 51)), + vector(({"job": "a"}, 5), ({"job": "a"}, 50)), + vector(({"job": "a"}, 5), ({"job": "b", "__name__": "wrong"}, 50)), + ] + for body in bad_results: + with self.subTest(body=body), self.assertRaises(ValueError): + compare_vector(body, expected, 1788825840) + + def test_timestamp_value_uses_epoch_but_sample_time_is_evaluation(self): + query = {"value_is_timestamp": True, "expected": [{"labels": {"job": "a"}, "value": 120}]} + expected = expected_samples(query, 1788825600) + self.assertEqual(expected[0]["value"], 1788825720) + compare_vector(vector(({"job": "a"}, 1788825720)), expected, 1788825840) + with self.assertRaises(ValueError): + compare_vector(vector(({"job": "a"}, 120)), expected, 1788825840) + wrong_time = vector(({"job": "a"}, 1788825720)) + wrong_time["data"]["result"][0]["value"][0] -= 60 + with self.assertRaises(ValueError): + compare_vector(wrong_time, expected, 1788825840) + + def test_topk_order_is_only_checked_when_requested(self): + expected = [{"labels": {"job": "b"}, "value": 50}, {"labels": {"job": "a"}, "value": 5}] + reverse = vector(({"job": "a"}, 5), ({"job": "b"}, 50)) + compare_vector(reverse, expected, 1788825840) + with self.assertRaises(ValueError): + compare_vector(reverse, expected, 1788825840, ordered=True) + + def test_error_or_wrong_type_does_not_pass_as_empty(self): + for body in ({"status": "error", "error": "unsupported"}, + {"status": "success", "data": {"resultType": "scalar", "result": [0, "0"]}}): + with self.subTest(body=body), self.assertRaises(ValueError): + compare_vector(body, [], 1788825840) + + +class CoverageTests(unittest.TestCase): + def setUp(self): + self.cases = json.loads((HERE / "cases.json").read_text()) + self.catalog = json.loads((HERE / "catalog.json").read_text()) + + def test_missing_function_cannot_be_reported_as_complete(self): + self.cases["queries"] = [q for q in self.cases["queries"] if q["function"] != "rate"] + with self.assertRaisesRegex(ValueError, "Missing rollup coverage"): + check_coverage(self.cases, self.catalog) + + def test_experimental_flag_and_duplicate_ids_are_checked(self): + bad = copy.deepcopy(self.cases) + next(q for q in bad["queries"] if q["function"] == "limitk")["experimental"] = False + with self.assertRaisesRegex(ValueError, "experimental flag"): + check_coverage(bad, self.catalog) + self.cases["queries"].append(self.cases["queries"][0]) + with self.assertRaisesRegex(ValueError, "Duplicate case IDs"): + check_coverage(self.cases, self.catalog) + + +if __name__ == "__main__": + unittest.main() From b6a540261bb297a46911dc2469c420a9606022eb Mon Sep 17 00:00:00 2001 From: zz_y Date: Sun, 13 Sep 2026 21:53:08 -0600 Subject: [PATCH 03/16] test(promql): satisfy clippy and record PR branch validation --- data_plane/tests/promql_exact_execution.rs | 2 +- tools/promql-smoke/.gitignore | 1 + tools/promql-smoke/RESULTS.md | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) create mode 100644 tools/promql-smoke/.gitignore diff --git a/data_plane/tests/promql_exact_execution.rs b/data_plane/tests/promql_exact_execution.rs index 41a89b08..21f55c4e 100644 --- a/data_plane/tests/promql_exact_execution.rs +++ b/data_plane/tests/promql_exact_execution.rs @@ -108,7 +108,7 @@ fn invalid_raw_snapshots_are_rejected() { .collect(), samples: vec![(1.0, 2.0), (1.0, 3.0)], }; - assert!(execute(&plan, &[series.clone()], 2.0, 300.0).is_err()); + assert!(execute(&plan, std::slice::from_ref(&series), 2.0, 300.0).is_err()); let unique = RawSeries { samples: vec![(1.0, 2.0)], ..series diff --git a/tools/promql-smoke/.gitignore b/tools/promql-smoke/.gitignore new file mode 100644 index 00000000..c18dd8d8 --- /dev/null +++ b/tools/promql-smoke/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tools/promql-smoke/RESULTS.md b/tools/promql-smoke/RESULTS.md index 3acc44a3..f7f7d3a1 100644 --- a/tools/promql-smoke/RESULTS.md +++ b/tools/promql-smoke/RESULTS.md @@ -20,6 +20,8 @@ Run date: 2026-09-13. | Native executor regression tests | 5/5 passed, including the 105-case corpus | | Planner frontend regression/conformance/lowering/equivalence tests | 162/162 passed | | Python comparator/coverage tests | 8/8 passed | +| Backend query parser regression tests | 7/7 passed | +| Targeted Clippy (`-D warnings`) and Cargo format check | Passed | All 14 aggregation operators and 25 range-vector functions in the pinned catalog have an executable exact binding. This measures float-sample function coverage, From 5bf7e8760fcf8813fd9c01dc716e8a8279f4a801 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 04:19:09 +0000 Subject: [PATCH 04/16] test: remove exact execution smoke scope from PR 728 --- Cargo.lock | 54 +- Cargo.toml | 11 +- control_plane/examples/promql_smoke.rs | 117 -- control_plane/src/physical/mod.rs | 3 +- control_plane/src/physical/promql_exact.rs | 382 ---- data_plane/examples/promql_exact_smoke.rs | 164 -- .../query_engines/canonical/exact_promql.rs | 579 ------ data_plane/src/query_engines/canonical/mod.rs | 2 - data_plane/tests/promql_exact_execution.rs | 155 -- tools/promql-smoke/.gitignore | 1 - tools/promql-smoke/README.md | 157 -- tools/promql-smoke/RESULTS.md | 59 - tools/promql-smoke/cases.json | 1849 ----------------- tools/promql-smoke/catalog.json | 174 -- tools/promql-smoke/run.py | 273 --- tools/promql-smoke/test_runner.py | 90 - 16 files changed, 42 insertions(+), 4028 deletions(-) delete mode 100644 control_plane/examples/promql_smoke.rs delete mode 100644 control_plane/src/physical/promql_exact.rs delete mode 100644 data_plane/examples/promql_exact_smoke.rs delete mode 100644 data_plane/src/query_engines/canonical/exact_promql.rs delete mode 100644 data_plane/tests/promql_exact_execution.rs delete mode 100644 tools/promql-smoke/.gitignore delete mode 100644 tools/promql-smoke/README.md delete mode 100644 tools/promql-smoke/RESULTS.md delete mode 100644 tools/promql-smoke/cases.json delete mode 100644 tools/promql-smoke/catalog.json delete mode 100644 tools/promql-smoke/run.py delete mode 100644 tools/promql-smoke/test_runner.py diff --git a/Cargo.lock b/Cargo.lock index 7e1a22ba..99de8350 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -364,9 +364,10 @@ dependencies = [ [[package]] name = "asap-aware-mapping" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f46cbf6c5738db2f4460d419baa8af5572f5276a#f46cbf6c5738db2f4460d419baa8af5572f5276a" dependencies = [ "asap-types", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib)", "serde", "serde_json", "thiserror 2.0.20", @@ -375,7 +376,7 @@ dependencies = [ [[package]] name = "asap-frontend-promql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f46cbf6c5738db2f4460d419baa8af5572f5276a#f46cbf6c5738db2f4460d419baa8af5572f5276a" dependencies = [ "asap-types", "promql-parser 0.10.0 (git+https://github.com/ProjectASAP/promql-parser?rev=9fede7eecca923c9882fe256484d00d37f8706cb)", @@ -384,7 +385,7 @@ dependencies = [ [[package]] name = "asap-frontend-sql" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f46cbf6c5738db2f4460d419baa8af5572f5276a#f46cbf6c5738db2f4460d419baa8af5572f5276a" dependencies = [ "asap-sql-function-catalog", "asap-types", @@ -397,7 +398,7 @@ name = "asap-precompute-rs" version = "0.1.0" source = "git+https://github.com/ProjectASAP/ASAPCollector?branch=main#1d8efd07e40fc151cbd4678a5c6aa9774b1aed34" dependencies = [ - "asap_sketchlib", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", "prost", "serde", "serde_json", @@ -407,12 +408,12 @@ dependencies = [ [[package]] name = "asap-sql-function-catalog" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f46cbf6c5738db2f4460d419baa8af5572f5276a#f46cbf6c5738db2f4460d419baa8af5572f5276a" [[package]] name = "asap-types" version = "0.1.0" -source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f27b16a747e5d7fcd70a5510075c0cd062f0dcea#f27b16a747e5d7fcd70a5510075c0cd062f0dcea" +source = "git+https://github.com/ProjectASAP/ASAPPlanner?rev=f46cbf6c5738db2f4460d419baa8af5572f5276a#f46cbf6c5738db2f4460d419baa8af5572f5276a" dependencies = [ "serde", "serde_json", @@ -447,6 +448,23 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "asap_sketchlib" +version = "0.3.0" +source = "git+https://github.com/ProjectASAP/asap_sketchlib#a66fad6ca21f45b0bef4a3fb32b42d79e887c8f1" +dependencies = [ + "bytes", + "prost", + "rand 0.9.5", + "rmp-serde", + "serde", + "serde-big-array", + "serde_bytes", + "smallvec", + "twox-hash 2.1.4", + "xxhash-rust", +] + [[package]] name = "asap_types" version = "0.1.0" @@ -1144,7 +1162,7 @@ dependencies = [ "asap-precompute-rs", "asap-types", "asap_otel_proto", - "asap_sketchlib", + "asap_sketchlib 0.3.0 (git+https://github.com/ProjectASAP/asap_sketchlib?branch=main)", "asap_types", "async-trait", "axum", @@ -1642,7 +1660,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2073,7 +2091,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.5", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -2259,7 +2277,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -2606,7 +2624,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3184,7 +3202,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.6.5", + "socket2 0.5.10", "thiserror 2.0.20", "tokio", "tracing", @@ -3222,9 +3240,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.5", + "socket2 0.5.10", "tracing", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3500,7 +3518,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3945,7 +3963,7 @@ dependencies = [ "getrandom 0.4.3", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -4457,7 +4475,7 @@ version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" dependencies = [ - "rand 0.10.2", + "rand 0.9.5", ] [[package]] @@ -4720,7 +4738,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0165c059..df62d75f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -18,12 +18,12 @@ version = "0.1.0" asap_sketchlib = { git = "https://github.com/ProjectASAP/asap_sketchlib", branch = "main" } [workspace.dependencies] -# Keep Planner frontends, selection, and IR on the same immutable revision (current-series Planner PR). +# Keep Planner frontends, selection, and IR on the same immutable revision. # Alias upstream asap-types because this workspace also defines asap_types. -planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } -asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } -asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } -asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f27b16a747e5d7fcd70a5510075c0cd062f0dcea" } +planner-types = { package = "asap-types", git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f46cbf6c5738db2f4460d419baa8af5572f5276a" } +asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f46cbf6c5738db2f4460d419baa8af5572f5276a" } +asap-frontend-promql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f46cbf6c5738db2f4460d419baa8af5572f5276a" } +asap-frontend-sql = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "f46cbf6c5738db2f4460d419baa8af5572f5276a" } # Shared external deps (used by 2+ crates) serde = { version = "1.0", features = ["derive"] } @@ -46,4 +46,3 @@ reqwest = { version = "0.12", default-features = false, features = ["json", "rus asap_types = { path = "crates/asap_types" } asap_otel_proto = { path = "crates/asap_otel_proto" } indexmap = { version = "2.0", features = ["serde"] } - diff --git a/control_plane/examples/promql_smoke.rs b/control_plane/examples/promql_smoke.rs deleted file mode 100644 index 2976f4ff..00000000 --- a/control_plane/examples/promql_smoke.rs +++ /dev/null @@ -1,117 +0,0 @@ -//! Run the smoke corpus through the backend's pinned PromQL planner bridge. -//! Binding is a capability diagnostic, not proof of execution or value correctness. -use control_plane::physical::post_asap::{bind_query_expr, PhysicalExpr, PostAsapPlan}; -use control_plane::physical::promql_exact::ExactPromqlPlan; -use control_plane::query_parser::parse_query_expr_canonical; -use planner_types::post_asap::SummaryExpr; -use planner_types::types::AccuracyTarget; -use serde_json::{json, Value}; -use std::collections::BTreeMap; - -fn inspect(query: &Value) -> Value { - let expr = query["expr"].as_str().expect("query expr must be a string"); - let mut row = json!({ - "id": query["id"], "category": query["category"], - "function": query["function"], "experimental": query["experimental"], - "expr": expr, - }); - match parse_query_expr_canonical(expr, AccuracyTarget::Exact) { - Err(error) => { - row["status"] = json!("PARSE_REJECTED"); - row["error"] = json!(error.to_string()); - } - Ok(tree) => { - row["canonical"] = json!(format!("{tree:#?}")); - match bind_query_expr(&tree, AccuracyTarget::Exact) { - Ok(plan) => { - let logical_only = matches!( - &plan, - PhysicalExpr::Committed(PostAsapPlan::Summary(node)) - if matches!(&node.expr, SummaryExpr::KeepPreAsap(_)) - ); - row["status"] = json!(if logical_only { - "LOGICAL_ONLY" - } else { - "BOUND" - }); - row["plan"] = json!(format!("{plan:#?}")); - } - Err(error) => { - row["status"] = json!("BIND_REJECTED"); - row["error"] = json!(error.to_string()); - } - } - } - } - row["summary_status"] = row["status"].take(); - row["summary_plan"] = row["plan"].take(); - row["summary_error"] = row["error"].take(); - match ExactPromqlPlan::bind(expr) { - Ok(plan) => { - row["status"] = json!("BOUND_EXACT"); - row["executor"] = json!("data_plane::query_engines::canonical::exact_promql"); - row["requires"] = json!("raw timestamped float samples"); - row["plan"] = json!(format!("{:#?}", plan.root())); - } - Err(error) => { - row["status"] = json!("EXACT_BIND_REJECTED"); - row["error"] = json!(error.to_string()); - } - } - row -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args().skip(1).collect(); - let json_output = args.iter().any(|arg| arg == "--json"); - let require_bound = args.iter().any(|arg| arg == "--require-bound"); - for arg in &args { - if arg.starts_with("--") && arg != "--json" && arg != "--require-bound" { - return Err(format!("unknown option: {arg}").into()); - } - } - let path = args - .iter() - .find(|arg| !arg.starts_with("--")) - .map(String::as_str) - .unwrap_or("tools/promql-smoke/cases.json"); - let cases: Value = serde_json::from_slice(&std::fs::read(path)?)?; - let mut results = Vec::new(); - let mut counts = BTreeMap::::new(); - for query in cases["queries"].as_array().ok_or("missing queries")? { - // A panic is a failed case; continue to expose the other unsupported queries. - let row = std::panic::catch_unwind(|| inspect(query)).unwrap_or_else( - |_| json!({"id": query["id"], "expr": query["expr"], "status": "PANIC"}), - ); - let status = row["status"].as_str().expect("case status"); - *counts.entry(status.to_owned()).or_default() += 1; - if !json_output { - println!( - "{status} {}: {}{}", - query["id"].as_str().unwrap_or("?"), - query["expr"].as_str().unwrap_or("?"), - row["error"] - .as_str() - .map(|e| format!(" — {e}")) - .unwrap_or_default(), - ); - } - results.push(row); - } - let has_unbound = results.iter().any(|row| row["status"] != "BOUND_EXACT"); - if json_output { - println!( - "{}", - serde_json::to_string_pretty(&json!({ - "accuracy": "Exact", "scope": "Canonical tree to native exact kernels; run promql_exact_execution for values", - "summary": counts, "queries": results, - }))? - ); - } else { - println!("Summary: {counts:?}; binding does not prove execution or correct values."); - } - if require_bound && has_unbound { - std::process::exit(1); - } - Ok(()) -} diff --git a/control_plane/src/physical/mod.rs b/control_plane/src/physical/mod.rs index 8ae2ca9b..fb61d2c4 100644 --- a/control_plane/src/physical/mod.rs +++ b/control_plane/src/physical/mod.rs @@ -13,8 +13,7 @@ pub mod sketch_catalog; pub mod summary_catalog; pub mod workload_cost; +pub mod plan_dot; pub mod publication; pub(crate) mod maintained_population; - -pub mod promql_exact; diff --git a/control_plane/src/physical/promql_exact.rs b/control_plane/src/physical/promql_exact.rs deleted file mode 100644 index 92ae2e56..00000000 --- a/control_plane/src/physical/promql_exact.rs +++ /dev/null @@ -1,382 +0,0 @@ -//! Executable exact kernels for the PromQL float-sample surface. -//! -//! Summary binding cannot implement ordered-window reducers or label-producing -//! operators with the existing five accumulator families. This plan binds each -//! operator to a backend kernel and explicitly requires raw timestamped samples. -use std::rc::Rc; - -use planner_types::pre_asap::{ - AggIntent, CompareOpKind, GroupKeys, QueryExpr, Reduction, SampleKind, ScalarValue, Source, - VectorMatchKind, -}; -use planner_types::types::AccuracyTarget; -use promql_parser::label::Matcher; -use promql_parser::parser::token; - -macro_rules! kernels { - ($name:ident { $($variant:ident => $text:literal),+ $(,)? }) => { - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub enum $name { $($variant),+ } - impl std::str::FromStr for $name { - type Err = anyhow::Error; - fn from_str(name: &str) -> anyhow::Result { - match name { $($text => Ok(Self::$variant),)+ _ => anyhow::bail!("no exact kernel for {name}") } - } - } - } -} -kernels!(AggregateKernel { - Sum => "sum", Avg => "avg", Count => "count", Min => "min", Max => "max", - Group => "group", Stddev => "stddev", Stdvar => "stdvar", TopK => "topk", - BottomK => "bottomk", CountValues => "count_values", Quantile => "quantile", - LimitK => "limitk", LimitRatio => "limit_ratio", -}); -kernels!(RangeKernel { - Avg => "avg_over_time", Min => "min_over_time", Max => "max_over_time", - Sum => "sum_over_time", Count => "count_over_time", Quantile => "quantile_over_time", - Stddev => "stddev_over_time", Stdvar => "stdvar_over_time", Last => "last_over_time", - Present => "present_over_time", Absent => "absent_over_time", Changes => "changes", - Delta => "delta", Deriv => "deriv", IDelta => "idelta", Increase => "increase", - IRate => "irate", PredictLinear => "predict_linear", Rate => "rate", Resets => "resets", - Smoothing => "double_exponential_smoothing", Mad => "mad_over_time", - TsMin => "ts_of_min_over_time", TsMax => "ts_of_max_over_time", TsLast => "ts_of_last_over_time", -}); -kernels!(BinaryKernel { - Add => "+", Sub => "-", Mul => "*", Div => "/", Mod => "%", Pow => "^", - And => "and", Or => "or", Unless => "unless", -}); - -#[derive(Debug, Clone)] -pub struct Grouping { - pub labels: Vec, - pub without: bool, -} - -#[derive(Debug, Clone)] -pub struct ExactSelector { - pub metric: String, - pub matchers: Vec, -} - -#[derive(Debug, Clone)] -pub enum ExactExpr { - Scalar(f64), - Select { - selector: ExactSelector, - range_seconds: Option, - }, - Aggregate { - kernel: AggregateKernel, - parameter: Option, - label: Option, - grouping: Grouping, - input: Box, - }, - Range { - kernel: RangeKernel, - parameters: Vec, - input: Box, - }, - Binary { - kernel: BinaryKernel, - lhs: Box, - rhs: Box, - }, -} - -#[derive(Debug, Clone)] -pub struct ExactPromqlPlan { - /// The canonical frontend result is retained for review and semantic regression checks. - pub canonical: Rc, - root: ExactExpr, -} - -impl ExactPromqlPlan { - pub fn bind(query: &str) -> anyhow::Result { - let canonical = Rc::new(crate::query_parser::parse_query_expr_canonical( - query, - AccuracyTarget::Exact, - )?); - Self::from_canonical(canonical) - } - - /// Bind the planner's canonical tree; execution never reparses the query text. - pub fn from_canonical(canonical: Rc) -> anyhow::Result { - let root = bind_expr(&canonical)?; - anyhow::ensure!( - !matches!( - &root, - ExactExpr::Scalar(_) - | ExactExpr::Select { - range_seconds: Some(_), - .. - } - ), - "exact endpoint requires an instant-vector result" - ); - Ok(Self { canonical, root }) - } - - pub fn root(&self) -> &ExactExpr { - &self.root - } -} - -fn grouping(keys: &GroupKeys, child: &QueryExpr) -> anyhow::Result { - let schema = child.output_schema()?; - let labels = keys - .keys() - .iter() - .map(|index| { - let column = schema - .columns - .get(*index) - .ok_or_else(|| anyhow::anyhow!("invalid grouping column {index}"))?; - anyhow::ensure!( - column.name != "ts" && column.name != "value", - "grouping requires label columns" - ); - Ok(column.name.clone()) - }) - .collect::>>()?; - Ok(Grouping { - labels, - without: keys.is_without(), - }) -} - -fn bind_expr(expr: &QueryExpr) -> anyhow::Result { - match expr { - QueryExpr::PromqlScalarBridge(child) => bind_expr(child), - QueryExpr::Literal(ScalarValue::Float64(value)) => Ok(ExactExpr::Scalar(*value)), - QueryExpr::Literal(ScalarValue::Int64(value)) => Ok(ExactExpr::Scalar(*value as f64)), - QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } => { - let mut matchers = Vec::new(); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - anyhow::bail!("unsupported exact scan predicate") - }; - let (QueryExpr::Column(index), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - anyhow::bail!("exact scan requires literal label matchers") - }; - let column = schema - .columns - .get(*index) - .ok_or_else(|| anyhow::anyhow!("invalid matcher column"))?; - let token = match op { - CompareOpKind::Eq => token::T_EQL, - CompareOpKind::Ne => token::T_NEQ, - CompareOpKind::Regex => token::T_EQL_REGEX, - CompareOpKind::NotRegex => token::T_NEQ_REGEX, - _ => anyhow::bail!("unsupported label comparison"), - }; - matchers.push( - Matcher::new_matcher(token, column.name.clone(), value.clone()) - .map_err(anyhow::Error::msg)?, - ); - } - Ok(ExactExpr::Select { - selector: ExactSelector { - metric: metric.clone(), - matchers, - }, - range_seconds: None, - }) - } - QueryExpr::TimeRange { range, child } => { - let mut input = bind_expr(child)?; - let ExactExpr::Select { range_seconds, .. } = &mut input else { - anyhow::bail!("exact range currently requires a raw selector") - }; - anyhow::ensure!( - range_seconds.is_none(), - "nested raw ranges are not supported" - ); - *range_seconds = Some(range.as_secs_f64()); - Ok(input) - } - QueryExpr::Aggregate { - reduction, - measures, - having: None, - child, - .. - } if measures.len() == 1 => { - let intent = &measures[0]; - let input = Box::new(bind_expr(child)?); - match reduction { - Reduction::PerEntity => { - anyhow::ensure!( - matches!( - input.as_ref(), - ExactExpr::Select { - range_seconds: Some(_), - .. - } - ), - "exact rollup needs a raw range input" - ); - let (kernel, parameters) = range_kernel(intent)?; - Ok(ExactExpr::Range { - kernel, - parameters, - input, - }) - } - Reduction::Reduce(keys) => { - let (kernel, parameter, label) = aggregate_kernel(intent)?; - Ok(ExactExpr::Aggregate { - kernel, - parameter, - label, - grouping: grouping(keys, child)?, - input, - }) - } - } - } - QueryExpr::Limit { - n, - offset: 0, - child, - } => { - let QueryExpr::Sort { - keys, - partition_by, - child: input, - } = child.as_ref() - else { - anyhow::bail!("limit requires a bound value sort") - }; - anyhow::ensure!(keys.len() == 1, "exact topk requires one value sort key"); - let QueryExpr::Column(index) = keys[0].expr else { - anyhow::bail!("topk must sort sample values") - }; - anyhow::ensure!( - input - .output_schema()? - .columns - .get(index) - .is_some_and(|c| c.name == "value"), - "topk must sort sample values" - ); - Ok(ExactExpr::Aggregate { - kernel: if keys[0].ascending { - AggregateKernel::BottomK - } else { - AggregateKernel::TopK - }, - parameter: Some(*n as f64), - label: None, - grouping: grouping(partition_by, input)?, - input: Box::new(bind_expr(input)?), - }) - } - QueryExpr::PromqlSeriesSample { by, kind, child } => { - let (kernel, parameter) = match kind { - SampleKind::LimitK(k) => (AggregateKernel::LimitK, *k as f64), - SampleKind::LimitRatio(r) => (AggregateKernel::LimitRatio, *r), - }; - Ok(ExactExpr::Aggregate { - kernel, - parameter: Some(parameter), - label: None, - grouping: grouping(by, child)?, - input: Box::new(bind_expr(child)?), - }) - } - QueryExpr::BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - if let Some(m) = vector_match { - anyhow::ensure!( - m.kind == VectorMatchKind::Ignoring - && m.labels.is_empty() - && m.grouping.is_none(), - "exact kernel does not implement explicit vector matching" - ); - } - Ok(ExactExpr::Binary { - kernel: op.to_string().to_lowercase().parse()?, - lhs: Box::new(bind_expr(lhs)?), - rhs: Box::new(bind_expr(rhs)?), - }) - } - _ => anyhow::bail!("no executable exact kernel for canonical node {expr:?}"), - } -} - -fn aggregate_kernel( - intent: &AggIntent, -) -> anyhow::Result<(AggregateKernel, Option, Option)> { - use AggregateKernel as K; - let (kernel, parameter, label) = match intent { - AggIntent::Sum { col: None } => (K::Sum, None, None), - AggIntent::Avg { col: None } => (K::Avg, None, None), - AggIntent::Count { .. } => (K::Count, None, None), - AggIntent::Min { col: None } => (K::Min, None, None), - AggIntent::Max { col: None } => (K::Max, None, None), - AggIntent::Group => (K::Group, None, None), - AggIntent::StdDev { - population: true, - col: None, - } => (K::Stddev, None, None), - AggIntent::Variance { - population: true, - col: None, - } => (K::Stdvar, None, None), - AggIntent::Quantile { q, col: None, .. } => (K::Quantile, Some(*q), None), - AggIntent::CountValues { label } => (K::CountValues, None, Some(label.clone())), - _ => anyhow::bail!("no exact aggregation kernel for {intent:?}"), - }; - Ok((kernel, parameter, label)) -} - -fn range_kernel(intent: &AggIntent) -> anyhow::Result<(RangeKernel, Vec)> { - use RangeKernel as K; - Ok(match intent { - AggIntent::Sum { col: None } => (K::Sum, vec![]), - AggIntent::Avg { col: None } => (K::Avg, vec![]), - AggIntent::Count { .. } => (K::Count, vec![]), - AggIntent::Min { col: None } => (K::Min, vec![]), - AggIntent::Max { col: None } => (K::Max, vec![]), - AggIntent::StdDev { - population: true, - col: None, - } => (K::Stddev, vec![]), - AggIntent::Variance { - population: true, - col: None, - } => (K::Stdvar, vec![]), - AggIntent::Quantile { q, col: None, .. } => (K::Quantile, vec![*q]), - AggIntent::LastOverTime => (K::Last, vec![]), - AggIntent::PresentOverTime => (K::Present, vec![]), - AggIntent::AbsentOverTime => (K::Absent, vec![]), - AggIntent::Changes => (K::Changes, vec![]), - AggIntent::Delta => (K::Delta, vec![]), - AggIntent::Deriv => (K::Deriv, vec![]), - AggIntent::IDelta => (K::IDelta, vec![]), - AggIntent::Increase => (K::Increase, vec![]), - AggIntent::IRate => (K::IRate, vec![]), - AggIntent::Rate => (K::Rate, vec![]), - AggIntent::Resets => (K::Resets, vec![]), - AggIntent::PredictLinear { seconds } => (K::PredictLinear, vec![*seconds]), - AggIntent::DoubleExpSmoothing { smoothing, trend } => { - (K::Smoothing, vec![*smoothing, *trend]) - } - AggIntent::MadOverTime => (K::Mad, vec![]), - AggIntent::TsOfMinOverTime => (K::TsMin, vec![]), - AggIntent::TsOfMaxOverTime => (K::TsMax, vec![]), - AggIntent::TsOfLastOverTime => (K::TsLast, vec![]), - _ => anyhow::bail!("no exact range kernel for {intent:?}"), - }) -} diff --git a/data_plane/examples/promql_exact_smoke.rs b/data_plane/examples/promql_exact_smoke.rs deleted file mode 100644 index 0ec221db..00000000 --- a/data_plane/examples/promql_exact_smoke.rs +++ /dev/null @@ -1,164 +0,0 @@ -//! Serve the small raw fixture using real canonical binding and native exact execution. -//! This is a local test server; it does not alter the production store or routing profile. -use axum::{ - extract::{Query, State}, - http::StatusCode, - routing::get, - Json, Router, -}; -use control_plane::physical::promql_exact::ExactPromqlPlan; -use data_plane::query_engines::canonical::exact_promql::{execute, Labels, RawSeries}; -use serde_json::{json, Value}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::Arc, -}; - -struct Snapshot { - data: Vec, - time: f64, -} -type Reply = (StatusCode, Json); -fn error(message: impl ToString) -> Reply { - ( - StatusCode::BAD_REQUEST, - Json(json!({"status":"error","errorType":"bad_data","error":message.to_string()})), - ) -} -fn timestamp( - params: &HashMap, - key: &str, - default: Option, -) -> anyhow::Result { - let value = match params.get(key) { - Some(value) => value.parse::()?, - None => default.ok_or_else(|| anyhow::anyhow!("missing {key}"))?, - }; - anyhow::ensure!(value.is_finite(), "nonfinite {key}"); - Ok(value) -} -fn sample_value(value: f64) -> String { - if value == f64::INFINITY { - "+Inf".into() - } else if value == f64::NEG_INFINITY { - "-Inf".into() - } else { - value.to_string() - } -} -fn result(data: Value) -> Reply { - ( - StatusCode::OK, - Json( - json!({"status":"success","data":data,"infos":["data_source: asap_exact","accuracy: exact","plan: bound canonical kernels"]}), - ), - ) -} -async fn instant( - State(snapshot): State>, - Query(params): Query>, -) -> Reply { - let run = (|| -> anyhow::Result { - let text = params - .get("query") - .ok_or_else(|| anyhow::anyhow!("missing query"))?; - let plan = ExactPromqlPlan::bind(text)?; - let time = timestamp(¶ms, "time", Some(snapshot.time))?; - let rows = execute(&plan, &snapshot.data, time, 300.0)?; - Ok( - json!({"resultType":"vector","result":rows.into_iter().map(|s|json!({"metric":s.labels,"value":[time,sample_value(s.value)]})).collect::>()}), - ) - })(); - match run { - Ok(data) => result(data), - Err(e) => error(e), - } -} -async fn range( - State(snapshot): State>, - Query(params): Query>, -) -> Reply { - let run = (|| -> anyhow::Result { - let text = params - .get("query") - .ok_or_else(|| anyhow::anyhow!("missing query"))?; - let plan = ExactPromqlPlan::bind(text)?; - let start = timestamp(¶ms, "start", None)?; - let end = timestamp(¶ms, "end", None)?; - let step = timestamp(¶ms, "step", None)?; - anyhow::ensure!(end >= start && step > 0.0, "invalid range bounds or step"); - let steps = ((end - start) / step).floor(); - anyhow::ensure!(steps < 11000.0, "too many evaluation steps"); - let count = steps as usize + 1; - let mut rows: BTreeMap> = BTreeMap::new(); - for i in 0..count { - let time = start + i as f64 * step; - for sample in execute(&plan, &snapshot.data, time, 300.0)? { - rows.entry(sample.labels) - .or_default() - .push(json!([time, sample_value(sample.value)])); - } - } - Ok( - json!({"resultType":"matrix","result":rows.into_iter().map(|(labels,values)|json!({"metric":labels,"values":values})).collect::>()}), - ) - })(); - match run { - Ok(data) => result(data), - Err(e) => error(e), - } -} -#[tokio::main] -async fn main() -> anyhow::Result<()> { - let mut args = std::env::args().skip(1); - let fixture = args - .next() - .unwrap_or_else(|| "tools/promql-smoke/cases.json".into()); - let listen = args.next().unwrap_or_else(|| "127.0.0.1:18081".into()); - anyhow::ensure!( - args.next().is_none(), - "usage: promql_exact_smoke [cases.json] [listen-address]" - ); - let cases: Value = serde_json::from_slice(&std::fs::read(fixture)?)?; - let start = cases["start"] - .as_f64() - .ok_or_else(|| anyhow::anyhow!("missing start"))?; - let interval = cases["interval"] - .as_f64() - .ok_or_else(|| anyhow::anyhow!("missing interval"))?; - let time = start - + cases["eval_offset"] - .as_f64() - .ok_or_else(|| anyhow::anyhow!("missing eval_offset"))?; - let mut data = Vec::new(); - for series in cases["series"] - .as_array() - .ok_or_else(|| anyhow::anyhow!("missing series"))? - { - let labels = serde_json::from_value(series["labels"].clone())?; - let samples = series["values"] - .as_array() - .ok_or_else(|| anyhow::anyhow!("missing values"))? - .iter() - .enumerate() - .filter_map(|(i, v)| v.as_f64().map(|v| (start + i as f64 * interval, v))) - .collect(); - data.push(RawSeries { labels, samples }); - } - let app = Router::new() - .route("/api/v1/query", get(instant)) - .route("/api/v1/query_range", get(range)) - .route("/api/v1/health", get(|| async { "ok" })) - .with_state(Arc::new(Snapshot { data, time })); - let listener = tokio::net::TcpListener::bind(&listen).await?; - eprintln!( - "Native exact PromQL smoke server at http://{}", - listener.local_addr()? - ); - axum::serve(listener, app) - .with_graceful_shutdown(async { - let _ = tokio::signal::ctrl_c().await; - }) - .await?; - Ok(()) -} diff --git a/data_plane/src/query_engines/canonical/exact_promql.rs b/data_plane/src/query_engines/canonical/exact_promql.rs deleted file mode 100644 index 4e81f446..00000000 --- a/data_plane/src/query_engines/canonical/exact_promql.rs +++ /dev/null @@ -1,579 +0,0 @@ -//! Execute the control plane's bound exact PromQL kernels over raw float samples. -//! No query is forwarded and no sketch value is substituted for a raw observation. -use std::collections::{BTreeMap, BTreeSet}; - -use control_plane::physical::promql_exact::{ - AggregateKernel as A, BinaryKernel as B, ExactExpr, ExactPromqlPlan, Grouping, RangeKernel as R, -}; -use serde::{Deserialize, Serialize}; - -pub type Labels = BTreeMap; - -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct RawSeries { - pub labels: Labels, - /// Unix seconds, strictly increasing; missing samples are absent from this list. - pub samples: Vec<(f64, f64)>, -} - -#[derive(Debug, Clone)] -pub struct ExactSample { - pub labels: Labels, - pub value: f64, -} - -enum Value { - Scalar(f64), - Vector(Vec), - Matrix { - series: Vec, - seconds: f64, - }, -} -impl Value { - fn vector(self) -> anyhow::Result> { - match self { - Self::Vector(v) => Ok(v), - _ => anyhow::bail!("expected an instant vector"), - } - } -} - -/// Evaluate only the bound program. Raw data must already be a consistent snapshot. -pub fn execute( - plan: &ExactPromqlPlan, - data: &[RawSeries], - evaluation: f64, - lookback: f64, -) -> anyhow::Result> { - anyhow::ensure!( - evaluation.is_finite() && lookback.is_finite() && lookback > 0.0, - "invalid evaluation time/lookback" - ); - let mut seen = BTreeSet::new(); - for series in data { - anyhow::ensure!(seen.insert(&series.labels), "duplicate input label set"); - anyhow::ensure!( - series.samples.iter().all(|(t, _)| t.is_finite()) - && series.samples.windows(2).all(|p| p[0].0 < p[1].0), - "raw samples must have unique increasing finite timestamps" - ); - } - let out = eval(plan.root(), data, evaluation, lookback)?.vector()?; - let mut seen = BTreeSet::new(); - anyhow::ensure!( - out.iter().all(|s| seen.insert(&s.labels)), - "duplicate output label set" - ); - Ok(out) -} - -fn eval(expr: &ExactExpr, data: &[RawSeries], time: f64, lookback: f64) -> anyhow::Result { - match expr { - ExactExpr::Scalar(value) => Ok(Value::Scalar(*value)), - ExactExpr::Select { - selector, - range_seconds, - } => { - let mut selected = Vec::new(); - for series in data { - if series.labels.get("__name__") != Some(&selector.metric) { - continue; - } - if !selector.matchers.iter().all(|m| { - m.is_match(series.labels.get(&m.name).map(String::as_str).unwrap_or("")) - }) { - continue; - } - let seconds = range_seconds.unwrap_or(lookback); - let mut samples: Vec<_> = series - .samples - .iter() - .copied() - .filter(|(t, _)| *t > time - seconds && *t <= time) - .collect(); - if range_seconds.is_none() && !samples.is_empty() { - samples = vec![*samples.last().unwrap()]; - } - if !samples.is_empty() { - selected.push(RawSeries { - labels: series.labels.clone(), - samples, - }); - } - } - selected.sort_by(|a, b| a.labels.cmp(&b.labels)); - Ok(match range_seconds { - Some(seconds) => Value::Matrix { - series: selected, - seconds: *seconds, - }, - None => Value::Vector( - selected - .into_iter() - .map(|s| ExactSample { - labels: s.labels, - value: s.samples[0].1, - }) - .collect(), - ), - }) - } - ExactExpr::Aggregate { - kernel, - parameter, - label, - grouping, - input, - } => { - let rows = eval(input, data, time, lookback)?.vector()?; - Ok(Value::Vector(aggregate( - *kernel, - *parameter, - label.as_deref(), - grouping, - rows, - )?)) - } - ExactExpr::Range { - kernel, - parameters, - input, - } => { - let Value::Matrix { series, seconds } = eval(input, data, time, lookback)? else { - anyhow::bail!("range kernel needs raw range samples") - }; - if *kernel == R::Absent { - let labels = absent_labels(input); - return Ok(Value::Vector(if series.is_empty() { - vec![ExactSample { labels, value: 1.0 }] - } else { - vec![] - })); - } - let mut out = Vec::new(); - for series in series { - if let Some(value) = rollup(*kernel, parameters, &series.samples, time, seconds)? { - let mut labels = series.labels; - if *kernel != R::Last { - labels.remove("__name__"); - } - out.push(ExactSample { labels, value }); - } - } - Ok(Value::Vector(out)) - } - ExactExpr::Binary { kernel, lhs, rhs } => binary( - *kernel, - eval(lhs, data, time, lookback)?, - eval(rhs, data, time, lookback)?, - ), - } -} - -fn absent_labels(input: &ExactExpr) -> Labels { - let mut labels = Labels::new(); - if let ExactExpr::Select { selector, .. } = input { - // Derive a label only when it has a single equality matcher. - let mut counts = BTreeMap::new(); - for m in &selector.matchers { - *counts.entry(&m.name).or_insert(0) += 1; - } - for m in &selector.matchers { - if m.name != "__name__" - && m.op.to_string() == "=" - && counts[&m.name] == 1 - && !m.value.is_empty() - { - labels.insert(m.name.clone(), m.value.clone()); - } - } - } - labels -} - -fn group_key(labels: &Labels, grouping: &Grouping) -> Labels { - labels - .iter() - .filter(|(key, _)| { - let included = grouping.labels.contains(key); - if grouping.without { - key.as_str() != "__name__" && !included - } else { - included - } - }) - .map(|(k, v)| (k.clone(), v.clone())) - .collect() -} - -fn aggregate( - kernel: A, - parameter: Option, - label: Option<&str>, - grouping: &Grouping, - rows: Vec, -) -> anyhow::Result> { - let mut groups: BTreeMap> = BTreeMap::new(); - for mut row in rows { - let mut key = group_key(&row.labels, grouping); - if kernel == A::CountValues { - let label = label.ok_or_else(|| anyhow::anyhow!("count_values requires a label"))?; - // The output value label participates in grouping even with no 'by'. - row.labels.insert(label.into(), float_label(row.value)); - key.insert(label.into(), float_label(row.value)); - } - groups.entry(key).or_default().push(row); - } - let mut out = Vec::new(); - for (labels, mut rows) in groups { - match kernel { - A::TopK | A::BottomK | A::LimitK => { - let k = parameter.unwrap_or(0.0); - anyhow::ensure!(k.is_finite() && k >= 0.0, "invalid selection count"); - if kernel != A::LimitK { - rows.sort_by(|a, b| { - if a.value.is_nan() { - return if b.value.is_nan() { - std::cmp::Ordering::Equal - } else { - std::cmp::Ordering::Greater - }; - } - if b.value.is_nan() { - return std::cmp::Ordering::Less; - } - if kernel == A::TopK { - b.value.total_cmp(&a.value) - } else { - a.value.total_cmp(&b.value) - } - }); - } - out.extend(rows.into_iter().take(k as usize)); - } - A::LimitRatio => { - let ratio = parameter.unwrap_or(0.0).clamp(-1.0, 1.0); - anyhow::ensure!(ratio.is_finite(), "invalid sampling ratio"); - out.extend(rows.into_iter().filter(|row| { - let mut bytes = Vec::new(); - for (key, value) in &row.labels { - bytes.extend(key.as_bytes()); - bytes.push(255); - bytes.extend(value.as_bytes()); - bytes.push(255); - } - let offset = xxhash_rust::xxh64::xxh64(&bytes, 0) as f64 / u64::MAX as f64; - if ratio < 0.0 { - offset >= 1.0 + ratio - } else { - offset < ratio - } - })); - } - _ => { - let values: Vec<_> = rows.iter().map(|s| s.value).collect(); - let value = match kernel { - A::Sum => values.iter().sum(), - A::Avg => mean(&values), - A::Count | A::CountValues => values.len() as f64, - A::Min => minimum(&values), - A::Max => maximum(&values), - A::Group => 1.0, - A::Stdvar => variance(&values), - A::Stddev => variance(&values).sqrt(), - A::Quantile => quantile(&values, parameter.unwrap_or(f64::NAN)), - _ => unreachable!(), - }; - out.push(ExactSample { labels, value }); - } - } - } - Ok(out) -} - -fn float_label(value: f64) -> String { - if value.is_nan() { - "NaN".into() - } else if value == f64::INFINITY { - "+Inf".into() - } else if value == f64::NEG_INFINITY { - "-Inf".into() - } else { - value.to_string() - } -} -fn minimum(values: &[f64]) -> f64 { - values.iter().copied().reduce(f64::min).unwrap_or(f64::NAN) -} -fn maximum(values: &[f64]) -> f64 { - values.iter().copied().reduce(f64::max).unwrap_or(f64::NAN) -} -fn mean(values: &[f64]) -> f64 { - values.iter().sum::() / values.len() as f64 -} -fn variance(values: &[f64]) -> f64 { - let mut mean = 0.0; - let mut m2 = 0.0; - for (i, value) in values.iter().enumerate() { - let d = value - mean; - mean += d / (i + 1) as f64; - m2 += d * (value - mean); - } - m2 / values.len() as f64 -} -fn quantile(values: &[f64], phi: f64) -> f64 { - if phi.is_nan() || values.is_empty() { - return f64::NAN; - } - if phi < 0.0 { - return f64::NEG_INFINITY; - } - if phi > 1.0 { - return f64::INFINITY; - } - let mut sorted = values.to_vec(); - sorted.sort_by(|a, b| match (a.is_nan(), b.is_nan()) { - (true, false) => std::cmp::Ordering::Less, - (false, true) => std::cmp::Ordering::Greater, - _ => a.total_cmp(b), - }); - let rank = phi * (sorted.len() - 1) as f64; - let lower = rank.floor() as usize; - let upper = rank.ceil() as usize; - let weight = rank - lower as f64; - sorted[lower] * (1.0 - weight) + sorted[upper] * weight -} - -fn rollup( - kernel: R, - parameters: &[f64], - samples: &[(f64, f64)], - time: f64, - seconds: f64, -) -> anyhow::Result> { - if samples.is_empty() { - return Ok(None); - } - let values: Vec<_> = samples.iter().map(|s| s.1).collect(); - let first = samples[0]; - let last = *samples.last().unwrap(); - let needs_two = matches!( - kernel, - R::Delta - | R::Deriv - | R::IDelta - | R::Increase - | R::IRate - | R::PredictLinear - | R::Rate - | R::Smoothing - ); - if needs_two && samples.len() < 2 { - return Ok(None); - } - Ok(Some(match kernel { - R::Avg => mean(&values), - R::Min => minimum(&values), - R::Max => maximum(&values), - R::Sum => values.iter().sum(), - R::Count => values.len() as f64, - R::Quantile => quantile(&values, parameters[0]), - R::Stddev => variance(&values).sqrt(), - R::Stdvar => variance(&values), - R::Last => last.1, - R::Present => 1.0, - R::Changes => values - .windows(2) - .filter(|p| p[0] != p[1] && !(p[0].is_nan() && p[1].is_nan())) - .count() as f64, - R::Resets => values.windows(2).filter(|p| p[1] < p[0]).count() as f64, - R::IDelta => last.1 - samples[samples.len() - 2].1, - R::IRate => { - let previous = samples[samples.len() - 2]; - let delta = if last.1 < previous.1 { - last.1 - } else { - last.1 - previous.1 - }; - delta / (last.0 - previous.0) - } - R::Rate | R::Increase | R::Delta => { - let counter = kernel != R::Delta; - let mut difference = last.1 - first.1; - if counter { - for pair in values.windows(2) { - if pair[1] < pair[0] { - difference += pair[0]; - } - } - } - let observed = last.0 - first.0; - let interval = observed / (samples.len() - 1) as f64; - let mut before = first.0 - (time - seconds); - let mut after = time - last.0; - if before >= interval * 1.1 { - before = interval / 2.0; - } - if after >= interval * 1.1 { - after = interval / 2.0; - } - if counter && difference > 0.0 && first.1 >= 0.0 { - before = before.min(observed * first.1 / difference); - } - difference * ((observed + before + after) / observed) - / if kernel == R::Rate { seconds } else { 1.0 } - } - R::Deriv | R::PredictLinear => { - // Center timestamps near the window to avoid losing precision on Unix time. - let xs: Vec<_> = samples.iter().map(|s| s.0 - time).collect(); - let mx = mean(&xs); - let my = mean(&values); - let slope = xs - .iter() - .zip(&values) - .map(|(x, y)| (x - mx) * (y - my)) - .sum::() - / xs.iter().map(|x| (x - mx).powi(2)).sum::(); - if kernel == R::Deriv { - slope - } else { - my + slope * (parameters[0] - mx) - } - } - R::Smoothing => { - let (sf, tf) = (parameters[0], parameters[1]); - anyhow::ensure!( - sf > 0.0 && sf < 1.0 && tf > 0.0 && tf < 1.0, - "invalid smoothing/trend factor" - ); - let mut level = first.1; - let mut previous = 0.0; - let mut trend = values[1] - values[0]; - for (i, value) in values.iter().enumerate().skip(1) { - if i > 1 { - trend = tf * (level - previous) + (1.0 - tf) * trend; - } - previous = level; - level = sf * value + (1.0 - sf) * (level + trend); - } - level - } - R::Mad => { - let median = quantile(&values, 0.5); - let deviations: Vec<_> = values.iter().map(|v| (v - median).abs()).collect(); - quantile(&deviations, 0.5) - } - R::TsLast => last.0, - R::TsMin | R::TsMax => { - let mut selected = first; - for sample in samples.iter().copied().skip(1) { - if selected.1.is_nan() - || (kernel == R::TsMin && sample.1 <= selected.1) - || (kernel == R::TsMax && sample.1 >= selected.1) - { - selected = sample; - } - } - selected.0 - } - R::Absent => unreachable!(), - })) -} - -fn arithmetic(kernel: B, a: f64, b: f64) -> anyhow::Result { - Ok(match kernel { - B::Add => a + b, - B::Sub => a - b, - B::Mul => a * b, - B::Div => a / b, - B::Mod => a % b, - B::Pow => a.powf(b), - _ => anyhow::bail!("set operator needs two vectors"), - }) -} -fn matching_key(labels: &Labels) -> Labels { - let mut key = labels.clone(); - key.remove("__name__"); - key -} -fn binary(kernel: B, lhs: Value, rhs: Value) -> anyhow::Result { - let vector = match (lhs, rhs) { - (Value::Scalar(a), Value::Scalar(b)) => { - return Ok(Value::Scalar(arithmetic(kernel, a, b)?)) - } - (Value::Vector(rows), Value::Scalar(scalar)) - | (Value::Scalar(scalar), Value::Vector(rows)) - if matches!(kernel, B::Add | B::Mul) => - { - rows.into_iter() - .map(|mut row| { - row.value = arithmetic(kernel, row.value, scalar)?; - row.labels.remove("__name__"); - Ok(row) - }) - .collect::>>()? - } - (Value::Vector(rows), Value::Scalar(scalar)) => rows - .into_iter() - .map(|mut row| { - row.value = arithmetic(kernel, row.value, scalar)?; - row.labels.remove("__name__"); - Ok(row) - }) - .collect::>>()?, - (Value::Scalar(scalar), Value::Vector(rows)) => rows - .into_iter() - .map(|mut row| { - row.value = arithmetic(kernel, scalar, row.value)?; - row.labels.remove("__name__"); - Ok(row) - }) - .collect::>>()?, - (Value::Vector(left), Value::Vector(right)) => { - let left_keys: BTreeSet<_> = left.iter().map(|s| matching_key(&s.labels)).collect(); - let right_keys: BTreeSet<_> = right.iter().map(|s| matching_key(&s.labels)).collect(); - match kernel { - B::Or => left - .into_iter() - .chain( - right - .into_iter() - .filter(|r| !left_keys.contains(&matching_key(&r.labels))), - ) - .collect(), - B::And => left - .into_iter() - .filter(|s| right_keys.contains(&matching_key(&s.labels))) - .collect(), - B::Unless => left - .into_iter() - .filter(|s| !right_keys.contains(&matching_key(&s.labels))) - .collect(), - _ => { - anyhow::ensure!( - left_keys.len() == left.len() && right_keys.len() == right.len(), - "non-unique vector match" - ); - let right: BTreeMap<_, _> = right - .into_iter() - .map(|s| (matching_key(&s.labels), s.value)) - .collect(); - let mut out = Vec::new(); - for row in left { - let key = matching_key(&row.labels); - if let Some(value) = right.get(&key) { - out.push(ExactSample { - labels: key, - value: arithmetic(kernel, row.value, *value)?, - }); - } - } - out - } - } - } - _ => anyhow::bail!("binary operator cannot consume a range vector"), - }; - Ok(Value::Vector(vector)) -} diff --git a/data_plane/src/query_engines/canonical/mod.rs b/data_plane/src/query_engines/canonical/mod.rs index 67fba694..c170e5d0 100644 --- a/data_plane/src/query_engines/canonical/mod.rs +++ b/data_plane/src/query_engines/canonical/mod.rs @@ -20,5 +20,3 @@ pub mod result { pub mod sds_resolver { pub use asap_types::sds::{DataDescriptorId, SummaryDescriptorId}; } - -pub mod exact_promql; diff --git a/data_plane/tests/promql_exact_execution.rs b/data_plane/tests/promql_exact_execution.rs deleted file mode 100644 index 21f55c4e..00000000 --- a/data_plane/tests/promql_exact_execution.rs +++ /dev/null @@ -1,155 +0,0 @@ -//! The same hand-checked fixture used by official promtool must execute locally. -use control_plane::physical::promql_exact::ExactPromqlPlan; -use data_plane::query_engines::canonical::exact_promql::{execute, Labels, RawSeries}; -use serde_json::Value; - -fn number(value: &Value) -> f64 { - value - .as_f64() - .unwrap_or_else(|| value.as_str().unwrap().parse().unwrap()) -} - -/// Every smoke query must bind to a real kernel and produce the official labels and values. -#[test] -fn all_smoke_queries_bind_and_execute_exactly() { - let cases: Value = - serde_json::from_str(include_str!("../../tools/promql-smoke/cases.json")).unwrap(); - let start = cases["start"].as_f64().unwrap(); - let interval = cases["interval"].as_f64().unwrap(); - let evaluation = start + cases["eval_offset"].as_f64().unwrap(); - let data: Vec<_> = cases["series"] - .as_array() - .unwrap() - .iter() - .map(|series| RawSeries { - labels: serde_json::from_value(series["labels"].clone()).unwrap(), - samples: series["values"] - .as_array() - .unwrap() - .iter() - .enumerate() - .filter_map(|(i, v)| v.as_f64().map(|value| (start + i as f64 * interval, value))) - .collect(), - }) - .collect(); - let mut failures = Vec::new(); - for query in cases["queries"].as_array().unwrap() { - let id = query["id"].as_str().unwrap(); - let expr = query["expr"].as_str().unwrap(); - let result = (|| -> anyhow::Result<()> { - let plan = ExactPromqlPlan::bind(expr)?; - let actual = execute(&plan, &data, evaluation, 300.0)?; - let expected = query["expected"].as_array().unwrap(); - anyhow::ensure!( - actual.len() == expected.len(), - "series count {} != {}", - actual.len(), - expected.len() - ); - for (index, sample) in expected.iter().enumerate() { - let labels: Labels = serde_json::from_value(sample["labels"].clone())?; - let value = number(&sample["value"]) - + if query["value_is_timestamp"] == true { - start - } else { - 0.0 - }; - let got = actual - .iter() - .find(|s| s.labels == labels) - .ok_or_else(|| anyhow::anyhow!("missing labels {labels:?}; got {actual:?}"))?; - let equal = if value.is_nan() { - got.value.is_nan() - } else if value.is_infinite() { - got.value == value - } else { - (got.value - value).abs() <= 1e-12 + 1e-12 * value.abs() - }; - anyhow::ensure!(equal, "{labels:?}: {} != {value}", got.value); - if query["ordered"] == true { - anyhow::ensure!(actual[index].labels == labels, "wrong series order"); - } - } - Ok(()) - })(); - match result { - Ok(()) => println!("PASS {id}"), - Err(error) => failures.push(format!("{id}: {expr}: {error}")), - } - } - assert!( - failures.is_empty(), - "{} queries failed:\n{}", - failures.len(), - failures.join("\n") - ); -} - -/// Unsupported shape modifiers must be rejected during binding, never ignored by execution. -#[test] -fn unsupported_exact_shapes_are_not_bound() { - for query in [ - "sum(smoke_gauge offset 1m)", - "sum(smoke_gauge @ 100)", - "sum_over_time(smoke_gauge[5m:1m])", - "smoke_gauge + on(job) smoke_gauge", - ] { - assert!(ExactPromqlPlan::bind(query).is_err(), "must reject {query}"); - } -} - -/// Duplicate timestamps and duplicate label identities cannot enter the evaluator silently. -#[test] -fn invalid_raw_snapshots_are_rejected() { - let plan = ExactPromqlPlan::bind("sum(smoke_gauge)").unwrap(); - let series = RawSeries { - labels: [("__name__".into(), "smoke_gauge".into())] - .into_iter() - .collect(), - samples: vec![(1.0, 2.0), (1.0, 3.0)], - }; - assert!(execute(&plan, std::slice::from_ref(&series), 2.0, 300.0).is_err()); - let unique = RawSeries { - samples: vec![(1.0, 2.0)], - ..series - }; - assert!(execute(&plan, &[unique.clone(), unique], 2.0, 300.0).is_err()); -} - -/// Counting two series with equal numeric values must return two, not one distinct value. -#[test] -fn count_preserves_equal_valued_series_multiplicity() { - let plan = ExactPromqlPlan::bind("count(smoke_gauge)").unwrap(); - let data: Vec<_> = ["a", "b"] - .into_iter() - .map(|job| RawSeries { - labels: [ - ("__name__".into(), "smoke_gauge".into()), - ("job".into(), job.into()), - ] - .into_iter() - .collect(), - samples: vec![(240.0, 5.0)], - }) - .collect(); - let result = execute(&plan, &data, 240.0, 300.0).unwrap(); - assert_eq!(result.len(), 1); - assert_eq!(result[0].value, 2.0); -} - -/// Negative ratios select from the upper end: -1 must retain the entire input. -#[test] -fn negative_full_ratio_keeps_every_series() { - let plan = ExactPromqlPlan::bind("limit_ratio(-1, smoke_gauge)").unwrap(); - let data = vec![RawSeries { - labels: [ - ("__name__".into(), "smoke_gauge".into()), - ("job".into(), "a".into()), - ] - .into_iter() - .collect(), - samples: vec![(240.0, 5.0)], - }]; - let result = execute(&plan, &data, 240.0, 300.0).unwrap(); - assert_eq!(result.len(), 1); -} diff --git a/tools/promql-smoke/.gitignore b/tools/promql-smoke/.gitignore deleted file mode 100644 index c18dd8d8..00000000 --- a/tools/promql-smoke/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/tools/promql-smoke/README.md b/tools/promql-smoke/README.md deleted file mode 100644 index fedd2b9d..00000000 --- a/tools/promql-smoke/README.md +++ /dev/null @@ -1,157 +0,0 @@ -# PromQL aggregation and rollup smoke tests - -Small fixtures for **Prometheus 3.5.0**: **14 aggregation operators**, **25 range-vector -functions**, **105 queries**, **14 series**, **64 samples**. Every query has explicit -expected labels and values in [cases.json](cases.json), plus a note explaining the -case. `null` means a missing sample, not zero. - -“Rollup” here means every registered Prometheus function accepting a -`ValueTypeMatrix` argument. This is function-name coverage using float samples, -not full PromQL conformance: native histograms, mixed sample types, staleness, -and all combinations of modifiers are not covered. Instant-vector histogram -helpers and scalar/math/label functions are outside this scope. - -The catalog is extracted from the **v3.5.0** official parser registry, with source -URLs and SHA-256 hashes in [catalog.json](catalog.json). Generation fails if any -catalog entry has no case. `--verify-catalog` also downloads the pinned sources -and verifies both their hashes and their registered names against the catalog. - -| Category | Functions | -| --- | --- | -| Aggregations | `sum`, `avg`, `count`, `min`, `max`, `group`, `stddev`, `stdvar`, `topk`, `bottomk`, `count_values`, `quantile` | -| Experimental aggregations | `limitk`, `limit_ratio` | -| Time-window aggregations | `avg_over_time`, `min_over_time`, `max_over_time`, `sum_over_time`, `count_over_time`, `quantile_over_time`, `stddev_over_time`, `stdvar_over_time`, `last_over_time`, `present_over_time` | -| Other range-vector functions | `absent_over_time`, `changes`, `delta`, `deriv`, `idelta`, `increase`, `irate`, `predict_linear`, `rate`, `resets` | -| Experimental range-vector functions | `double_exponential_smoothing`, `mad_over_time`, `ts_of_min_over_time`, `ts_of_max_over_time`, `ts_of_last_over_time` | - -The extra cases cover empty inputs, grouping, repeated values, sparse sampling, -single-sample ranges, counter resets and zero-point extrapolation, left-open -window boundaries, interpolated p99, tied timestamp extrema, and nonfinite values. -The two original gauges remain `1,2,3,4,5` and `10,20,30,40,50`. - -## Run official reference tests - -From the repository root, using the official **3.5.0** promtool binary: - -```bash -python3 tools/promql-smoke/run.py --promtool /path/to/promtool --verify-catalog -``` - -This checks the binary version and runs both suites, enabling -`promql-experimental-functions` only for the experimental suite. Ordinary runs -can omit `--verify-catalog` to work offline. Outputs default to -`/tmp/asap-promql-smoke`; change that with `--output-dir`. - -If using Docker, generate once and run both suites: - -```bash -python3 tools/promql-smoke/run.py --verify-catalog - -docker run --rm -v /tmp/asap-promql-smoke:/tests:ro --entrypoint promtool \ - prom/prometheus:v3.5.0 test rules /tests/rules.test.yml - -docker run --rm -v /tmp/asap-promql-smoke:/tests:ro --entrypoint promtool \ - prom/prometheus:v3.5.0 --enable-feature=promql-experimental-functions \ - test rules /tests/experimental.test.yml -``` - -The Python `--promtool` runner additionally saves per-case JUnit results, logs, -and `reference-results.json`. It returns nonzero on failure. Expected finite -values use promtool's one-bit floating-point tolerance. - -Promtool 3.5 does not consider NaN equal to NaN. That one reference case uses -`x != bool x` to prove the result is NaN; its original expression and raw NaN -expectations are retained for planner and HTTP tests. The override is explicit -in `cases.json`. The infinity cases compare raw values directly. - -## Bind and execute through ASAPPlanner and the backend - -The workspace pins the published planner commit -`f27b16a747e5d7fcd70a5510075c0cd062f0dcea` -([ASAPPlanner PR #413](https://github.com/ProjectASAP/ASAPPlanner/pull/413)). -This commit applies PR #413 on top of the backend’s existing planner pin -`029ff2fe041172c94c2d32c90b185bc83c5e8a57`, preserving its interfaces. -No adjacent planner checkout or local Cargo patch is required. The planner -preserves `irate`, series-count semantics, and special quantile parameters in -the canonical tree. - -```bash -cargo +1.98.0 test --locked -p data_plane --test promql_exact_execution -cargo +1.98.0 run --locked -p control_plane --example promql_smoke -- --require-bound - -# Save canonical trees, exact plans, and the original summary binder diagnostics. -cargo +1.98.0 run --locked -p control_plane --example promql_smoke -- \ - --json --require-bound > /tmp/asap-promql-smoke/planner-results.json -``` - -`ExactPromqlPlan::bind` calls the backend's `parse_query_expr_canonical` with -`AccuracyTarget::Exact`, then compiles that canonical tree into typed native -kernels. `BOUND_EXACT` means an executable exact plan; unsupported shapes are -`EXACT_BIND_REJECTED`. `--require-bound` fails on any rejection. Original sketch -binder diagnostics remain under `summary_status`; they do not determine exact -execution support. - -The Rust execution test evaluates all 105 queries against raw timestamped float -samples and compares full labels, values, and requested ordering with the same -fixture expectations verified by official Prometheus. Additional regression tests -cover invalid snapshots, unsupported modifiers, equal-valued series counts, and -negative sampling ratios. - -## Run the local native HTTP smoke server - -In one terminal: - -```bash -cargo +1.98.0 run --locked -p data_plane --example promql_exact_smoke -``` - -In another: - -```bash -python3 tools/promql-smoke/run.py --backend-url http://127.0.0.1:18081 -``` - -This example loads `cases.json` directly and serves `/api/v1/query` and -`/api/v1/query_range` using the canonical binder and backend exact executor. -It performs no Prometheus forwarding. Responses identify `data_source: asap_exact`. -Optional positional arguments are the fixture path and listening address. -This is a local test entry point; production storage and routing are not wired -into this new raw-sample execution path. Exact plans require raw samples, which -cannot in general be reconstructed from sketches. - -## Compare backend HTTP results - -After loading `samples.openmetrics` through the deployment's ingest path: - -```bash -python3 tools/promql-smoke/run.py --backend-url http://127.0.0.1:8080 -``` - -The runner does **not** load data or install a plan into the backend. It queries -all cases, including experimental ones, and reports unsupported queries as -failures. Fixture timestamps are in seconds, beginning at `1788825600`, sampled -every 60 seconds, and evaluated at `1788825840`. The official promtool suite uses -relative times starting at zero; `ts_of_*` expected values are shifted to epoch -time for HTTP comparisons. - -The comparator checks all labels including metric names, result type, the full -series set, evaluation timestamps, finite values (`rtol=atol=1e-12`), NaN/Inf, -and explicitly requested topk/bottomk ordering. Missing series are not replaced -with zero. Full responses, including provenance annotations, are saved in -`backend-results.json`. A matching response can still be a fallback; inspect its -provenance separately. Approximate sketch answers can fail these exact checks. - -## Validation - -```bash -python3 -m unittest discover -s tools/promql-smoke -p 'test_*.py' -v -``` - -[RESULTS.md](RESULTS.md) records the observed official and planner results from -2026-09-13. It is a snapshot, not a substitute for rerunning after changes. - -Official sources: -[operators](https://prometheus.io/docs/prometheus/3.5/querying/operators/), -[functions](https://prometheus.io/docs/prometheus/3.5/querying/functions/), -[function registry](https://github.com/prometheus/prometheus/blob/v3.5.0/promql/parser/functions.go), -[aggregation registry](https://github.com/prometheus/prometheus/blob/v3.5.0/promql/parser/lex.go). diff --git a/tools/promql-smoke/RESULTS.md b/tools/promql-smoke/RESULTS.md deleted file mode 100644 index f7f7d3a1..00000000 --- a/tools/promql-smoke/RESULTS.md +++ /dev/null @@ -1,59 +0,0 @@ -# Recorded smoke results - -Run date: 2026-09-13. - -- Fixture SHA-256: `7df76b148e7b1d4c11260f23679d5138d1bb6c14ededf9ff8d1d9415f8146eeb`. -- Reference: official Prometheus/promtool 3.5.0 (`8be3a9560fbdd18a94dedec4b747c35178177202`). -- Backend PR base: `8cf1890b` on `origin/main`. -- Planner dependency: published commit `f27b16a747e5d7fcd70a5510075c0cd062f0dcea` - (the fix from [PR #413](https://github.com/ProjectASAP/ASAPPlanner/pull/413) - applied to the existing backend planner pin); no local path patch. - -| Check | Result | -| --- | --- | -| Official promtool | 84 stable + 21 experimental cases passed | -| Official Prometheus HTTP against isolated fixture TSDB | 105/105 passed | -| Canonical tree → exact kernel binding | 105/105 `BOUND_EXACT` | -| Backend native exact execution against official-verified expectations | 105/105 passed | -| Local native backend HTTP `/api/v1/query` | 105/105 passed; every response identifies `asap_exact` | -| Local native HTTP range consistency | 105/105 range queries match per-step instant results | -| Native executor regression tests | 5/5 passed, including the 105-case corpus | -| Planner frontend regression/conformance/lowering/equivalence tests | 162/162 passed | -| Python comparator/coverage tests | 8/8 passed | -| Backend query parser regression tests | 7/7 passed | -| Targeted Clippy (`-D warnings`) and Cargo format check | Passed | - -All 14 aggregation operators and 25 range-vector functions in the pinned catalog -have an executable exact binding. This measures float-sample function coverage, -not full PromQL conformance. Native histograms, mixed types, staleness, offsets, -`@`, subqueries, and explicit vector matching are outside this exact smoke path; -unsupported query shapes are rejected rather than silently stripped. - -The exact plan consumes the real ASAPPlanner canonical tree. Execution uses native -backend kernels and timestamped raw samples; it does not forward queries to -Prometheus or read expected results from the fixture. The HTTP check uses the -`promql_exact_smoke` example, not the production ingestion/storage/router path. -Production use still needs a raw-sample source and routing integration. - -## Semantic behavior exercised - -- Existing upstream `irate` retains a distinct canonical intent from `rate`. -- Existing upstream `count` counts series, including equal-valued series, instead of distinct numbers. -- Quantile phi outside [0,1] and NaN survives lowering and produces the defined - `-Inf`, `+Inf`, or `NaN` result in the smoke cases. -- Negative `limit_ratio` uses the upper hash interval; `-1` keeps every series. - -The original summary/sketch binder diagnostics remain in `summary_status` in the -planner JSON report. Exact bindings do not imply these functions can execute from -existing sketches alone. - -Original smoke logs are under `/tmp/asap-promql-smoke/`, including -`reference-results.json`, `planner-results.json`, `backend-results.json`, -`native-exact-results.log`, `planner-regressions-after.log`, -`planner-types-mapping.log`, and `http-reference/reference-http-results.json`. -These temporary artifacts may be removed; reproduce the checks with [README.md](README.md). - -PR-branch reruns of promtool, canonical binding, native execution, native HTTP -instant/range checks, planner frontend tests, and Python tests are recorded under -`/tmp/promql-pr-smoke/` and `/tmp/promql-pr-*.log`. The official HTTP reference -check was recorded with the identical fixture before the rebase. diff --git a/tools/promql-smoke/cases.json b/tools/promql-smoke/cases.json deleted file mode 100644 index 3b13f6ee..00000000 --- a/tools/promql-smoke/cases.json +++ /dev/null @@ -1,1849 +0,0 @@ -{ - "prometheus_version": "3.5.0", - "start": 1788825600, - "interval": 60, - "eval_offset": 240, - "series": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "a" - }, - "values": [ - 1, - 2, - 3, - 4, - 5 - ] - }, - { - "labels": { - "__name__": "smoke_gauge", - "job": "b" - }, - "values": [ - 10, - 20, - 30, - 40, - 50 - ] - }, - { - "labels": { - "__name__": "smoke_counter_total", - "job": "steady" - }, - "values": [ - 60, - 120, - 180, - 240, - 300 - ] - }, - { - "labels": { - "__name__": "smoke_counter_total", - "job": "reset" - }, - "values": [ - 60, - 120, - 180, - 30, - 90 - ] - }, - { - "labels": { - "__name__": "smoke_counter_total", - "job": "last_reset" - }, - "values": [ - 60, - 120, - 180, - 240, - 30 - ] - }, - { - "labels": { - "__name__": "smoke_zero_counter_total", - "job": "zero" - }, - "values": [ - 0, - 60, - 120, - 180, - 240 - ] - }, - { - "labels": { - "__name__": "smoke_constant", - "job": "constant" - }, - "values": [ - 7, - 7, - 7, - 7, - 7 - ] - }, - { - "labels": { - "__name__": "smoke_repeat", - "job": "repeat" - }, - "values": [ - 3, - 1, - 3, - 1, - 2 - ] - }, - { - "labels": { - "__name__": "smoke_sparse", - "job": "sparse" - }, - "values": [ - 1, - null, - 3, - null, - 5 - ] - }, - { - "labels": { - "__name__": "smoke_single", - "job": "single" - }, - "values": [ - null, - null, - null, - null, - 5 - ] - }, - { - "labels": { - "__name__": "smoke_group", - "job": "a", - "instance": "x" - }, - "values": [ - 1, - 1, - 1, - 1, - 1 - ] - }, - { - "labels": { - "__name__": "smoke_group", - "job": "a", - "instance": "y" - }, - "values": [ - 3, - 3, - 3, - 3, - 3 - ] - }, - { - "labels": { - "__name__": "smoke_group", - "job": "b", - "instance": "x" - }, - "values": [ - 2, - 2, - 2, - 2, - 2 - ] - }, - { - "labels": { - "__name__": "smoke_group", - "job": "b", - "instance": "y" - }, - "values": [ - 6, - 6, - 6, - 6, - 6 - ] - } - ], - "queries": [ - { - "id": "selector", - "category": "selector", - "function": "selector", - "experimental": false, - "expr": "smoke_gauge{job=\"a\"}", - "expected": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "a" - }, - "value": 5 - } - ], - "note": "Baseline label filter." - }, - { - "id": "agg_sum", - "category": "aggregation", - "function": "sum", - "experimental": false, - "expr": "sum(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 55 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_avg", - "category": "aggregation", - "function": "avg", - "experimental": false, - "expr": "avg(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 27.5 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_count", - "category": "aggregation", - "function": "count", - "experimental": false, - "expr": "count(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 2 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_min", - "category": "aggregation", - "function": "min", - "experimental": false, - "expr": "min(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 5 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_max", - "category": "aggregation", - "function": "max", - "experimental": false, - "expr": "max(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 50 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_group", - "category": "aggregation", - "function": "group", - "experimental": false, - "expr": "group(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 1 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_stddev", - "category": "aggregation", - "function": "stddev", - "experimental": false, - "expr": "stddev(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 22.5 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_stdvar", - "category": "aggregation", - "function": "stdvar", - "experimental": false, - "expr": "stdvar(smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 506.25 - } - ], - "note": "Aggregate the final values 5 and 50." - }, - { - "id": "agg_topk", - "category": "aggregation", - "function": "topk", - "experimental": false, - "expr": "topk(1, smoke_gauge)", - "expected": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "b" - }, - "value": 50 - } - ], - "note": "Largest series; preserve its labels and metric name." - }, - { - "id": "agg_bottomk", - "category": "aggregation", - "function": "bottomk", - "experimental": false, - "expr": "bottomk(1, smoke_gauge)", - "expected": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "a" - }, - "value": 5 - } - ], - "note": "Smallest series; preserve its labels and metric name." - }, - { - "id": "agg_count_values", - "category": "aggregation", - "function": "count_values", - "experimental": false, - "expr": "count_values(\"sample\", smoke_gauge % 5)", - "expected": [ - { - "labels": { - "sample": "0" - }, - "value": 2 - } - ], - "note": "Both final values modulo 5 are zero; count repeated values." - }, - { - "id": "agg_quantile", - "category": "aggregation", - "function": "quantile", - "experimental": false, - "expr": "quantile(0.5, smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 27.5 - } - ], - "note": "Median of 5 and 50 is linearly interpolated." - }, - { - "id": "agg_limitk", - "category": "aggregation", - "function": "limitk", - "experimental": true, - "expr": "limitk(1, smoke_gauge{job=\"a\"})", - "expected": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "a" - }, - "value": 5 - } - ], - "note": "Singleton selection has an unambiguous identity; separate case checks sampling two series." - }, - { - "id": "agg_limit_ratio", - "category": "aggregation", - "function": "limit_ratio", - "experimental": true, - "expr": "limit_ratio(1, smoke_gauge)", - "expected": [ - { - "labels": { - "job": "a", - "__name__": "smoke_gauge" - }, - "value": 5 - }, - { - "labels": { - "job": "b", - "__name__": "smoke_gauge" - }, - "value": 50 - } - ], - "note": "Ratio 1 selects the entire input without changing labels." - }, - { - "id": "rollup_avg_over_time", - "category": "rollup", - "function": "avg_over_time", - "experimental": false, - "expr": "avg_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 3 - }, - { - "labels": { - "job": "b" - }, - "value": 30 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_count_over_time", - "category": "rollup", - "function": "count_over_time", - "experimental": false, - "expr": "count_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 5 - }, - { - "labels": { - "job": "b" - }, - "value": 5 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_last_over_time", - "category": "rollup", - "function": "last_over_time", - "experimental": false, - "expr": "last_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a", - "__name__": "smoke_gauge" - }, - "value": 5 - }, - { - "labels": { - "job": "b", - "__name__": "smoke_gauge" - }, - "value": 50 - } - ], - "note": "last_over_time preserves the metric name, unlike the other numeric rollups." - }, - { - "id": "rollup_max_over_time", - "category": "rollup", - "function": "max_over_time", - "experimental": false, - "expr": "max_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 5 - }, - { - "labels": { - "job": "b" - }, - "value": 50 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_min_over_time", - "category": "rollup", - "function": "min_over_time", - "experimental": false, - "expr": "min_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 10 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_present_over_time", - "category": "rollup", - "function": "present_over_time", - "experimental": false, - "expr": "present_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 1 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_stddev_over_time", - "category": "rollup", - "function": "stddev_over_time", - "experimental": false, - "expr": "stddev_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1.4142135623730951 - }, - { - "labels": { - "job": "b" - }, - "value": 14.142135623730951 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_stdvar_over_time", - "category": "rollup", - "function": "stdvar_over_time", - "experimental": false, - "expr": "stdvar_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 2 - }, - { - "labels": { - "job": "b" - }, - "value": 200 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_sum_over_time", - "category": "rollup", - "function": "sum_over_time", - "experimental": false, - "expr": "sum_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 15 - }, - { - "labels": { - "job": "b" - }, - "value": 150 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_mad_over_time", - "category": "rollup", - "function": "mad_over_time", - "experimental": true, - "expr": "mad_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 10 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_delta", - "category": "rollup", - "function": "delta", - "experimental": false, - "expr": "delta(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 5 - }, - { - "labels": { - "job": "b" - }, - "value": 50 - } - ], - "note": "Observed difference 4/40 extrapolated from 240s to the 300s window: 5/50." - }, - { - "id": "rollup_idelta", - "category": "rollup", - "function": "idelta", - "experimental": false, - "expr": "idelta(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 10 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_deriv", - "category": "rollup", - "function": "deriv", - "experimental": false, - "expr": "deriv(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 0.016666666666666666 - }, - { - "labels": { - "job": "b" - }, - "value": 0.16666666666666666 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_changes", - "category": "rollup", - "function": "changes", - "experimental": false, - "expr": "changes(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 4 - }, - { - "labels": { - "job": "b" - }, - "value": 4 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_quantile_over_time", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(0.5, smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 3 - }, - { - "labels": { - "job": "b" - }, - "value": 30 - } - ], - "note": "Evaluate the five samples at t=240s; [5m] includes t=0." - }, - { - "id": "rollup_predict_linear", - "category": "rollup", - "function": "predict_linear", - "experimental": false, - "expr": "predict_linear(smoke_gauge[5m], 60)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 6 - }, - { - "labels": { - "job": "b" - }, - "value": 60 - } - ], - "note": "Linear trend projected 60 seconds beyond evaluation time." - }, - { - "id": "rollup_double_exponential_smoothing", - "category": "rollup", - "function": "double_exponential_smoothing", - "experimental": true, - "expr": "double_exponential_smoothing(smoke_gauge[5m], 0.5, 0.5)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 5 - }, - { - "labels": { - "job": "b" - }, - "value": 50 - } - ], - "note": "The input is a perfect linear trend; smoothing follows it exactly." - }, - { - "id": "rollup_rate", - "category": "rollup", - "function": "rate", - "experimental": false, - "expr": "rate(smoke_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "steady" - }, - "value": 1 - }, - { - "labels": { - "job": "reset" - }, - "value": 0.875 - }, - { - "labels": { - "job": "last_reset" - }, - "value": 0.875 - } - ], - "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." - }, - { - "id": "rollup_increase", - "category": "rollup", - "function": "increase", - "experimental": false, - "expr": "increase(smoke_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "steady" - }, - "value": 300 - }, - { - "labels": { - "job": "reset" - }, - "value": 262.5 - }, - { - "labels": { - "job": "last_reset" - }, - "value": 262.5 - } - ], - "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." - }, - { - "id": "rollup_irate", - "category": "rollup", - "function": "irate", - "experimental": false, - "expr": "irate(smoke_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "steady" - }, - "value": 1 - }, - { - "labels": { - "job": "reset" - }, - "value": 1 - }, - { - "labels": { - "job": "last_reset" - }, - "value": 0.5 - } - ], - "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." - }, - { - "id": "rollup_resets", - "category": "rollup", - "function": "resets", - "experimental": false, - "expr": "resets(smoke_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "steady" - }, - "value": 0 - }, - { - "labels": { - "job": "reset" - }, - "value": 1 - }, - { - "labels": { - "job": "last_reset" - }, - "value": 1 - } - ], - "note": "Compare steady growth, an interior reset, and a reset in the last pair. rate/increase extrapolate over 300s." - }, - { - "id": "rollup_absent_over_time", - "category": "rollup", - "function": "absent_over_time", - "experimental": false, - "expr": "absent_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [ - { - "labels": { - "job": "missing" - }, - "value": 1 - } - ], - "note": "Missing range yields 1 and derives equality-matcher labels." - }, - { - "id": "rollup_ts_of_max_over_time", - "category": "rollup", - "function": "ts_of_max_over_time", - "experimental": true, - "expr": "ts_of_max_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 240 - }, - { - "labels": { - "job": "b" - }, - "value": 240 - } - ], - "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", - "value_is_timestamp": true - }, - { - "id": "rollup_ts_of_min_over_time", - "category": "rollup", - "function": "ts_of_min_over_time", - "experimental": true, - "expr": "ts_of_min_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 0 - }, - { - "labels": { - "job": "b" - }, - "value": 0 - } - ], - "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", - "value_is_timestamp": true - }, - { - "id": "rollup_ts_of_last_over_time", - "category": "rollup", - "function": "ts_of_last_over_time", - "experimental": true, - "expr": "ts_of_last_over_time(smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 240 - }, - { - "labels": { - "job": "b" - }, - "value": 240 - } - ], - "note": "Sample timestamps are relative to fixture start; HTTP expected values add the epoch start.", - "value_is_timestamp": true - }, - { - "id": "empty_avg_over_time", - "category": "rollup", - "function": "avg_over_time", - "experimental": false, - "expr": "avg_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_count_over_time", - "category": "rollup", - "function": "count_over_time", - "experimental": false, - "expr": "count_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_last_over_time", - "category": "rollup", - "function": "last_over_time", - "experimental": false, - "expr": "last_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_max_over_time", - "category": "rollup", - "function": "max_over_time", - "experimental": false, - "expr": "max_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_min_over_time", - "category": "rollup", - "function": "min_over_time", - "experimental": false, - "expr": "min_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_present_over_time", - "category": "rollup", - "function": "present_over_time", - "experimental": false, - "expr": "present_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_stddev_over_time", - "category": "rollup", - "function": "stddev_over_time", - "experimental": false, - "expr": "stddev_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_stdvar_over_time", - "category": "rollup", - "function": "stdvar_over_time", - "experimental": false, - "expr": "stdvar_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_sum_over_time", - "category": "rollup", - "function": "sum_over_time", - "experimental": false, - "expr": "sum_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_mad_over_time", - "category": "rollup", - "function": "mad_over_time", - "experimental": true, - "expr": "mad_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_delta", - "category": "rollup", - "function": "delta", - "experimental": false, - "expr": "delta(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_idelta", - "category": "rollup", - "function": "idelta", - "experimental": false, - "expr": "idelta(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_deriv", - "category": "rollup", - "function": "deriv", - "experimental": false, - "expr": "deriv(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_changes", - "category": "rollup", - "function": "changes", - "experimental": false, - "expr": "changes(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_quantile_over_time", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(0.5, smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_predict_linear", - "category": "rollup", - "function": "predict_linear", - "experimental": false, - "expr": "predict_linear(smoke_missing{job=\"missing\"}[5m], 60)", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_double_exponential_smoothing", - "category": "rollup", - "function": "double_exponential_smoothing", - "experimental": true, - "expr": "double_exponential_smoothing(smoke_missing{job=\"missing\"}[5m], 0.5, 0.5)", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_rate", - "category": "rollup", - "function": "rate", - "experimental": false, - "expr": "rate(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_increase", - "category": "rollup", - "function": "increase", - "experimental": false, - "expr": "increase(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_irate", - "category": "rollup", - "function": "irate", - "experimental": false, - "expr": "irate(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_resets", - "category": "rollup", - "function": "resets", - "experimental": false, - "expr": "resets(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero." - }, - { - "id": "empty_absent_over_time", - "category": "rollup", - "function": "absent_over_time", - "experimental": false, - "expr": "absent_over_time(smoke_gauge[5m])", - "expected": [], - "note": "An existing range must not trigger absence." - }, - { - "id": "empty_ts_of_max_over_time", - "category": "rollup", - "function": "ts_of_max_over_time", - "experimental": true, - "expr": "ts_of_max_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero.", - "value_is_timestamp": true - }, - { - "id": "empty_ts_of_min_over_time", - "category": "rollup", - "function": "ts_of_min_over_time", - "experimental": true, - "expr": "ts_of_min_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero.", - "value_is_timestamp": true - }, - { - "id": "empty_ts_of_last_over_time", - "category": "rollup", - "function": "ts_of_last_over_time", - "experimental": true, - "expr": "ts_of_last_over_time(smoke_missing{job=\"missing\"}[5m])", - "expected": [], - "note": "An empty input must produce no series, not a fabricated zero.", - "value_is_timestamp": true - }, - { - "id": "empty_sum", - "category": "aggregation", - "function": "sum", - "experimental": false, - "expr": "sum(smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_count", - "category": "aggregation", - "function": "count", - "experimental": false, - "expr": "count(smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_group", - "category": "aggregation", - "function": "group", - "experimental": false, - "expr": "group(smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_topk", - "category": "aggregation", - "function": "topk", - "experimental": false, - "expr": "topk(1, smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_quantile", - "category": "aggregation", - "function": "quantile", - "experimental": false, - "expr": "quantile(0.5, smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_limitk", - "category": "aggregation", - "function": "limitk", - "experimental": true, - "expr": "limitk(1, smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "empty_limit_ratio", - "category": "aggregation", - "function": "limit_ratio", - "experimental": true, - "expr": "limit_ratio(1, smoke_missing)", - "expected": [], - "note": "An aggregate over no series yields an empty vector." - }, - { - "id": "sum_by", - "category": "aggregation", - "function": "sum", - "experimental": false, - "expr": "sum by (job) (smoke_group)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 4 - }, - { - "labels": { - "job": "b" - }, - "value": 8 - } - ], - "note": "Group four series into two jobs." - }, - { - "id": "sum_without", - "category": "aggregation", - "function": "sum", - "experimental": false, - "expr": "sum without (instance) (smoke_group)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 4 - }, - { - "labels": { - "job": "b" - }, - "value": 8 - } - ], - "note": "Drop instance and metric name from output labels." - }, - { - "id": "avg_by", - "category": "aggregation", - "function": "avg", - "experimental": false, - "expr": "avg by (job) (smoke_group)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 2 - }, - { - "labels": { - "job": "b" - }, - "value": 4 - } - ], - "note": "Average separately within each job." - }, - { - "id": "count_by", - "category": "aggregation", - "function": "count", - "experimental": false, - "expr": "count by (job) (smoke_group)", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 2 - }, - { - "labels": { - "job": "b" - }, - "value": 2 - } - ], - "note": "Count series separately within each job." - }, - { - "id": "quantile_p99", - "category": "aggregation", - "function": "quantile", - "experimental": false, - "expr": "quantile(0.99, smoke_gauge)", - "expected": [ - { - "labels": {}, - "value": 49.55 - } - ], - "note": "Interpolate between 5 and 50." - }, - { - "id": "limitk_count", - "category": "aggregation", - "function": "limitk", - "experimental": true, - "expr": "count(limitk(1, smoke_gauge))", - "expected": [ - { - "labels": {}, - "value": 1 - } - ], - "note": "Select exactly one of two series without depending on a hash-selected identity." - }, - { - "id": "limit_ratio_complement", - "category": "aggregation", - "function": "limit_ratio", - "experimental": true, - "expr": "sum(limit_ratio(0.5, smoke_gauge) or limit_ratio(-0.5, smoke_gauge))", - "expected": [ - { - "labels": {}, - "value": 55 - } - ], - "note": "Positive and negative ratios cover the full input." - }, - { - "id": "limit_ratio_disjoint", - "category": "aggregation", - "function": "limit_ratio", - "experimental": true, - "expr": "limit_ratio(0.5, smoke_gauge) and limit_ratio(-0.5, smoke_gauge)", - "expected": [], - "note": "Complementary subsets must not overlap." - }, - { - "id": "p99", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(0.99, smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 4.96 - }, - { - "labels": { - "job": "b" - }, - "value": 49.6 - } - ], - "note": "Small-sample p99 requires interpolation." - }, - { - "id": "window_left_open", - "category": "rollup", - "function": "sum_over_time", - "experimental": false, - "expr": "sum_over_time(smoke_gauge[4m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 14 - }, - { - "labels": { - "job": "b" - }, - "value": 140 - } - ], - "note": "The sample at t=0 is exactly on the left boundary and is excluded." - }, - { - "id": "window_single", - "category": "rollup", - "function": "count_over_time", - "experimental": false, - "expr": "count_over_time(smoke_gauge[1m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 1 - } - ], - "note": "Only t=240 is included; t=180 is excluded." - }, - { - "id": "sparse_average", - "category": "rollup", - "function": "avg_over_time", - "experimental": false, - "expr": "avg_over_time(smoke_sparse[5m])", - "expected": [ - { - "labels": { - "job": "sparse" - }, - "value": 3 - } - ], - "note": "Missing samples are omitted, not zero-filled." - }, - { - "id": "sparse_count", - "category": "rollup", - "function": "count_over_time", - "experimental": false, - "expr": "count_over_time(smoke_sparse[5m])", - "expected": [ - { - "labels": { - "job": "sparse" - }, - "value": 3 - } - ], - "note": "Count only the three present samples." - }, - { - "id": "constant_changes", - "category": "rollup", - "function": "changes", - "experimental": false, - "expr": "changes(smoke_constant[5m])", - "expected": [ - { - "labels": { - "job": "constant" - }, - "value": 0 - } - ], - "note": "Equal adjacent values do not count as changes." - }, - { - "id": "constant_stddev", - "category": "rollup", - "function": "stddev_over_time", - "experimental": false, - "expr": "stddev_over_time(smoke_constant[5m])", - "expected": [ - { - "labels": { - "job": "constant" - }, - "value": 0 - } - ], - "note": "A constant series has zero standard deviation." - }, - { - "id": "zero_rate", - "category": "rollup", - "function": "rate", - "experimental": false, - "expr": "rate(smoke_zero_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "zero" - }, - "value": 0.8 - } - ], - "note": "Counter starts at zero: extrapolation must not invent a negative prior counter." - }, - { - "id": "zero_increase", - "category": "rollup", - "function": "increase", - "experimental": false, - "expr": "increase(smoke_zero_counter_total[5m])", - "expected": [ - { - "labels": { - "job": "zero" - }, - "value": 240 - } - ], - "note": "Zero-point clamping limits the extrapolated increase to 240." - }, - { - "id": "single_delta", - "category": "rollup", - "function": "delta", - "experimental": false, - "expr": "delta(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_deriv", - "category": "rollup", - "function": "deriv", - "experimental": false, - "expr": "deriv(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_idelta", - "category": "rollup", - "function": "idelta", - "experimental": false, - "expr": "idelta(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_rate", - "category": "rollup", - "function": "rate", - "experimental": false, - "expr": "rate(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_irate", - "category": "rollup", - "function": "irate", - "experimental": false, - "expr": "irate(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_increase", - "category": "rollup", - "function": "increase", - "experimental": false, - "expr": "increase(smoke_single[5m])", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_predict_linear", - "category": "rollup", - "function": "predict_linear", - "experimental": false, - "expr": "predict_linear(smoke_single[5m], 60)", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "single_double_exponential_smoothing", - "category": "rollup", - "function": "double_exponential_smoothing", - "experimental": true, - "expr": "double_exponential_smoothing(smoke_single[5m], 0.5, 0.5)", - "expected": [], - "note": "A single sample is insufficient; return no series." - }, - { - "id": "ties_ts_of_max_over_time", - "category": "rollup", - "function": "ts_of_max_over_time", - "experimental": true, - "expr": "ts_of_max_over_time(smoke_repeat[5m])", - "expected": [ - { - "labels": { - "job": "repeat" - }, - "value": 120 - } - ], - "note": "For tied extrema choose the latest sample timestamp.", - "value_is_timestamp": true - }, - { - "id": "ties_ts_of_min_over_time", - "category": "rollup", - "function": "ts_of_min_over_time", - "experimental": true, - "expr": "ts_of_min_over_time(smoke_repeat[5m])", - "expected": [ - { - "labels": { - "job": "repeat" - }, - "value": 180 - } - ], - "note": "For tied extrema choose the latest sample timestamp.", - "value_is_timestamp": true - }, - { - "id": "ties_ts_of_last_over_time", - "category": "rollup", - "function": "ts_of_last_over_time", - "experimental": true, - "expr": "ts_of_last_over_time(smoke_repeat[5m])", - "expected": [ - { - "labels": { - "job": "repeat" - }, - "value": 240 - } - ], - "note": "For tied extrema choose the latest sample timestamp.", - "value_is_timestamp": true - }, - { - "id": "phi_-0.1", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(-0.1, smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": "-Inf" - }, - { - "labels": { - "job": "b" - }, - "value": "-Inf" - } - ], - "note": "Out-of-range quantiles return the corresponding infinity." - }, - { - "id": "phi_1.1", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(1.1, smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": "+Inf" - }, - { - "labels": { - "job": "b" - }, - "value": "+Inf" - } - ], - "note": "Out-of-range quantiles return the corresponding infinity." - }, - { - "id": "phi_nan", - "category": "rollup", - "function": "quantile_over_time", - "experimental": false, - "expr": "quantile_over_time(NaN, smoke_gauge[5m])", - "expected": [ - { - "labels": { - "job": "a" - }, - "value": "NaN" - }, - { - "labels": { - "job": "b" - }, - "value": "NaN" - } - ], - "note": "NaN quantile parameter returns NaN. promtool 3.5 cannot compare NaN expectations; the reference expression checks x != x with bool, which is true only for NaN. HTTP comparison checks raw NaN directly.", - "promtool_expr": "(quantile_over_time(NaN, smoke_gauge[5m])) != bool (quantile_over_time(NaN, smoke_gauge[5m]))", - "promtool_expected": [ - { - "labels": { - "job": "a" - }, - "value": 1 - }, - { - "labels": { - "job": "b" - }, - "value": 1 - } - ] - }, - { - "id": "topk_order", - "category": "aggregation", - "function": "topk", - "experimental": false, - "expr": "topk(2, smoke_gauge)", - "expected": [ - { - "labels": { - "__name__": "smoke_gauge", - "job": "b" - }, - "value": 50 - }, - { - "labels": { - "__name__": "smoke_gauge", - "job": "a" - }, - "value": 5 - } - ], - "note": "Instant topk returns descending values; HTTP comparator also verifies order.", - "ordered": true - }, - { - "id": "bottomk_order", - "category": "aggregation", - "function": "bottomk", - "experimental": false, - "expr": "bottomk(2, smoke_gauge)", - "expected": [ - { - "labels": { - "job": "a", - "__name__": "smoke_gauge" - }, - "value": 5 - }, - { - "labels": { - "job": "b", - "__name__": "smoke_gauge" - }, - "value": 50 - } - ], - "note": "Instant bottomk returns ascending values; HTTP comparator also verifies order.", - "ordered": true - } - ] -} diff --git a/tools/promql-smoke/catalog.json b/tools/promql-smoke/catalog.json deleted file mode 100644 index 2437d66d..00000000 --- a/tools/promql-smoke/catalog.json +++ /dev/null @@ -1,174 +0,0 @@ -{ - "prometheus_version": "3.5.0", - "scope": "All aggregation operators and all functions with a ValueTypeMatrix argument; float-sample smoke coverage, not full type/edge-case conformance.", - "sources": { - "functions.go": { - "url": "https://raw.githubusercontent.com/prometheus/prometheus/v3.5.0/promql/parser/functions.go", - "sha256": "11881bfeb3093bea274f16f55a681bb1510a81c9cdc795756ad7256b2ef514cf" - }, - "lex.go": { - "url": "https://raw.githubusercontent.com/prometheus/prometheus/v3.5.0/promql/parser/lex.go", - "sha256": "a8d67c46cf53a10b2b3721d58a7734868f67f0467bbe6469e59bd97c5b1ac5fc" - } - }, - "aggregation": [ - { - "name": "sum", - "experimental": false - }, - { - "name": "avg", - "experimental": false - }, - { - "name": "count", - "experimental": false - }, - { - "name": "min", - "experimental": false - }, - { - "name": "max", - "experimental": false - }, - { - "name": "group", - "experimental": false - }, - { - "name": "stddev", - "experimental": false - }, - { - "name": "stdvar", - "experimental": false - }, - { - "name": "topk", - "experimental": false - }, - { - "name": "bottomk", - "experimental": false - }, - { - "name": "count_values", - "experimental": false - }, - { - "name": "quantile", - "experimental": false - }, - { - "name": "limitk", - "experimental": true - }, - { - "name": "limit_ratio", - "experimental": true - } - ], - "rollup": [ - { - "name": "absent_over_time", - "experimental": false - }, - { - "name": "avg_over_time", - "experimental": false - }, - { - "name": "changes", - "experimental": false - }, - { - "name": "count_over_time", - "experimental": false - }, - { - "name": "delta", - "experimental": false - }, - { - "name": "deriv", - "experimental": false - }, - { - "name": "double_exponential_smoothing", - "experimental": true - }, - { - "name": "idelta", - "experimental": false - }, - { - "name": "increase", - "experimental": false - }, - { - "name": "irate", - "experimental": false - }, - { - "name": "last_over_time", - "experimental": false - }, - { - "name": "mad_over_time", - "experimental": true - }, - { - "name": "max_over_time", - "experimental": false - }, - { - "name": "min_over_time", - "experimental": false - }, - { - "name": "ts_of_max_over_time", - "experimental": true - }, - { - "name": "ts_of_min_over_time", - "experimental": true - }, - { - "name": "ts_of_last_over_time", - "experimental": true - }, - { - "name": "predict_linear", - "experimental": false - }, - { - "name": "present_over_time", - "experimental": false - }, - { - "name": "quantile_over_time", - "experimental": false - }, - { - "name": "rate", - "experimental": false - }, - { - "name": "resets", - "experimental": false - }, - { - "name": "stddev_over_time", - "experimental": false - }, - { - "name": "stdvar_over_time", - "experimental": false - }, - { - "name": "sum_over_time", - "experimental": false - } - ] -} diff --git a/tools/promql-smoke/run.py b/tools/promql-smoke/run.py deleted file mode 100644 index 6a952fb9..00000000 --- a/tools/promql-smoke/run.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 -"""Run small, explicit fixtures for every Prometheus 3.5 aggregation and rollup.""" -import argparse -import hashlib -import json -import math -from pathlib import Path -import re -import subprocess -import urllib.error -import urllib.parse -import urllib.request -import xml.etree.ElementTree as ET - -HERE = Path(__file__).resolve().parent -FEATURE = "promql-experimental-functions" -SPECIAL_YAML = {"NaN": ".nan", "+Inf": ".inf", "-Inf": "-.inf"} - - -def selector(labels): - name = labels.get("__name__", "") - rest = ",".join( - f"{key}={json.dumps(value)}" - for key, value in sorted(labels.items()) if key != "__name__" - ) - return name + "{" + rest + "}" - - -def save_json(path, value): - path.write_text(json.dumps(value, indent=2, allow_nan=False) + "\n") - - -def check_coverage(cases, catalog, verify_source=False): - """Fail if any registered function is missing, even if all present tests pass.""" - if cases["prometheus_version"] != catalog["prometheus_version"]: - raise ValueError("Fixture and catalog versions differ") - ids = [query["id"] for query in cases["queries"]] - if len(ids) != len(set(ids)): - raise ValueError("Duplicate case IDs") - report = {"prometheus_version": catalog["prometheus_version"], "scope": catalog["scope"]} - for category in ("aggregation", "rollup"): - entries = {entry["name"]: entry for entry in catalog[category]} - covered = {} - for query in cases["queries"]: - if query["category"] != category: - continue - name = query["function"] - if name not in entries: - raise ValueError(f"Unregistered {category}: {name}") - if query["experimental"] != entries[name]["experimental"]: - raise ValueError(f"Incorrect experimental flag: {query['id']}") - if not re.search(rf"\b{re.escape(name)}\s*(?:\(|by\b|without\b)", query["expr"]): - raise ValueError(f"Case does not exercise its declared function: {query['id']}") - covered.setdefault(name, []).append(query["id"]) - missing = sorted(entries.keys() - covered.keys()) - if missing: - raise ValueError(f"Missing {category} coverage: {missing}") - report[category] = {"covered": len(covered), "total": len(entries), "cases": covered} - report["source_verified"] = False - if verify_source: - texts = {} - for name, source in catalog["sources"].items(): - with urllib.request.urlopen(source["url"], timeout=30) as response: - raw = response.read() - actual = hashlib.sha256(raw).hexdigest() - if actual != source["sha256"]: - raise ValueError(f"Upstream source hash changed: {source['url']}") - texts[name] = raw.decode() - blocks = re.findall(r'\n\t"([^"]+)": \{(.*?)\n\t\},', texts["functions.go"], re.S) - rollups = { - name: bool(re.search(r"Experimental:\s*true", body)) - for name, body in blocks if "ValueTypeMatrix" in body - } - block = texts["lex.go"].split("// Aggregators.")[1].split("// Keywords.")[0] - aggregates = { - name: name in ("limitk", "limit_ratio") - for name in re.findall(r'"([^"]+)":', block) - } - for category, actual in (("aggregation", aggregates), ("rollup", rollups)): - expected = {entry["name"]: entry["experimental"] for entry in catalog[category]} - if actual != expected: - raise ValueError(f"Catalog does not match official {category} registry") - report["source_verified"] = True - report["case_count"] = len(cases["queries"]) - report["experimental_case_count"] = sum(q["experimental"] for q in cases["queries"]) - return report - - -def expected_samples(query, start=0): - return [ - {"labels": sample["labels"], "value": ( - sample["value"] + start if query.get("value_is_timestamp") else sample["value"] - )} - for sample in query["expected"] - ] - - -def generate(cases, out): - lines = [] - previous_metric = None - count = 0 - for series in cases["series"]: - metric = series["labels"]["__name__"] - if metric != previous_metric: - family, kind = (metric[:-6], "counter") if metric.endswith("_total") else (metric, "gauge") - lines.append(f"# TYPE {family} {kind}") - previous_metric = metric - for index, value in enumerate(series["values"]): - if value is None: - continue # A missing sample is not a zero-valued observation. - timestamp = cases["start"] + index * cases["interval"] - lines.append(f'{selector(series["labels"])} {value} {timestamp}') - count += 1 - (out / "samples.openmetrics").write_text("\n".join(lines + ["# EOF", ""])) - inputs = [ - {"series": selector(series["labels"]), "values": " ".join( - "_" if value is None else str(value) for value in series["values"] - )} - for series in cases["series"] - ] - suites = [] - for experimental in (False, True): - groups = [] - for query in cases["queries"]: - if query["experimental"] != experimental: - continue - groups.append({ - "name": query["id"], "interval": f'{cases["interval"]}s', - "input_series": inputs, - "promql_expr_test": [{ - "expr": query.get("promtool_expr", query["expr"]), - "eval_time": f'{cases["eval_offset"]}s', - "exp_samples": [ - {"labels": selector(s["labels"]), "value": s["value"]} - for s in query.get("promtool_expected", expected_samples(query)) - ], - }], - }) - path = out / ("experimental.test.yml" if experimental else "rules.test.yml") - # JSON is YAML; special float expectations require YAML numeric scalars. - rendered = json.dumps({"fuzzy_compare": True, "tests": groups}, indent=2) - for value, yaml in SPECIAL_YAML.items(): - rendered = rendered.replace(f'"value": "{value}"', f'"value": {yaml}') - path.write_text(rendered + "\n") - suites.append((path, experimental, len(groups))) - print(f"Generated {len(cases['series'])} series / {count} samples / {len(cases['queries'])} queries in {out}", flush=True) - return suites - - -def reference_checks(promtool, suites, out, version): - version_run = subprocess.run([promtool, "--version"], capture_output=True, text=True, check=True) - version_text = version_run.stdout + version_run.stderr - if not re.search(rf"version {re.escape(version)}(?:\s|\(|,|$)", version_text): - raise ValueError(f"Expected promtool {version}, got: {version_text.strip()}") - results = [] - for path, experimental, count in suites: - command = [promtool] - if experimental: - command.append(f"--enable-feature={FEATURE}") - command += ["test", "rules", f"--junit={path.with_suffix('.xml')}", str(path)] - run = subprocess.run(command, capture_output=True, text=True) - log = run.stdout + run.stderr - path.with_suffix(".log").write_text(log) - status = "PASS" if run.returncode == 0 else "FAIL" - if run.returncode: - print(log, flush=True) - case_results = [] - junit = path.with_suffix(".xml") - if junit.exists(): - for case in ET.parse(junit).iter("testcase"): - passed = not any(case.find(tag) is not None for tag in ("failure", "error", "skipped")) - case_results.append({"id": case.attrib["name"], "status": "PASS" if passed else "FAIL"}) - if len(case_results) != count or any(row["status"] != "PASS" for row in case_results): - status = "FAIL" - print(f"Prometheus {path.name}: {status} ({count} cases)", flush=True) - results.append({"suite": path.name, "case_count": count, "status": status, "cases": case_results, - "command": command, "exit_code": run.returncode, "log": log}) - report = {"version": version_text.strip(), "suites": results} - save_json(out / "reference-results.json", report) - return all(row["status"] == "PASS" for row in results) - - -def compare_vector(body, expected, evaluation, ordered=False): - if body.get("status") != "success": - raise ValueError(f"Query error: {body.get('errorType')}: {body.get('error')}") - if body["data"]["resultType"] != "vector": - raise ValueError(f"Expected vector, got {body['data']['resultType']}") - actual = body["data"]["result"] - key = lambda labels: tuple(sorted(labels.items())) - wanted = {key(sample["labels"]): sample["value"] for sample in expected} - if len(actual) != len(wanted): - raise ValueError(f"Expected {len(wanted)} series, got {len(actual)}") - seen = set() - for series in actual: - labels = key(series["metric"]) - if labels in seen or labels not in wanted: - raise ValueError(f"Unexpected or duplicate labels: {labels}") - seen.add(labels) - timestamp, value = series["value"] - if float(timestamp) != evaluation: - raise ValueError(f"Expected timestamp {evaluation}, got {timestamp}") - actual_value, reference_value = float(value), float(wanted[labels]) - equal = ( - math.isnan(actual_value) and math.isnan(reference_value) - if math.isnan(reference_value) - else math.isclose(actual_value, reference_value, rel_tol=1e-12, abs_tol=1e-12) - ) - if not equal: - raise ValueError(f"{labels}: expected {reference_value}, got {actual_value}") - if ordered and [key(s["metric"]) for s in actual] != [key(s["labels"]) for s in expected]: - raise ValueError("Series order differs") - - -def backend_checks(base_url, cases, out): - evaluation = cases["start"] + cases["eval_offset"] - results = [] - for query in cases["queries"]: - expected = expected_samples(query, cases["start"]) - record = {"id": query["id"], "query": query["expr"], "expected": expected} - url = base_url.rstrip("/") + "/api/v1/query?" + urllib.parse.urlencode( - {"query": query["expr"], "time": evaluation} - ) - try: - try: - response = urllib.request.urlopen(url, timeout=15) - except urllib.error.HTTPError as error: - response = error # Retain the actual error body in the report. - with response: - record["http_status"] = response.code - body = json.load(response) - record["response"] = body - if record["http_status"] != 200: - raise ValueError(f"HTTP {record['http_status']}: {body}") - compare_vector(body, expected, evaluation, query.get("ordered", False)) - record["status"] = "PASS" - except (ValueError, KeyError, TypeError, OSError) as error: - record.update(status="FAIL", error=str(error)) - results.append(record) - print(record["status"], query["id"], record.get("error", ""), flush=True) - save_json(out / "backend-results.json", results) - print("Exact HTTP result checks only; inspect saved responses for execution/fallback provenance.") - return all(row["status"] == "PASS" for row in results) - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--output-dir", type=Path, default=Path("/tmp/asap-promql-smoke")) - parser.add_argument("--promtool", help="Run both stable and experimental official expression suites") - parser.add_argument("--backend-url", help="Query a backend already loaded with these samples") - parser.add_argument("--verify-catalog", action="store_true", help="Verify pinned official registry source hashes online") - args = parser.parse_args() - cases = json.loads((HERE / "cases.json").read_text()) - catalog = json.loads((HERE / "catalog.json").read_text()) - out = args.output_dir.resolve() - out.mkdir(parents=True, exist_ok=True) - coverage = check_coverage(cases, catalog, args.verify_catalog) - save_json(out / "coverage.json", coverage) - print("Coverage: " + ", ".join( - f"{coverage[category]['covered']}/{coverage[category]['total']} {category}" - for category in ("aggregation", "rollup") - )) - suites = generate(cases, out) - success = True - if args.promtool: - success = reference_checks(args.promtool, suites, out, catalog["prometheus_version"]) - if args.backend_url: - success = backend_checks(args.backend_url, cases, out) and success - if not success: - raise SystemExit(1) - - -if __name__ == "__main__": - main() diff --git a/tools/promql-smoke/test_runner.py b/tools/promql-smoke/test_runner.py deleted file mode 100644 index 07c6e37c..00000000 --- a/tools/promql-smoke/test_runner.py +++ /dev/null @@ -1,90 +0,0 @@ -"""Regression checks for failures that could otherwise turn a mismatch into a pass.""" -import copy -import json -from pathlib import Path -import unittest - -from run import check_coverage, compare_vector, expected_samples - -HERE = Path(__file__).resolve().parent - - -def vector(*samples): - return {"status": "success", "data": {"resultType": "vector", "result": [ - {"metric": labels, "value": [1788825840, str(value)]} - for labels, value in samples - ]}} - - -class ComparatorTests(unittest.TestCase): - def test_nonfinite_values(self): - for value in ("NaN", "+Inf", "-Inf"): - compare_vector(vector(({}, value)), [{"labels": {}, "value": value}], 1788825840) - for actual, wanted in (("NaN", 0), (0, "NaN"), ("-Inf", "+Inf"), (0, "+Inf")): - with self.subTest(actual=actual, wanted=wanted), self.assertRaises(ValueError): - compare_vector(vector(({}, actual)), [{"labels": {}, "value": wanted}], 1788825840) - - def test_empty_is_not_zero(self): - compare_vector(vector(), [], 1788825840) - with self.assertRaises(ValueError): - compare_vector(vector(({}, 0)), [], 1788825840) - - def test_checks_every_series_and_full_labels(self): - expected = [{"labels": {"job": "a"}, "value": 5}, {"labels": {"job": "b"}, "value": 50}] - bad_results = [ - vector(({"job": "a"}, 5), ({"job": "b"}, 51)), - vector(({"job": "a"}, 5), ({"job": "a"}, 50)), - vector(({"job": "a"}, 5), ({"job": "b", "__name__": "wrong"}, 50)), - ] - for body in bad_results: - with self.subTest(body=body), self.assertRaises(ValueError): - compare_vector(body, expected, 1788825840) - - def test_timestamp_value_uses_epoch_but_sample_time_is_evaluation(self): - query = {"value_is_timestamp": True, "expected": [{"labels": {"job": "a"}, "value": 120}]} - expected = expected_samples(query, 1788825600) - self.assertEqual(expected[0]["value"], 1788825720) - compare_vector(vector(({"job": "a"}, 1788825720)), expected, 1788825840) - with self.assertRaises(ValueError): - compare_vector(vector(({"job": "a"}, 120)), expected, 1788825840) - wrong_time = vector(({"job": "a"}, 1788825720)) - wrong_time["data"]["result"][0]["value"][0] -= 60 - with self.assertRaises(ValueError): - compare_vector(wrong_time, expected, 1788825840) - - def test_topk_order_is_only_checked_when_requested(self): - expected = [{"labels": {"job": "b"}, "value": 50}, {"labels": {"job": "a"}, "value": 5}] - reverse = vector(({"job": "a"}, 5), ({"job": "b"}, 50)) - compare_vector(reverse, expected, 1788825840) - with self.assertRaises(ValueError): - compare_vector(reverse, expected, 1788825840, ordered=True) - - def test_error_or_wrong_type_does_not_pass_as_empty(self): - for body in ({"status": "error", "error": "unsupported"}, - {"status": "success", "data": {"resultType": "scalar", "result": [0, "0"]}}): - with self.subTest(body=body), self.assertRaises(ValueError): - compare_vector(body, [], 1788825840) - - -class CoverageTests(unittest.TestCase): - def setUp(self): - self.cases = json.loads((HERE / "cases.json").read_text()) - self.catalog = json.loads((HERE / "catalog.json").read_text()) - - def test_missing_function_cannot_be_reported_as_complete(self): - self.cases["queries"] = [q for q in self.cases["queries"] if q["function"] != "rate"] - with self.assertRaisesRegex(ValueError, "Missing rollup coverage"): - check_coverage(self.cases, self.catalog) - - def test_experimental_flag_and_duplicate_ids_are_checked(self): - bad = copy.deepcopy(self.cases) - next(q for q in bad["queries"] if q["function"] == "limitk")["experimental"] = False - with self.assertRaisesRegex(ValueError, "experimental flag"): - check_coverage(bad, self.catalog) - self.cases["queries"].append(self.cases["queries"][0]) - with self.assertRaisesRegex(ValueError, "Duplicate case IDs"): - check_coverage(self.cases, self.catalog) - - -if __name__ == "__main__": - unittest.main() From 3a2d31ad2307cba908382f6e8988944a8d27df19 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 04:26:49 +0000 Subject: [PATCH 05/16] test: verify issue 754 workload physical plans using shared cases --- .github/workflows/mvp-ci.yml | 9 ++ control_plane/tests/issue754_level1.rs | 128 ++++++++++++++++++++++ promql-compliance/datasets/issue-754.yaml | 14 +++ promql-compliance/suites/issue-754.yaml | 36 ++++++ 4 files changed, 187 insertions(+) create mode 100644 control_plane/tests/issue754_level1.rs create mode 100644 promql-compliance/datasets/issue-754.yaml create mode 100644 promql-compliance/suites/issue-754.yaml diff --git a/.github/workflows/mvp-ci.yml b/.github/workflows/mvp-ci.yml index 03ec24b2..0dcba896 100644 --- a/.github/workflows/mvp-ci.yml +++ b/.github/workflows/mvp-ci.yml @@ -79,9 +79,18 @@ jobs: CARGO_NET_GIT_FETCH_WITH_CLI: "true" # Give background flusher/sealer waits headroom on hosted runners. ASAP_TEST_TIMEOUT_SCALE: "4" + ASAP_LEVEL1_ARTIFACT_DIR: ${{ github.workspace }}/artifacts/issue754-level1 working-directory: ASAPQuery-backend # Persistence tests wait on background flushers with bounded deadlines. # Match scripts/e2e.sh to avoid competing flushers exhausting those waits. run: | export ASAP_E2E_CONTROL_PLANE_BIN="$PWD/target/debug/control_plane" cargo test --workspace -- --test-threads=1 + + - name: Upload issue 754 level-1 plans + if: always() + uses: actions/upload-artifact@v4 + with: + name: issue754-level1-plans + path: artifacts/issue754-level1 + if-no-files-found: ignore diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs new file mode 100644 index 00000000..371a0994 --- /dev/null +++ b/control_plane/tests/issue754_level1.rs @@ -0,0 +1,128 @@ +//! Issue #754 level 1: every shared workload query has a valid physical plan. +use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; +use control_plane::physical::executable_binding::validate_query_plan; +use control_plane::physical::workload_cost::enumerate_exact_and_materialized_candidates; +use control_plane::query_plan::QueryPlanNode; +use serde::Deserialize; +use serde_json::Value; + +#[derive(Deserialize)] +struct Suite { + queries: Vec, +} + +#[derive(Deserialize)] +struct Case { + name: String, + expr: String, +} + +fn expected_summary(name: &str) -> Option<&'static str> { + match name { + "spatial-sum" | "temporal-sum" | "grouped-temporal-sum" => Some("Sum"), + "spatial-quantile" | "temporal-quantile" => Some("DDSketch"), + "temporal-rate" | "grouped-rate" | "topk-rate" => Some("Increase"), + "spatial-topk" | "quantile-ratio" => None, + other => panic!("no level-1 plan expectation for {other}"), + } +} + +/// The same ten expressions used by level 2 must compile to typed, connected plans. +#[test] +fn issue754_queries_have_valid_physical_plans() { + let suite: Suite = serde_yaml::from_str(include_str!( + "../../promql-compliance/suites/issue-754.yaml" + )) + .unwrap(); + assert_eq!(suite.queries.len(), 10, "the issue-754 contract changed"); + for case in suite.queries { + let expected = expected_summary(&case.name); + let mut snapshot: Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + snapshot["query_workload"]["repeating_queries"][0]["query"] = case.expr.clone().into(); + let input: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); + let (request, environment) = input.into_physical_compilation_request().unwrap(); + let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); + let mut valid_plans = Vec::new(); + let mut errors = Vec::new(); + for candidate in candidates { + match PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) { + Ok(plan) => { + let entry = plan.query_plan.lookup(&case.expr).unwrap(); + assert_eq!(entry.canonical_query, case.expr); + assert!( + entry.nodes.contains_key(&entry.root), + "query root must exist" + ); + if let Some(installed) = + plan.precompute_plan.executable_dags.get(&entry.query_id) + { + installed.validate().expect("typed DAG is valid"); + validate_query_plan(installed, entry).expect("DAG/query bindings agree"); + } else { + assert!( + plan.precompute_plan.materializations.is_empty(), + "summary plan must retain its Planner DAG" + ); + } + let dot = control_plane::physical::plan_dot::render(&plan); + assert!(dot.contains("PrecomputePlan") && dot.contains("QueryPlan:")); + valid_plans.push(plan); + } + Err(error) => errors.push(error.to_string()), + } + } + assert!( + !valid_plans.is_empty(), + "{} has no valid physical plan: {errors:?}", + case.name + ); + if let Some(family) = expected { + assert!( + valid_plans.iter().any(|plan| { + plan.precompute_plan + .materializations + .iter() + .any(|m| format!("{:?}", m.aggregation_type) == family) + && plan.query_plan.entries.values().all(|entry| { + entry.nodes.values().any(|node| { + matches!(node, QueryPlanNode::ReadMaterialization { .. }) + }) && !entry + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })) + }) + }), + "{} lacks a readable {family} summary candidate: {errors:?}", + case.name + ); + } + if let Ok(directory) = std::env::var("ASAP_LEVEL1_ARTIFACT_DIR") { + let plan = valid_plans + .iter() + .find(|plan| { + expected.is_some_and(|family| { + plan.precompute_plan + .materializations + .iter() + .any(|m| format!("{:?}", m.aggregation_type) == family) + }) + }) + .unwrap_or(&valid_plans[0]); + std::fs::create_dir_all(&directory).unwrap(); + let base = std::path::Path::new(&directory).join(&case.name); + std::fs::write( + base.with_extension("json"), + serde_json::to_vec_pretty(plan).unwrap(), + ) + .unwrap(); + std::fs::write( + base.with_extension("dot"), + control_plane::physical::plan_dot::render(plan), + ) + .unwrap(); + } + } +} diff --git a/promql-compliance/datasets/issue-754.yaml b/promql-compliance/datasets/issue-754.yaml new file mode 100644 index 00000000..4b0d611a --- /dev/null +++ b/promql-compliance/datasets/issue-754.yaml @@ -0,0 +1,14 @@ +name: issue-754 +series: + - metric: data + labels: {label_0: a, instance: a1} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 1, base: 10, modulo: 120} + - metric: data + labels: {label_0: a, instance: a2} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 2, base: 20, modulo: 120} + - metric: data + labels: {label_0: b, instance: b1} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 3, base: 30, modulo: 120} + - metric: data + labels: {label_0: b, instance: b2} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 4, base: 40, modulo: 120} diff --git a/promql-compliance/suites/issue-754.yaml b/promql-compliance/suites/issue-754.yaml new file mode 100644 index 00000000..69674b39 --- /dev/null +++ b/promql-compliance/suites/issue-754.yaml @@ -0,0 +1,36 @@ +name: issue-754 +comparison_defaults: + value_tolerance: + relative: 0.01 + absolute: 0.000001 +queries: + - name: spatial-sum + expr: 'sum by (label_0) (data)' + instant_offsets_seconds: [120] + - name: spatial-topk + expr: 'topk by (label_0) (3, data)' + instant_offsets_seconds: [120] + - name: spatial-quantile + expr: 'quantile by (label_0) (0.9, data)' + instant_offsets_seconds: [120] + - name: temporal-sum + expr: 'sum_over_time(data[1m])' + instant_offsets_seconds: [120] + - name: temporal-quantile + expr: 'quantile_over_time(0.9, data[1m])' + instant_offsets_seconds: [120] + - name: temporal-rate + expr: 'rate(data[1m])' + instant_offsets_seconds: [120] + - name: grouped-rate + expr: 'sum by (label_0) (rate(data[1m]))' + instant_offsets_seconds: [120] + - name: grouped-temporal-sum + expr: 'sum by (label_0) (sum_over_time(data[1m]))' + instant_offsets_seconds: [120] + - name: topk-rate + expr: 'topk by (label_0) (3, rate(data[1m]))' + instant_offsets_seconds: [120] + - name: quantile-ratio + expr: 'quantile_over_time(0.9, data[1m]) / quantile_over_time(0.5, data[1m])' + instant_offsets_seconds: [120] From f7d7fb114e8ee83a0dc92b57aacd2ef3129896be Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 04:28:14 +0000 Subject: [PATCH 06/16] test: share instant and range evaluations for issue 754 --- promql-compliance/datasets/issue-754.yaml | 8 +++--- promql-compliance/suites/issue-754.yaml | 30 +++++++++++++++-------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/promql-compliance/datasets/issue-754.yaml b/promql-compliance/datasets/issue-754.yaml index 4b0d611a..609db0ca 100644 --- a/promql-compliance/datasets/issue-754.yaml +++ b/promql-compliance/datasets/issue-754.yaml @@ -2,13 +2,13 @@ name: issue-754 series: - metric: data labels: {label_0: a, instance: a1} - generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 1, base: 10, modulo: 120} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 1, base: 10, modulo: 120} - metric: data labels: {label_0: a, instance: a2} - generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 2, base: 20, modulo: 120} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 2, base: 20, modulo: 120} - metric: data labels: {label_0: b, instance: b1} - generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 3, base: 30, modulo: 120} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 3, base: 30, modulo: 120} - metric: data labels: {label_0: b, instance: b2} - generated_samples: {start_offset_seconds: 0, end_offset_seconds: 120, step_seconds: 0.1, multiplier: 4, base: 40, modulo: 120} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 4, base: 40, modulo: 120} diff --git a/promql-compliance/suites/issue-754.yaml b/promql-compliance/suites/issue-754.yaml index 69674b39..cb04ecf1 100644 --- a/promql-compliance/suites/issue-754.yaml +++ b/promql-compliance/suites/issue-754.yaml @@ -6,31 +6,41 @@ comparison_defaults: queries: - name: spatial-sum expr: 'sum by (label_0) (data)' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: spatial-topk expr: 'topk by (label_0) (3, data)' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: spatial-quantile expr: 'quantile by (label_0) (0.9, data)' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: temporal-sum expr: 'sum_over_time(data[1m])' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: temporal-quantile expr: 'quantile_over_time(0.9, data[1m])' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: temporal-rate expr: 'rate(data[1m])' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: grouped-rate expr: 'sum by (label_0) (rate(data[1m]))' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: grouped-temporal-sum expr: 'sum by (label_0) (sum_over_time(data[1m]))' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: topk-rate expr: 'topk by (label_0) (3, rate(data[1m]))' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} - name: quantile-ratio expr: 'quantile_over_time(0.9, data[1m]) / quantile_over_time(0.5, data[1m])' - instant_offsets_seconds: [120] + instant_offsets_seconds: [120, 180] + range: {start_offset_seconds: 120, end_offset_seconds: 180, step_seconds: 60} From 2f28cfc6fc02f0966c923eae40441055deb15ceb Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 04:41:16 +0000 Subject: [PATCH 07/16] test: assert cost-selected issue 754 plans and explicit exact gaps --- control_plane/tests/issue754_level1.rs | 89 +++++++++++++++++++++----- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 371a0994..4ac1944d 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -1,7 +1,11 @@ //! Issue #754 level 1: every shared workload query has a valid physical plan. -use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; +use control_plane::physical::compiler::{ + BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, +}; use control_plane::physical::executable_binding::validate_query_plan; -use control_plane::physical::workload_cost::enumerate_exact_and_materialized_candidates; +use control_plane::physical::workload_cost::{ + enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, +}; use control_plane::query_plan::QueryPlanNode; use serde::Deserialize; use serde_json::Value; @@ -42,13 +46,14 @@ fn issue754_queries_have_valid_physical_plans() { )) .unwrap(); snapshot["query_workload"]["repeating_queries"][0]["query"] = case.expr.clone().into(); - let input: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); - let (request, environment) = input.into_physical_compilation_request().unwrap(); + let mut input: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); + let (request, environment) = input.clone().into_physical_compilation_request().unwrap(); let candidates = enumerate_exact_and_materialized_candidates(request).unwrap(); let mut valid_plans = Vec::new(); + let mut quotes = Vec::new(); let mut errors = Vec::new(); for candidate in candidates { - match PhysicalPlanCompiler.compile_promql(candidate, environment.clone()) { + match PhysicalPlanCompiler.compile_promql(candidate.clone(), environment.clone()) { Ok(plan) => { let entry = plan.query_plan.lookup(&case.expr).unwrap(); assert_eq!(entry.canonical_query, case.expr); @@ -69,6 +74,17 @@ fn issue754_queries_have_valid_physical_plans() { } let dot = control_plane::physical::plan_dot::render(&plan); assert!(dot.contains("PrecomputePlan") && dot.contains("QueryPlan:")); + let cost = if valid_plans.is_empty() { 1.0 } else { 1e12 }; + let manifest = manifest(&plan, &candidate.queries).unwrap(); + quotes.push(WorkloadQuote { + unit_costs: manifest + .components + .keys() + .map(|key| (key.clone(), cost)) + .collect(), + manifest, + executable: true, + }); valid_plans.push(plan); } Err(error) => errors.push(error.to_string()), @@ -99,18 +115,59 @@ fn issue754_queries_have_valid_physical_plans() { case.name ); } + input.workload_cost_evidence = Some(WorkloadCostEvidence { + backend_revision: BACKEND_REVISION.into(), + planner_revision: PLANNER_REVISION.into(), + data_snapshot_id: "issue-754-level1".into(), + model_version: "deterministic-test-costs".into(), + observed_at_unix_ms: environment.observed_at_unix_ms, + valid_for_ms: environment.max_evidence_age_ms, + quotes, + }); + let selected = input + .compile_promql() + .unwrap_or_else(|error| panic!("{} selected plan failed: {error}", case.name)); + let selected_entry = selected.query_plan.lookup(&case.expr).unwrap(); + assert!(selected_entry.nodes.contains_key(&selected_entry.root)); + if let Some(family) = expected { + assert!( + selected + .precompute_plan + .materializations + .iter() + .any(|m| format!("{:?}", m.aggregation_type) == family), + "{} selected plan lost the expected {family} summary", + case.name + ); + assert!( + !selected_entry + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), + "{} silently fell back despite a selected summary", + case.name + ); + } else { + assert!(selected.precompute_plan.materializations.is_empty()); + assert!( + matches!( + selected_entry.nodes.get(&selected_entry.root), + Some(QueryPlanNode::ExactFallback { .. }) + ), + "{} must expose an explicit exact fallback", + case.name + ); + } + if let Some(installed) = selected + .precompute_plan + .executable_dags + .get(&selected_entry.query_id) + { + installed.validate().unwrap(); + validate_query_plan(installed, selected_entry).unwrap(); + } if let Ok(directory) = std::env::var("ASAP_LEVEL1_ARTIFACT_DIR") { - let plan = valid_plans - .iter() - .find(|plan| { - expected.is_some_and(|family| { - plan.precompute_plan - .materializations - .iter() - .any(|m| format!("{:?}", m.aggregation_type) == family) - }) - }) - .unwrap_or(&valid_plans[0]); + let plan = &selected; std::fs::create_dir_all(&directory).unwrap(); let base = std::path::Path::new(&directory).join(&case.name); std::fs::write( From 4638fc9e3a16cae48634421a5d26d2a09546ffe0 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 13:12:47 +0000 Subject: [PATCH 08/16] test: make topk truncate within each label group --- promql-compliance/datasets/issue-754.yaml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/promql-compliance/datasets/issue-754.yaml b/promql-compliance/datasets/issue-754.yaml index 609db0ca..d84b1e28 100644 --- a/promql-compliance/datasets/issue-754.yaml +++ b/promql-compliance/datasets/issue-754.yaml @@ -7,8 +7,20 @@ series: labels: {label_0: a, instance: a2} generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 2, base: 20, modulo: 120} - metric: data - labels: {label_0: b, instance: b1} + labels: {label_0: a, instance: a3} generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 3, base: 30, modulo: 120} - metric: data - labels: {label_0: b, instance: b2} + labels: {label_0: a, instance: a4} generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 4, base: 40, modulo: 120} + - metric: data + labels: {label_0: b, instance: b1} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 5, base: 50, modulo: 120} + - metric: data + labels: {label_0: b, instance: b2} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 6, base: 60, modulo: 120} + - metric: data + labels: {label_0: b, instance: b3} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 7, base: 70, modulo: 120} + - metric: data + labels: {label_0: b, instance: b4} + generated_samples: {start_offset_seconds: 0, end_offset_seconds: 180, step_seconds: 0.1, multiplier: 8, base: 80, modulo: 120} From e9274a3fa817064300d06860d38d0f3801f35c03 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 13:29:14 +0000 Subject: [PATCH 09/16] test: assert query-specific issue 754 physical DAGs --- control_plane/tests/issue754_level1.rs | 222 ++++++++++++++++++++----- 1 file changed, 185 insertions(+), 37 deletions(-) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 4ac1944d..31b14984 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -8,7 +8,7 @@ use control_plane::physical::workload_cost::{ }; use control_plane::query_plan::QueryPlanNode; use serde::Deserialize; -use serde_json::Value; +use serde_json::{json, Value}; #[derive(Deserialize)] struct Suite { @@ -21,16 +21,192 @@ struct Case { expr: String, } -fn expected_summary(name: &str) -> Option<&'static str> { +struct ExpectedPlan { + family: Option<&'static str>, + partitioning: &'static str, + readout: &'static str, + root_operation: Option<&'static str>, +} + +// These are semantic contracts for the installed query DAG, not a snapshot of +// generated node IDs or cost-dependent summary IDs. +fn expected_plan(name: &str) -> ExpectedPlan { match name { - "spatial-sum" | "temporal-sum" | "grouped-temporal-sum" => Some("Sum"), - "spatial-quantile" | "temporal-quantile" => Some("DDSketch"), - "temporal-rate" | "grouped-rate" | "topk-rate" => Some("Increase"), - "spatial-topk" | "quantile-ratio" => None, + "spatial-sum" => ExpectedPlan { + family: Some("Sum"), + partitioning: "grouped", + readout: "sum", + root_operation: None, + }, + "spatial-topk" => ExpectedPlan { + family: None, + partitioning: "", + readout: "", + root_operation: None, + }, + "spatial-quantile" => ExpectedPlan { + family: Some("DDSketch"), + partitioning: "grouped", + readout: "quantile", + root_operation: None, + }, + "temporal-sum" => ExpectedPlan { + family: Some("Sum"), + partitioning: "per_entity", + readout: "sum", + root_operation: None, + }, + "temporal-quantile" => ExpectedPlan { + family: Some("DDSketch"), + partitioning: "per_entity", + readout: "quantile", + root_operation: None, + }, + "temporal-rate" => ExpectedPlan { + family: Some("Increase"), + partitioning: "per_entity", + readout: "rate", + root_operation: None, + }, + "grouped-rate" => ExpectedPlan { + family: Some("Increase"), + partitioning: "per_entity", + readout: "rate", + root_operation: Some("aggregate"), + }, + "grouped-temporal-sum" => ExpectedPlan { + family: Some("Sum"), + partitioning: "per_entity", + readout: "sum", + root_operation: Some("aggregate"), + }, + "topk-rate" => ExpectedPlan { + family: Some("Increase"), + partitioning: "per_entity", + readout: "rate", + root_operation: Some("top_k_selection"), + }, + "quantile-ratio" => ExpectedPlan { + family: None, + partitioning: "", + readout: "", + root_operation: None, + }, other => panic!("no level-1 plan expectation for {other}"), } } +fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) { + let expected = expected_plan(name); + let artifact = serde_json::to_value(plan).unwrap(); + let entries = artifact["query_plan"]["entries"].as_object().unwrap(); + assert_eq!(entries.len(), 1, "{name}: expected one query plan"); + let entry = entries.values().next().unwrap(); + let nodes = entry["nodes"].as_object().unwrap(); + let mut node = &nodes[&entry["root"].as_u64().unwrap().to_string()]; + let materializations = artifact["precompute_plan"]["materializations"] + .as_array() + .unwrap(); + let Some(family) = expected.family else { + assert_eq!(nodes.len(), 1, "{name}: fallback must be the entire plan"); + assert_eq!( + node["op"], "exact_fallback", + "{name}: ASAP plan is not implemented" + ); + assert!( + materializations.is_empty(), + "{name}: fallback cannot claim a summary" + ); + return; + }; + if let Some(operation) = expected.root_operation { + assert_eq!(node["op"], "logical", "{name}: missing root operator"); + assert_eq!( + node["operator"]["kind"], operation, + "{name}: wrong root operator" + ); + if operation == "aggregate" { + assert_eq!(node["operator"]["operation"], "sum"); + } else { + assert_eq!(node["operator"]["k"], 3, "{name}: wrong TopK limit"); + } + assert_eq!( + node["operator"]["grouping"], + json!({"labels":["label_0"],"without":false}), + "{name}: wrong grouping" + ); + let inputs = node["inputs"].as_array().unwrap(); + assert_eq!(inputs.len(), 1, "{name}: root must have one input"); + node = &nodes[&inputs[0].to_string()]; + } + assert_eq!( + nodes.len(), + if expected.root_operation.is_some() { + 3 + } else { + 2 + }, + "{name}: unexpected DAG nodes" + ); + if expected.readout == "quantile" { + assert_eq!( + node["op"], "summary_estimate", + "{name}: missing sketch readout" + ); + assert_eq!( + node["query"], + json!({"kind":"quantile","q":0.9}), + "{name}: wrong quantile" + ); + } else { + assert_eq!(node["op"], "exact_readout", "{name}: wrong readout node"); + assert_eq!(node["readout"], expected.readout, "{name}: wrong readout"); + } + let leaf = &nodes[&node["input"].to_string()]; + assert_eq!( + leaf["op"], "read_materialization", + "{name}: missing summary read" + ); + assert_eq!( + materializations.len(), + 1, + "{name}: expected one summary producer" + ); + let summary = &materializations[0]; + assert_eq!( + summary["aggregation_type"], family, + "{name}: wrong summary family" + ); + assert_eq!(summary["metric"], "data", "{name}: wrong source metric"); + assert_eq!( + summary["partitioning"], expected.partitioning, + "{name}: wrong population partitioning" + ); + let spatial = expected.partitioning == "grouped"; + assert_eq!( + summary["window_size"], + if spatial { 5 } else { 60 }, + "{name}: wrong summary window" + ); + assert_eq!( + leaf["binding"]["output_grouping"]["mode"], + if spatial { "reduce" } else { "per_entity" }, + "{name}: wrong read grouping" + ); + if spatial { + assert_eq!(summary["grouping_labels"]["labels"], json!(["label_0"])); + assert_eq!( + leaf["binding"]["output_grouping"]["keys"], + json!(["label_0"]) + ); + } else { + assert_eq!( + leaf["binding"]["readout_lookback_ms"], 60_000, + "{name}: wrong PromQL range" + ); + } +} + /// The same ten expressions used by level 2 must compile to typed, connected plans. #[test] fn issue754_queries_have_valid_physical_plans() { @@ -40,7 +216,7 @@ fn issue754_queries_have_valid_physical_plans() { .unwrap(); assert_eq!(suite.queries.len(), 10, "the issue-754 contract changed"); for case in suite.queries { - let expected = expected_summary(&case.name); + let expected = expected_plan(&case.name); let mut snapshot: Value = serde_json::from_str(include_str!( "../../docs/examples/asapquery-planning-snapshot.json" )) @@ -95,7 +271,7 @@ fn issue754_queries_have_valid_physical_plans() { "{} has no valid physical plan: {errors:?}", case.name ); - if let Some(family) = expected { + if let Some(family) = expected.family { assert!( valid_plans.iter().any(|plan| { plan.precompute_plan @@ -129,35 +305,7 @@ fn issue754_queries_have_valid_physical_plans() { .unwrap_or_else(|error| panic!("{} selected plan failed: {error}", case.name)); let selected_entry = selected.query_plan.lookup(&case.expr).unwrap(); assert!(selected_entry.nodes.contains_key(&selected_entry.root)); - if let Some(family) = expected { - assert!( - selected - .precompute_plan - .materializations - .iter() - .any(|m| format!("{:?}", m.aggregation_type) == family), - "{} selected plan lost the expected {family} summary", - case.name - ); - assert!( - !selected_entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), - "{} silently fell back despite a selected summary", - case.name - ); - } else { - assert!(selected.precompute_plan.materializations.is_empty()); - assert!( - matches!( - selected_entry.nodes.get(&selected_entry.root), - Some(QueryPlanNode::ExactFallback { .. }) - ), - "{} must expose an explicit exact fallback", - case.name - ); - } + assert_selected_plan(&case.name, &selected); if let Some(installed) = selected .precompute_plan .executable_dags From b847b84f7aa7d28f22350665244bbd7e0bdc85bd Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 13:31:02 +0000 Subject: [PATCH 10/16] test: fail level one when ASAP plans fall back to exact --- control_plane/tests/issue754_level1.rs | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 31b14984..248e2306 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -96,7 +96,7 @@ fn expected_plan(name: &str) -> ExpectedPlan { } } -fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) { +fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) -> Option { let expected = expected_plan(name); let artifact = serde_json::to_value(plan).unwrap(); let entries = artifact["query_plan"]["entries"].as_object().unwrap(); @@ -117,7 +117,9 @@ fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) { materializations.is_empty(), "{name}: fallback cannot claim a summary" ); - return; + return Some(format!( + "{name}: issue #754 requires an ASAP-local physical plan; exact fallback is not a correct answer" + )); }; if let Some(operation) = expected.root_operation { assert_eq!(node["op"], "logical", "{name}: missing root operator"); @@ -205,6 +207,7 @@ fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) { "{name}: wrong PromQL range" ); } + None } /// The same ten expressions used by level 2 must compile to typed, connected plans. @@ -215,6 +218,7 @@ fn issue754_queries_have_valid_physical_plans() { )) .unwrap(); assert_eq!(suite.queries.len(), 10, "the issue-754 contract changed"); + let mut missing_local_plans = Vec::new(); for case in suite.queries { let expected = expected_plan(&case.name); let mut snapshot: Value = serde_json::from_str(include_str!( @@ -305,7 +309,9 @@ fn issue754_queries_have_valid_physical_plans() { .unwrap_or_else(|error| panic!("{} selected plan failed: {error}", case.name)); let selected_entry = selected.query_plan.lookup(&case.expr).unwrap(); assert!(selected_entry.nodes.contains_key(&selected_entry.root)); - assert_selected_plan(&case.name, &selected); + if let Some(error) = assert_selected_plan(&case.name, &selected) { + missing_local_plans.push(error); + } if let Some(installed) = selected .precompute_plan .executable_dags @@ -330,4 +336,9 @@ fn issue754_queries_have_valid_physical_plans() { .unwrap(); } } + assert!( + missing_local_plans.is_empty(), + "{}", + missing_local_plans.join("\n") + ); } From 8641eeebc09357730f646e54d124a5139a30b4fe Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 13:51:04 +0000 Subject: [PATCH 11/16] test: expose planner candidates and assert reviewed physical plans --- control_plane/tests/issue754_level1.rs | 110 +++++++++++++++++++++---- 1 file changed, 92 insertions(+), 18 deletions(-) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 248e2306..39cc7a12 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -45,7 +45,7 @@ fn expected_plan(name: &str) -> ExpectedPlan { root_operation: None, }, "spatial-quantile" => ExpectedPlan { - family: Some("DDSketch"), + family: Some("QuantileSketch"), partitioning: "grouped", readout: "quantile", root_operation: None, @@ -57,7 +57,7 @@ fn expected_plan(name: &str) -> ExpectedPlan { root_operation: None, }, "temporal-quantile" => ExpectedPlan { - family: Some("DDSketch"), + family: Some("QuantileSketch"), partitioning: "per_entity", readout: "quantile", root_operation: None, @@ -76,9 +76,9 @@ fn expected_plan(name: &str) -> ExpectedPlan { }, "grouped-temporal-sum" => ExpectedPlan { family: Some("Sum"), - partitioning: "per_entity", + partitioning: "grouped", readout: "sum", - root_operation: Some("aggregate"), + root_operation: None, }, "topk-rate" => ExpectedPlan { family: Some("Increase"), @@ -96,6 +96,14 @@ fn expected_plan(name: &str) -> ExpectedPlan { } } +fn family_matches(expected: &str, actual: &str) -> bool { + if expected == "QuantileSketch" { + matches!(actual, "DDSketch" | "DatasketchesKLL" | "HydraKLL") + } else { + expected == actual + } +} + fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) -> Option { let expected = expected_plan(name); let artifact = serde_json::to_value(plan).unwrap(); @@ -107,19 +115,59 @@ fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) -> Option Option { let entry = plan.query_plan.lookup(&case.expr).unwrap(); @@ -254,7 +303,32 @@ fn issue754_queries_have_valid_physical_plans() { } let dot = control_plane::physical::plan_dot::render(&plan); assert!(dot.contains("PrecomputePlan") && dot.contains("QueryPlan:")); - let cost = if valid_plans.is_empty() { 1.0 } else { 1e12 }; + if let Ok(directory) = std::env::var("ASAP_LEVEL1_ARTIFACT_DIR") { + let base = std::path::Path::new(&directory) + .join("candidates") + .join(format!("{}-{candidate_index}", case.name)); + std::fs::create_dir_all(base.parent().unwrap()).unwrap(); + std::fs::write( + base.with_extension("json"), + serde_json::to_vec_pretty(&plan).unwrap(), + ) + .unwrap(); + std::fs::write(base.with_extension("dot"), &dot).unwrap(); + } + // The level-1 acceptance target is a backend-local plan. + // Price explicit exact fallbacks above every local candidate. + let cost = if plan.query_plan.entries.values().any(|entry| { + entry + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })) + }) { + 1e12 + } else if plan.precompute_plan.materializations.is_empty() { + 2.0 + } else { + 1.0 + }; let manifest = manifest(&plan, &candidate.queries).unwrap(); quotes.push(WorkloadQuote { unit_costs: manifest @@ -281,7 +355,7 @@ fn issue754_queries_have_valid_physical_plans() { plan.precompute_plan .materializations .iter() - .any(|m| format!("{:?}", m.aggregation_type) == family) + .any(|m| family_matches(family, &format!("{:?}", m.aggregation_type))) && plan.query_plan.entries.values().all(|entry| { entry.nodes.values().any(|node| { matches!(node, QueryPlanNode::ReadMaterialization { .. }) From 6950d07866b831fb77f6024193abfee0a1728bb1 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 14:57:03 +0000 Subject: [PATCH 12/16] fix: preserve Planner exact families through physical execution --- control_plane/src/clickhouse.rs | 4 +- control_plane/src/physical/compiler.rs | 66 ++++----- control_plane/src/physical/post_asap/lower.rs | 47 +------ control_plane/src/physical/post_asap/tests.rs | 22 +-- .../src/physical/runtime_capability.rs | 53 ++++---- control_plane/src/workload.rs | 84 +++++++----- control_plane/tests/issue754_level1.rs | 74 ++++++---- crates/asap_types/src/accumulator_spec.rs | 8 ++ crates/asap_types/src/aggregation_type.rs | 6 + crates/asap_types/src/sds.rs | 13 +- .../drivers/ingest/prometheus_remote_write.rs | 1 + data_plane/src/drivers/query/servers/http.rs | 2 + .../precompute_engine/accumulator_factory.rs | 10 +- .../operators/multiple_sum_accumulator.rs | 126 +++++++++++++++++- .../operators/sum_accumulator.rs | 22 ++- .../asap_query_engine/catalog_resolver.rs | 9 +- .../asap_query_engine/exact_subqueries.rs | 4 +- .../asap_query_engine/post_asap_readout.rs | 4 +- .../asap_query_engine/summary_executor.rs | 52 ++++---- .../src/storage_engines/sketch_db/accuracy.rs | 2 + 20 files changed, 364 insertions(+), 245 deletions(-) diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f1e1b1df..cbf69121 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -378,7 +378,7 @@ fn materialize_selected_sql( let aggregation = BackendAggregation { aggregation_id: String::new(), metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: crate::physical::compiler::physical_materialization_family(family), + family: family.clone(), window_secs, spatial_filter: String::new(), grouping: grouping.names(), @@ -591,7 +591,7 @@ fn bind_selected_node( .. } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = crate::physical::compiler::physical_materialization_family(family); + let expected = family.clone(); let selected = select_materialization( &request.precompute_plan.materializations, &table_ref, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 747fc171..125d5aa4 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1262,10 +1262,7 @@ impl PhysicalPlanCompiler { .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - // Rate is a readout over the same reset-aware counter state - // as Increase. Keep that semantic distinction in QueryPlan, - // while the physical store binds both to Increase state. - let physical_family = physical_materialization_family(&selected.family); + let physical_family = selected.family.clone(); let physical_algorithm = match &physical_family { SummaryFamilyType::ExactAggregate(kind, _) => { format!("{kind:?}").to_ascii_lowercase() @@ -1704,7 +1701,7 @@ impl PhysicalPlanCompiler { .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? .family; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != physical_materialization_family(node_family) + if materialization_family != *node_family || window_ms == 0 || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 @@ -2828,7 +2825,9 @@ fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), A::DDSketch => 64 * 1024, A::Sum + | A::Count | A::Increase + | A::Rate | A::Min | A::Max | A::MultipleSum @@ -2852,7 +2851,13 @@ fn retained_partition_count( if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( materialization.aggregation_type, - A::Increase | A::MultipleIncrease | A::Min | A::Max | A::MultipleMin | A::MultipleMax + A::Increase + | A::Rate + | A::MultipleIncrease + | A::Min + | A::Max + | A::MultipleMin + | A::MultipleMax ) || !materialization.grouping_labels.names().is_empty() { @@ -3270,7 +3275,7 @@ fn physical_aggregation( BackendAggregation { aggregation_id, metric_name: selected.metric.clone(), - family: physical_materialization_family(&selected.family), + family: selected.family.clone(), window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected @@ -3680,26 +3685,6 @@ fn collect_selected_materializations( Ok(selected) } -pub(crate) fn physical_materialization_family(family: &SummaryFamilyType) -> SummaryFamilyType { - match family { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) => { - // The SummaryStore Sum accumulator retains the observation count - // alongside its sum. Both logical states can share this producer. - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ) - } - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Rate, _) => { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, - ) - } - _ => family.clone(), - } -} - fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { @@ -4540,8 +4525,7 @@ pub(crate) mod tests { .find(|materialization| { matches!( materialization.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Rate ) }) .expect("reset-aware exact counter"); @@ -5070,6 +5054,7 @@ pub(crate) mod tests { .all(|m| !matches!( m.aggregation_type, asap_types::AggregationType::Increase + | asap_types::AggregationType::Rate | asap_types::AggregationType::MultipleIncrease ))); let entry = plan.query_plan.entries.values().next().unwrap(); @@ -5706,7 +5691,7 @@ pub(crate) mod tests { } #[test] - fn rate_and_increase_share_physical_counter_state() { + fn rate_and_increase_keep_planner_families_distinct() { let mut workload = request("rate", "rate(m[1m])"); workload .queries @@ -5715,10 +5700,14 @@ pub(crate) mod tests { .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert_eq!(collector.materializations[0].algorithm, "increase"); + let algorithms: std::collections::BTreeSet<_> = collector + .materializations + .iter() + .map(|materialization| materialization.algorithm.as_str()) + .collect(); + assert_eq!(algorithms, ["increase", "rate"].into()); } } @@ -5738,8 +5727,7 @@ pub(crate) mod tests { } #[test] - fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { - // Both dashboard roots use one packed raw accumulator, with explicit readouts. + fn exact_dashboard_preserves_distinct_sum_and_count_producers() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5755,7 +5743,7 @@ pub(crate) mod tests { entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { assert!( @@ -5765,7 +5753,7 @@ pub(crate) mod tests { )), "{entry:?}" ); - assert_eq!(entry.materialization_bindings().len(), 1); + assert!(!entry.materialization_bindings().is_empty()); } assert!(bundle .query_plan @@ -7479,8 +7467,8 @@ pub(crate) mod tests { assert_eq!( materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, + planner_types::post_asap::ExactKind::Rate, + planner_types::post_asap::ExactParams::Rate, ) ); } diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index 5d168c89..87729ac6 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -8,14 +8,6 @@ //! variants when a binding rule fires; everything else stays inside //! `Logical(…)`." //! -//! Two node shapes are rewritten *before* selecting a candidate, because -//! the upstream strategy can produce summaries this deployment's data plane -//! doesn't (or, deliberately, shouldn't) serve — not something the -//! `CostModel` hook can reach, since the decision of *whether* to call -//! into `rank_candidates`/`size_params` at all is made before the -//! `CostModel` is ever consulted. See each helper's docs for the specific -//! reason. -//! //! `AggIntent::Extension` (the `Frequency` point-query) needs no such //! pre-pass anymore: `ControlPlaneCostModel::realize_extension`/ //! `readout_extension` (ASAPController#150) now realize it as a real @@ -143,41 +135,8 @@ fn bind_recursive( )) } - _ => { - let rewritten = rewrite_rate_to_increase(expr); - let node = crate::planner_selection::select_summary(&rewritten, cost_model)?; - Ok(PostAsapPlan::Summary(node)) - } - } -} - -/// Rewrite Rate to Increase along the aggregate spine traversed by Planner. -/// This deployment computes rate by dividing the Increase readout by window -/// seconds, rather than storing a separate Rate accumulator. -fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs - .iter() - .map(|intent| { - if matches!(intent, AggIntent::Rate) { - AggIntent::Increase - } else { - intent.clone() - } - }) - .collect(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(rewrite_rate_to_increase(child)), - }, - other => other.clone(), + _ => Ok(PostAsapPlan::Summary( + crate::planner_selection::select_summary(expr, cost_model)?, + )), } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 3f12dec6..99785151 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -556,14 +556,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { } /// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// `bind_query_expr` (not `implement_tree` directly) rewrites -/// `AggIntent::Rate` to `AggIntent::Increase` before binding (see -/// `lower.rs`'s `rewrite_rate_to_increase` — this deployment's data -/// plane has no Rate accumulator). The old -/// `AggregationType::MultipleIncrease` identity is now -/// `SummaryKind::Increase` with a non-empty `by`. +/// Planner preserves the Rate family and the `by` reduction independently. #[test] -fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { +fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), measures: vec![AggIntent::Rate], @@ -579,11 +574,11 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { } => { assert_eq!( family, - &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } - other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), + other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), }, other => panic!("expected Committed(Summary(_)), got {other:?}"), } @@ -716,12 +711,9 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { ); } -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Control plane path: `Aggregate{Rate}` over -/// `Window` → `bind_query_expr` rewrites `Rate` to `Increase` and binds an -/// exact accumulator (`SummaryAgg{Increase}`) — no approximate summary -/// family. Both paths produce a single non-summary streaming row; the L5 -/// emitter is the one that picks the actual MultipleIncrease processor. +/// A Rate query keeps Planner's exact Rate family through binding. The +/// physical emitter chooses the runtime processor without changing that +/// family identity. #[test] fn phase_b_e2e_rate_falls_through_to_logical() { let bound = pipeline_l1_to_l4( diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 5b9f407c..f1733d0c 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -101,7 +101,7 @@ pub enum Capability { /// * Sum-over-time requires archive execution because cumulative samples /// cannot be reconstructed from delta state alone. /// -/// Rate and Increase require the Increase capability; plain sum requires Sum. +/// Rate and Increase have distinct exact-family capabilities. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -369,9 +369,8 @@ fn multi_pop_satisfies_single(required: AggregationType, available: AggregationT #[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` for intents that have no ASAP-tier sketch -/// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent -/// — see [`AggIntent::archive_only`]). +/// answer it. Returns `None` when no deployed ASAP-tier capability can +/// satisfy the intent. /// /// This is a runtime routing requirement, not a summary-selection rule. /// ASAPPlanner owns legal implementations and candidate enumeration; this @@ -395,15 +394,17 @@ pub fn capability_for(intent: &AggIntent) -> Option { } match intent { AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Count { accuracy } if is_exact(accuracy) => { + Some(Capability::ExactAgg(AggregationType::Count)) + } // Direction is part of the capability: a stored minimum cannot // answer `max_over_time` and vice versa, so these must not // collapse onto one `ExactAgg` the way they did while Planner // had a single `MinMax` accumulator. AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase | AggIntent::Rate => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } + AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), + AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::QuantileApprox(None)) } @@ -532,18 +533,14 @@ mod tests { } #[test] - fn capability_for_count_exact_routes_to_archive() { - // `count_over_time` lowers to `Count{accuracy:Exact}`. The - // PR #200/#201 follow-up briefly routed this to - // `ExactAgg(Sum)`, but the data plane has no count - // accumulator — `SumAccumulator` returns its `sum` for both - // `Statistic::Sum` and `Statistic::Count`, so the result was - // sum-of-values, not sample-count. Reverted to `None` (archive - // routing) until a real `SumCountAccumulator` lands. + fn capability_for_count_exact_preserves_count_family() { let intent = AggIntent::Count { accuracy: AccuracyTarget::Exact, }; - assert_eq!(capability_for(&intent), None); + assert_eq!( + capability_for(&intent), + Some(Capability::ExactAgg(AggregationType::Count)) + ); } #[test] @@ -597,12 +594,12 @@ mod tests { } #[test] - fn capability_for_rate_increase_route_to_exact_agg_increase() { - // PR-6 follow-up: Rate and Increase route to ASAP-tier - // ExactAgg(Increase) — the counter-reset-aware exact precompute. - // Pre-follow-up this returned `None`. + fn capability_for_rate_and_increase_preserves_family() { let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!(capability_for(&AggIntent::Rate), exact_inc); + assert_eq!( + capability_for(&AggIntent::Rate), + Some(Capability::ExactAgg(AggregationType::Rate)) + ); assert_eq!(capability_for(&AggIntent::Increase), exact_inc); } @@ -905,30 +902,26 @@ mod tests { // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] - fn exact_agg_routing_covers_sum_rate_increase_only() { - // `Capability::ExactAgg` routing covers the three intents the - // data plane has a real accumulator for: `Sum` (SumAccumulator) - // and `Rate` / `Increase` (IncreaseAccumulator). + fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { assert_eq!( capability_for(&AggIntent::Sum { col: None }), Some(Capability::ExactAgg(AggregationType::Sum)) ); assert_eq!( capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Increase)) + Some(Capability::ExactAgg(AggregationType::Rate)) ); assert_eq!( capability_for(&AggIntent::Increase), Some(Capability::ExactAgg(AggregationType::Increase)) ); - // `Count{Exact}` (count_over_time) and `Avg` both need a real - // count accumulator that doesn't exist yet — they route to - // archive until `SumCountAccumulator` lands. + // Exact count follows the Planner Count family; Avg still needs + // its own composition contract. assert_eq!( capability_for(&AggIntent::Count { accuracy: AccuracyTarget::Exact, }), - None + Some(Capability::ExactAgg(AggregationType::Count)) ); assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); } diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 6bb7fc32..fc0e756f 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -18,8 +18,7 @@ use planner_types::pre_asap::AggIntent; /// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` /// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): /// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Sum`] -/// (rate binds to ExactAgg(Sum)-shaped capability) +/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] /// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] /// /// Before this enum: the `WorkloadStore` was keyed by metric name alone @@ -39,15 +38,15 @@ pub enum AggRole { /// or workload entries with `sketch_family_override: DDSketch | KLL`. /// Routes to a quantile-shaped sketch (DDSketch / KLL). Quantile, - /// Bare counter selector, `sum(...)`, `sum_over_time(...)`, - /// `rate(...)`, `increase(...)`. All bind to ExactAgg(Sum)-shaped - /// capability on the data plane; the streaming-config emits an - /// `aggregation_type: Sum` rather than a sketch. + /// Bare selector or Sum-shaped exact aggregation. Sum, + /// Reset-aware per-second counter rate. + Rate, + /// Reset-aware counter increase over the selected window. + Increase, /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to a Sum-as-count - /// exact-aggregation. + /// to HLL when a sketch is appropriate, otherwise to exact Count. Count, /// `topk(...)`, or workload entries with /// `sketch_family_override: CountSketch | CountMinSketch`. Routes @@ -70,6 +69,8 @@ impl AggRole { match self { AggRole::Quantile => "quantile", AggRole::Sum => "sum", + AggRole::Rate => "rate", + AggRole::Increase => "increase", AggRole::Count => "count", AggRole::Topk => "topk", AggRole::Other => "other", @@ -97,15 +98,16 @@ impl std::fmt::Display for AggRole { /// through the same canonical pipeline the live serving path uses /// (`query_parser::parse_query_expr_canonical` → /// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent (the one bound to the data-plane capability) is matched: +/// intent is matched, except that a Sum wrapping a counter function +/// keeps the inner Rate or Increase role: /// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] /// * [`AggIntent::TopK`] → [`AggRole::Topk`] /// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the /// windowed-Count-as-Frequency extension /// (`intent_algebra::as_frequency`) → [`AggRole::Count`] /// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → [`AggRole::Sum`] -/// * Anything else recognised but not one of the four shapes above +/// → their respective roles +/// * Anything else recognised but not one of the listed shapes above /// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → /// [`AggRole::Other`]. /// * Bare metric selector (no `Aggregate` node at all) → @@ -125,11 +127,7 @@ impl std::fmt::Display for AggRole { /// semantics, whichever the lowerer picks). The Sum-shaped /// alternative is rare in practice; users who want it write /// `sum_over_time(count(...))` which classifies as Sum. -/// * `rate` / `irate` / `increase` — Sum. `irate` folds onto -/// `AggIntent::Rate` at L3 same as `rate`; both bind to -/// ExactAgg(Increase) on the data plane (see -/// `data_plane/src/precompute_engine/ingest_handler.rs`'s handling -/// of `AggKind::ExactAgg { Increase }`). +/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { // 1. `sketch_family_override` wins. if let Some(family) = entry.sketch_family_override.as_ref() { @@ -167,11 +165,30 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { if crate::planner_selection::as_frequency(outer).is_some() { return AggRole::Count; } + // A spatial `sum by (...)` around a counter function still requires the + // counter family's state; using Sum as the registration key would let it + // overwrite a bare Sum workload for the same metric. + if matches!(outer, AggIntent::Sum { .. }) { + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Rate)) + { + return AggRole::Rate; + } + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Increase)) + { + return AggRole::Increase; + } + } match outer { AggIntent::Quantile { .. } => AggRole::Quantile, AggIntent::TopK { .. } => AggRole::Topk, AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } | AggIntent::Rate | AggIntent::Increase => AggRole::Sum, + AggIntent::Sum { .. } => AggRole::Sum, + AggIntent::Rate => AggRole::Rate, + AggIntent::Increase => AggRole::Increase, _ => AggRole::Other, } } @@ -970,13 +987,7 @@ mod tests { #[test] fn agg_role_sum_query_strings() { - for q in [ - "sum by (zone) (m)", - "sum_over_time(m[5m])", - "rate(m[5m])", - "increase(m[5m])", - "sum by (zone) (rate(m[5m]))", - ] { + for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { assert_eq!( derive_agg_role(&entry("m", Some(q), None)), AggRole::Sum, @@ -985,6 +996,18 @@ mod tests { } } + #[test] + fn counter_functions_have_distinct_workload_roles() { + for (query, expected) in [ + ("rate(m[5m])", AggRole::Rate), + ("sum by (zone) (rate(m[5m]))", AggRole::Rate), + ("increase(m[5m])", AggRole::Increase), + ("sum by (zone) (increase(m[5m]))", AggRole::Increase), + ] { + assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); + } + } + #[test] fn agg_role_count_query_strings() { for q in [ @@ -1095,7 +1118,7 @@ mod tests { } #[test] - fn three_synthetic_http_requests_total_entries_classify_to_two_distinct_roles() { + fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { // Synthetic mirror of `deploy/configs/mvp-workload.yaml` // entries 2/3/4 — proves `derive_agg_role` produces distinct // roles for the three http_requests_total shapes. Pre-B2 the @@ -1120,15 +1143,8 @@ mod tests { ), ]; let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Sum, AggRole::Count]); - // The store distinguishes Sum vs Count keys, so two of the - // three entries (the two Sum-shaped ones) still collide - // under (metric, role). That's the documented behaviour — - // two YAML entries with the SAME (metric, role) overwrite, - // which is the legitimate "operator updated their workload" - // path. The fix scope is collisions across DIFFERENT shapes, - // not idempotent re-registers. + assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); + assert_eq!(distinct.len(), 3); } } diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 39cc7a12..9776b250 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -1,12 +1,14 @@ //! Issue #754 level 1: every shared workload query has a valid physical plan. use control_plane::physical::compiler::{ - BackendLocalPlanningInput, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, + BackendLocalPlanningInput, CompiledPhysicalPlan, PhysicalPlanCompiler, BACKEND_REVISION, + PLANNER_REVISION, }; use control_plane::physical::executable_binding::validate_query_plan; use control_plane::physical::workload_cost::{ enumerate_exact_and_materialized_candidates, manifest, WorkloadCostEvidence, WorkloadQuote, }; use control_plane::query_plan::QueryPlanNode; +use planner_types::post_asap::{ExactKind, SketchAlgorithm, SummaryFamilyType}; use serde::Deserialize; use serde_json::{json, Value}; @@ -22,18 +24,23 @@ struct Case { } struct ExpectedPlan { - family: Option<&'static str>, + family: Option, partitioning: &'static str, readout: &'static str, root_operation: Option<&'static str>, } +enum ExpectedFamily { + Exact(ExactKind), + QuantileSketch, +} + // These are semantic contracts for the installed query DAG, not a snapshot of // generated node IDs or cost-dependent summary IDs. fn expected_plan(name: &str) -> ExpectedPlan { match name { "spatial-sum" => ExpectedPlan { - family: Some("Sum"), + family: Some(ExpectedFamily::Exact(ExactKind::Sum)), partitioning: "grouped", readout: "sum", root_operation: None, @@ -45,43 +52,43 @@ fn expected_plan(name: &str) -> ExpectedPlan { root_operation: None, }, "spatial-quantile" => ExpectedPlan { - family: Some("QuantileSketch"), + family: Some(ExpectedFamily::QuantileSketch), partitioning: "grouped", readout: "quantile", root_operation: None, }, "temporal-sum" => ExpectedPlan { - family: Some("Sum"), + family: Some(ExpectedFamily::Exact(ExactKind::Sum)), partitioning: "per_entity", readout: "sum", root_operation: None, }, "temporal-quantile" => ExpectedPlan { - family: Some("QuantileSketch"), + family: Some(ExpectedFamily::QuantileSketch), partitioning: "per_entity", readout: "quantile", root_operation: None, }, "temporal-rate" => ExpectedPlan { - family: Some("Increase"), + family: Some(ExpectedFamily::Exact(ExactKind::Rate)), partitioning: "per_entity", readout: "rate", root_operation: None, }, "grouped-rate" => ExpectedPlan { - family: Some("Increase"), + family: Some(ExpectedFamily::Exact(ExactKind::Rate)), partitioning: "per_entity", readout: "rate", root_operation: Some("aggregate"), }, "grouped-temporal-sum" => ExpectedPlan { - family: Some("Sum"), + family: Some(ExpectedFamily::Exact(ExactKind::Sum)), partitioning: "grouped", readout: "sum", root_operation: None, }, "topk-rate" => ExpectedPlan { - family: Some("Increase"), + family: Some(ExpectedFamily::Exact(ExactKind::Rate)), partitioning: "per_entity", readout: "rate", root_operation: Some("top_k_selection"), @@ -96,15 +103,22 @@ fn expected_plan(name: &str) -> ExpectedPlan { } } -fn family_matches(expected: &str, actual: &str) -> bool { - if expected == "QuantileSketch" { - matches!(actual, "DDSketch" | "DatasketchesKLL" | "HydraKLL") - } else { - expected == actual +fn family_matches(expected: &ExpectedFamily, actual: &SummaryFamilyType) -> bool { + match (expected, actual) { + (ExpectedFamily::Exact(expected), SummaryFamilyType::ExactAggregate(actual, _)) => { + expected == actual + } + (ExpectedFamily::QuantileSketch, SummaryFamilyType::Sketch(kind, _)) => { + matches!( + kind.algorithm(), + SketchAlgorithm::DDSketch | SketchAlgorithm::Kll + ) + } + _ => false, } } -fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) -> Option { +fn assert_selected_plan(name: &str, plan: &CompiledPhysicalPlan) -> Option { let expected = expected_plan(name); let artifact = serde_json::to_value(plan).unwrap(); let entries = artifact["query_plan"]["entries"].as_object().unwrap(); @@ -156,17 +170,16 @@ fn assert_selected_plan(name: &str, plan: &impl serde::Serialize) -> Option Option ( + SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count), + false, + ), Increase => ( SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), false, ), + Rate => ( + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), + false, + ), Min => ( SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), false, diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ccdcbec0..846cf03e 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -14,7 +14,9 @@ use std::str::FromStr; pub enum AggregationType { // ---------- single-population (non-keyed) ---------- Sum, + Count, Increase, + Rate, Min, Max, DatasketchesKLL, @@ -41,7 +43,9 @@ impl AggregationType { pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", + AggregationType::Count => "Count", AggregationType::Increase => "Increase", + AggregationType::Rate => "Rate", AggregationType::Min => "Min", AggregationType::Max => "Max", AggregationType::DatasketchesKLL => "DatasketchesKLL", @@ -93,7 +97,9 @@ impl FromStr for AggregationType { match s { // Canonical names "Sum" => Ok(AggregationType::Sum), + "Count" => Ok(AggregationType::Count), "Increase" => Ok(AggregationType::Increase), + "Rate" => Ok(AggregationType::Rate), "Min" => Ok(AggregationType::Min), "Max" => Ok(AggregationType::Max), "DatasketchesKLL" => Ok(AggregationType::DatasketchesKLL), diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index c0621614..92742e07 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -668,10 +668,19 @@ impl FidelityGuarantee { (aggregation_type, self), (A::UnivMon, UnivMonFrequency { .. }) | ( - A::Sum | A::MultipleSum | A::Min | A::Max | A::MultipleMin | A::MultipleMax, + A::Sum + | A::Count + | A::MultipleSum + | A::Min + | A::Max + | A::MultipleMin + | A::MultipleMax, Exact ) - | (A::Increase | A::MultipleIncrease, ExactCounter { .. }) + | ( + A::Increase | A::Rate | A::MultipleIncrease, + ExactCounter { .. } + ) | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) | (A::DDSketch, DdSketchRelativeError { .. }) | (A::HLL, HllCardinalityError { .. }) diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index b6d3bfd7..75745163 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -717,6 +717,7 @@ fn route_messages( && matches!( config.aggregation_type, asap_types::AggregationType::Increase + | asap_types::AggregationType::Rate | asap_types::AggregationType::MultipleIncrease | asap_types::AggregationType::Min | asap_types::AggregationType::Max diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 382ec0b4..40605789 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1004,8 +1004,10 @@ fn metric_has_exact_agg_sum_sid( cap, Capability::ExactAgg( AggregationType::Sum + | AggregationType::Count | AggregationType::MultipleSum | AggregationType::Increase + | AggregationType::Rate | AggregationType::MultipleIncrease ) ) { diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 43a779fc..64903747 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -424,7 +424,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { fn memory_usage_bytes(&self) -> usize { std::mem::size_of::() - + self.acc.sums.len() * (std::mem::size_of::() + 8) + + self.acc.sums.len() * (std::mem::size_of::() + 16) } } @@ -1040,10 +1040,10 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), false) => { Box::new(SumAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), true) => { Box::new(MultipleSumAccumulatorUpdater::new()) } @@ -1065,10 +1065,10 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), false) => { Box::new(IncreaseAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), true) => { Box::new(MultipleIncreaseAccumulatorUpdater::new()) } diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs index 85a9982d..55b27b76 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs @@ -13,20 +13,34 @@ use asap_types::Statistic; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct MultipleSumAccumulator { pub sums: HashMap, + #[serde(default)] + pub counts: HashMap, } impl MultipleSumAccumulator { pub fn new() -> Self { Self { sums: HashMap::new(), + counts: HashMap::new(), } } pub fn update(&mut self, key: KeyByLabelValues, value: f64) { - *self.sums.entry(key).or_insert(0.0) += value; + let is_new = !self.sums.contains_key(&key); + *self.sums.entry(key.clone()).or_insert(0.0) += value; + if let Some(count) = self.counts.get(&key).copied() { + if let Some(next) = count.checked_add(1).filter(|next| *next != u64::MAX) { + self.counts.insert(key, next); + } else { + self.counts.remove(&key); + } + } else if is_new { + self.counts.insert(key, 1); + } } pub fn add_sum(&mut self, key: KeyByLabelValues, sum: f64) { + self.counts.remove(&key); self.sums.insert(key, sum); } @@ -43,7 +57,19 @@ impl MultipleSumAccumulator { sums.insert(key, sum); } - Ok(Self { sums }) + let mut counts = HashMap::new(); + if let Some(counts_data) = data.get("counts").and_then(Value::as_object) { + for (key_str, value) in counts_data { + let key_json: Value = serde_json::from_str(key_str)?; + let key = KeyByLabelValues::deserialize_from_json(&key_json)?; + let count = value.as_u64().ok_or("Invalid count value")?; + if !sums.contains_key(&key) { + return Err("Count key missing from sums".into()); + } + counts.insert(key, count); + } + } + Ok(Self { sums, counts }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -62,6 +88,7 @@ impl MultipleSumAccumulator { offset += 4; let mut sums = HashMap::new(); + let mut keys = Vec::new(); for _ in 0..num_entries { // Read key length and data @@ -99,10 +126,27 @@ impl MultipleSumAccumulator { ]); offset += 8; + keys.push(key.clone()); sums.insert(key, sum); } - - Ok(Self { sums }) + let remaining = buffer.len() - offset; + let count_bytes = num_entries + .checked_mul(8) + .ok_or("Count section too large")?; + if remaining != 0 && remaining != count_bytes { + return Err("Invalid count section length".into()); + } + let mut counts = HashMap::new(); + if remaining != 0 { + for key in keys { + let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); + offset += 8; + if count != u64::MAX { + counts.insert(key, count); + } + } + } + Ok(Self { sums, counts }) } } @@ -124,8 +168,15 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + let mut counts_obj = serde_json::Map::new(); + for (key, count) in &self.counts { + let key_str = serde_json::to_string(&key.serialize_to_json()).unwrap(); + counts_obj.insert(key_str, Value::from(*count)); + } + serde_json::json!({ - "sums": sums_obj + "sums": sums_obj, + "counts": counts_obj }) } @@ -136,7 +187,9 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&(self.sums.len() as u32).to_le_bytes()); // Write each key-value pair + let mut ordered_keys = Vec::with_capacity(self.sums.len()); for (key, sum) in &self.sums { + ordered_keys.push(key); let key_bytes = key.serialize_to_bytes(); // Write key length and data @@ -147,6 +200,17 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&sum.to_le_bytes()); } + for key in ordered_keys { + buffer.extend_from_slice( + &self + .counts + .get(key) + .copied() + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + } + buffer } } @@ -200,7 +264,7 @@ impl AggregateCore for MultipleSumAccumulator { fn approx_memory_bytes(&self) -> usize { // HashMap. Label strings dominate; use a // conservative per-entry estimate plus HashMap overhead. - const BYTES_PER_ENTRY: usize = 96; + const BYTES_PER_ENTRY: usize = 112; std::mem::size_of::() + self.sums.len() * BYTES_PER_ENTRY } @@ -230,11 +294,20 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { _query_kwargs: Option<&HashMap>, ) -> Result> { match statistic { - Statistic::Sum | Statistic::Count => self + Statistic::Sum => self .sums .get(key) .copied() .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), + Statistic::Count => self + .counts + .get(key) + .map(|count| *count as f64) + .ok_or_else(|| { + "Sample count unavailable in MultipleSumAccumulator" + .to_string() + .into() + }), _ => Err( format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), ), @@ -257,6 +330,26 @@ impl MergeableAccumulator for MultipleSumAccumulator { let mut result = MultipleSumAccumulator::new(); for acc in accumulators { + for key in acc.sums.keys() { + match ( + result.counts.get(key).copied(), + acc.counts.get(key).copied(), + ) { + (None, Some(count)) if !result.sums.contains_key(key) => { + result.counts.insert(key.clone(), count); + } + (Some(existing), Some(count)) => { + if let Some(total) = existing.checked_add(count) { + result.counts.insert(key.clone(), total); + } else { + result.counts.remove(key); + } + } + _ => { + result.counts.remove(key); + } + } + } for (key, sum) in acc.sums { *result.sums.entry(key).or_insert(0.0) += sum; } @@ -294,6 +387,25 @@ mod tests { assert_eq!(acc.sums.get(&key2), Some(&20.0)); } + #[test] + fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut first = MultipleSumAccumulator::new(); + first.update(key.clone(), 10.0); + first.update(key.clone(), 20.0); + let mut second = MultipleSumAccumulator::new(); + second.update(key.clone(), 7.0); + let merged = MultipleSumAccumulator::merge_accumulators(vec![first, second]).unwrap(); + for acc in [ + merged.clone(), + MultipleSumAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + MultipleSumAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + ] { + assert_eq!(acc.query(Statistic::Sum, &key, None).unwrap(), 37.0); + assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); + } + } + #[test] fn test_multiple_sum_accumulator_query() { let mut acc = MultipleSumAccumulator::new(); diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 2cbe66fe..d5ff3b02 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -197,7 +197,11 @@ impl SingleSubpopulationAggregate for SumAccumulator { } match statistic { - Statistic::Sum | Statistic::Count => Ok(self.sum), + Statistic::Sum => Ok(self.sum), + Statistic::Count => self + .observation_count + .map(|count| count as f64) + .ok_or_else(|| "sample count is unavailable for this Sum payload".into()), _ => Err(format!("Unsupported statistic in SumAccumulator: {statistic:?}").into()), } } @@ -308,10 +312,7 @@ mod tests { crate::SingleSubpopulationAggregate::query(&acc, Statistic::Sum, None).unwrap(), 42.0 ); - assert_eq!( - crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), - 42.0 - ); + assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).is_err()); assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Min, None).is_err()); // SumAccumulator is a single subpopulation accumulator, doesn't need key-based queries @@ -321,6 +322,17 @@ mod tests { ); } + #[test] + fn count_readout_uses_observation_count_not_sum() { + let mut acc = SumAccumulator::new(); + acc.update(10.0); + acc.update(20.0); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), + 2.0 + ); + } + #[test] fn test_sum_accumulator_merge() { let acc1 = SumAccumulator::with_sum(10.0); diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index ba07c0b9..0721c8a0 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -52,8 +52,10 @@ impl ResolvedMaterialization<'_> { &self.summary.operator, SummaryOperator::Configured { aggregation_type: AggregationType::Sum + | AggregationType::Count | AggregationType::MultipleSum | AggregationType::Increase + | AggregationType::Rate | AggregationType::MultipleIncrease | AggregationType::Min | AggregationType::Max @@ -78,10 +80,9 @@ impl ResolvedMaterialization<'_> { match node { QueryPlanNode::ExactReadout { readout, .. } => match readout { ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Sum, - ExactReadout::Increase | ExactReadout::Rate => { - matches!(aggregation_type, Increase | MultipleIncrease) - } + ExactReadout::Count => *aggregation_type == Count, + ExactReadout::Increase => matches!(aggregation_type, Increase | MultipleIncrease), + ExactReadout::Rate => *aggregation_type == Rate, // Direction is the family now -- no `aggregation_sub_type` // cross-check, and a minimum summary can no longer be // offered up for a maximum readout. diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 2a451d6a..1fb3c3c5 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -902,9 +902,9 @@ mod tests { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 41b41ab7..20ff521d 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1366,9 +1366,9 @@ mod tests { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index f48ba405..97458531 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -204,10 +204,12 @@ impl GroupState { return None; }; let stat = match agg_type { - AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease => asap_types::Statistic::Sum, + AggregationType::Sum | AggregationType::MultipleSum => asap_types::Statistic::Sum, + AggregationType::Count => asap_types::Statistic::Count, + AggregationType::Increase | AggregationType::MultipleIncrease => { + asap_types::Statistic::Increase + } + AggregationType::Rate => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -224,9 +226,7 @@ impl GroupState { .ok() } - /// Finalize a compiler-declared exact readout. Rate and increase share - /// reset-aware Increase state physically, but remain distinct operations - /// in QueryPlan so serving never infers semantics from PromQL text. + /// Finalize the Planner-declared exact family with its matching readout. pub fn exact_value_for( &self, readout: asap_types::query_plan::ExactReadout, @@ -238,7 +238,7 @@ impl GroupState { return None; }; let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { + (asap_types::query_plan::ExactReadout::Count, AggregationType::Count) => { asap_types::Statistic::Count } ( @@ -249,10 +249,9 @@ impl GroupState { asap_types::query_plan::ExactReadout::Increase, AggregationType::Increase | AggregationType::MultipleIncrease, ) => asap_types::Statistic::Increase, - ( - asap_types::query_plan::ExactReadout::Rate, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Rate, + (asap_types::query_plan::ExactReadout::Rate, AggregationType::Rate) => { + asap_types::Statistic::Rate + } ( asap_types::query_plan::ExactReadout::Min, AggregationType::Min | AggregationType::MultipleMin, @@ -269,7 +268,7 @@ impl GroupState { // instead of allocating a boxed trait object for every pane. if matches!( agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + AggregationType::Increase | AggregationType::Rate | AggregationType::MultipleIncrease ) { let accumulators = entries .iter() @@ -592,7 +591,9 @@ impl QueryExecutionContext<'_> { Candidate::ExactAgg(agg_type) => { if matches!( agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + AggregationType::Increase + | AggregationType::Rate + | AggregationType::MultipleIncrease ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -666,7 +667,9 @@ impl QueryExecutionContext<'_> { // remain contiguous because a missing pane is not zero. if matches!( agg_type, - AggregationType::Sum | AggregationType::MultipleSum + AggregationType::Sum + | AggregationType::Count + | AggregationType::MultipleSum ) { check_panes(windows.keys().copied().collect())?; } @@ -1255,17 +1258,10 @@ fn summary_family_matches_sketch( /// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> /// ExactKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease -/// -> ExactKind::Increase` — confirmed against that module's own -/// dispatch table rather than invented here). +/// side. Count and Rate remain distinct families even though their runtime +/// accumulators share implementations with Sum and Increase. /// -/// `ExactKind::Count`/`Rate`/`Min`/`Max` are not matched by this legacy -/// family-discovery path. For `Count`/`Rate` the final operation is ambiguous -/// from the stored accumulator alone. `Min`/`Max` were excluded for a reason -/// that no longer holds -- direction used to be unrecoverable once a summary -/// reached `AggKind::ExactAgg`, and is now the family itself -- but admitting -/// them here widens candidate discovery beyond the family split and is left -/// as follow-up. Installed QueryPlans carry an explicit `ExactReadout`, and +/// Installed QueryPlans carry an explicit `ExactReadout`, and /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( @@ -1273,9 +1269,15 @@ fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: Aggregatio ( SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), AggregationType::Sum | AggregationType::MultipleSum, + ) | ( + SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count), + AggregationType::Count, ) | ( SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), AggregationType::Increase | AggregationType::MultipleIncrease, + ) | ( + SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), + AggregationType::Rate, ) ) } diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index f3780b04..63199784 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -91,7 +91,9 @@ fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { // `DeltaSetAggregator` exact-set-membership family lived // here too before its retirement.) AggregationType::Sum + | AggregationType::Count | AggregationType::Increase + | AggregationType::Rate | AggregationType::Min | AggregationType::Max | AggregationType::MultipleSum From c4bcdc9eaf9dc53f153750fac7a3915764408693 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 15:18:21 +0000 Subject: [PATCH 13/16] refactor: carry Planner families through catalog and DAG execution --- control_plane/tests/issue754_level1.rs | 18 ++ crates/asap_types/src/accumulator_spec.rs | 43 +---- crates/asap_types/src/aggregation_type.rs | 50 +++++ crates/asap_types/src/query_plan.rs | 16 ++ crates/asap_types/src/sds.rs | 65 ++++++- crates/asap_types/src/summary_catalog.rs | 2 +- data_plane/src/lib.rs | 2 +- .../precompute_engine/accumulator_factory.rs | 41 +++-- ...ator.rs => keyed_sum_count_accumulator.rs} | 170 +++++++++++------ .../src/precompute_engine/operators/mod.rs | 4 +- data_plane/src/precompute_engine/worker.rs | 12 +- .../asap_query_engine/catalog_resolver.rs | 57 ++---- .../asap_query_engine/summary_executor.rs | 174 ++++++++++-------- .../storage_engines/sketch_db/index/mod.rs | 12 +- data_plane/src/tests/trait_design_tests.rs | 8 +- .../summary-catalog-sds-architecture.md | 10 +- 16 files changed, 436 insertions(+), 248 deletions(-) rename data_plane/src/precompute_engine/operators/{multiple_sum_accumulator.rs => keyed_sum_count_accumulator.rs} (71%) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 9776b250..92c649ee 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -1,4 +1,5 @@ //! Issue #754 level 1: every shared workload query has a valid physical plan. +use asap_types::sds::SummaryOperator; use control_plane::physical::compiler::{ BackendLocalPlanningInput, CompiledPhysicalPlan, PhysicalPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, @@ -119,6 +120,23 @@ fn family_matches(expected: &ExpectedFamily, actual: &SummaryFamilyType) -> bool } fn assert_selected_plan(name: &str, plan: &CompiledPhysicalPlan) -> Option { + for materialization in &plan.precompute_plan.materializations { + let definition = plan + .summary_catalog + .materializations + .get(&materialization.policy_fingerprint().into()) + .expect("precompute producer has no catalog definition"); + let descriptor = + &plan.summary_catalog.summary_descriptors[&definition.summary_descriptor_id]; + let SummaryOperator::Configured { family, .. } = &descriptor.operator else { + panic!("{name}: producer catalog descriptor lacks Planner family"); + }; + assert_eq!( + family, + &materialization.accumulator_spec().unwrap().family, + "{name}: precompute producer and catalog disagree about Planner family" + ); + } let expected = expected_plan(name); let artifact = serde_json::to_value(plan).unwrap(); let entries = artifact["query_plan"]["entries"].as_object().unwrap(); diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 6258570f..9fee3d02 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -227,29 +227,10 @@ impl AggregationConfig { ) }; let (family, keyed) = match self.aggregation_type { - Sum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - Count => ( - SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count), - false, - ), - Increase => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - Rate => ( - SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - false, - ), - Min => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - Max => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, + Sum | Count | Increase | Rate | Min | Max | MultipleSum | MultipleIncrease + | MultipleMin | MultipleMax => ( + self.aggregation_type.planner_exact_family().unwrap(), + self.aggregation_type.is_keyed(), ), DatasketchesKLL => ( independent_sketch( @@ -260,22 +241,6 @@ impl AggregationConfig { ), false, ), - MultipleSum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - MultipleIncrease => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - MultipleMin => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - MultipleMax => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), HydraKLL => { let k = kll_k_param(self) as u32; ( diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index 846cf03e..469d6d43 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -40,6 +40,22 @@ pub enum AggregationType { } impl AggregationType { + /// Adapt a storage/processor tag to Planner's exact family. Keyed storage + /// changes the payload layout, not the semantic family. + pub fn planner_exact_family(self) -> Option { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum | Self::MultipleSum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase | Self::MultipleIncrease => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min | Self::MultipleMin => (ExactKind::Min, ExactParams::Min), + Self::Max | Self::MultipleMax => (ExactKind::Max, ExactParams::Max), + _ => return None, + }; + Some(SummaryFamilyType::ExactAggregate(kind, params)) + } + pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", @@ -174,3 +190,37 @@ impl<'de> Deserialize<'de> for AggregationType { s.parse().map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + #[test] + fn storage_layout_tags_do_not_create_planner_families() { + for (storage, expected) in [ + (AggregationType::Sum, ExactKind::Sum), + (AggregationType::MultipleSum, ExactKind::Sum), + (AggregationType::Count, ExactKind::Count), + (AggregationType::Increase, ExactKind::Increase), + (AggregationType::MultipleIncrease, ExactKind::Increase), + (AggregationType::Rate, ExactKind::Rate), + ] { + let family = storage.planner_exact_family().unwrap(); + assert!( + matches!(family, SummaryFamilyType::ExactAggregate(kind, _) if kind == expected) + ); + } + assert_eq!( + AggregationType::Rate.planner_exact_family(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Rate, + ExactParams::Rate + )) + ); + assert_ne!( + AggregationType::Rate.planner_exact_family(), + AggregationType::Increase.planner_exact_family() + ); + } +} diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 5bdd2605..2ebb6404 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -637,6 +637,22 @@ pub enum ExactReadout { Max, } +impl ExactReadout { + /// Planner family required by this installed DAG readout node. + pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + }; + SummaryFamilyType::ExactAggregate(kind, params) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryReadout { diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 92742e07..24bbbbdd 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -383,6 +383,9 @@ pub enum SummaryOperator { /// Complete planner materialization configuration, including heap/Hydra /// dimensions and readout/update subtype. Never equal to a legacy projection. Configured { + /// Planner-selected semantic family; grouping and pane layout live in + /// the data descriptor and summary definition, respectively. + family: planner_types::post_asap::SummaryFamilyType, aggregation_type: AggregationType, aggregation_sub_type: String, parameters: BTreeMap, @@ -598,13 +601,48 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if let SummaryOperator::Configured { + family, + aggregation_type, + .. + } = &operator + { + if let Some(expected) = aggregation_type.planner_exact_family() { + if family != &expected { + return Err(SdsError( + "configured storage type disagrees with Planner family".into(), + )); + } + } else { + use AggregationType as A; + let expected = match aggregation_type { + A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), + A::CountMinSketch => Some(SketchAlgorithm::Cms), + A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), + A::CountSketch => Some(SketchAlgorithm::CountSketch), + A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), + A::DDSketch => Some(SketchAlgorithm::DDSketch), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let Some(expected) = expected { + if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) + { + return Err(SdsError( + "configured sketch storage disagrees with Planner family".into(), + )); + } + } + } + } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), )); } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); + let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); Ok(Self { id, operator, @@ -640,6 +678,10 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { + family: config + .accumulator_spec() + .map_err(|error| SdsError(error.to_string()))? + .family, aggregation_type: config.aggregation_type, aggregation_sub_type: config.aggregation_sub_type.clone(), parameters: config @@ -1453,6 +1495,7 @@ mod tests { .is_err()); assert!(SummaryDescriptor::new( SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::new(), @@ -1477,6 +1520,24 @@ mod tests { ) .is_err()); } + + #[test] + fn configured_descriptor_rejects_family_storage_disagreement() { + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + family: AggregationType::Rate.planner_exact_family().unwrap(), + aggregation_type: AggregationType::Increase, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, + 2, + ) + .is_err()); + } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); @@ -1533,11 +1594,13 @@ mod tests { #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), }; let b = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4d1ac062..223bdf37 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -13,7 +13,7 @@ use crate::PolicyFingerprint; use crate::WindowMaterializationLayout; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 2; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 3; /// Stable materialization identity binds operator and population descriptors. /// Concrete intervals, groups and completeness belong to runtime instances. diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2721bc68..b6ebecf7 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -43,7 +43,7 @@ pub use storage_engines::types::{ }; pub use precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; pub use storage_engines::StoreResult; diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 64903747..16b4226a 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,9 +1,9 @@ use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, - HydraKllSketchAccumulator, IncreaseAccumulator, MaxAccumulator, MinAccumulator, - MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, - MultipleSumAccumulator, SumAccumulator, + HydraKllSketchAccumulator, IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, + MinAccumulator, MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, + SumAccumulator, }; use crate::storage_engines::types::{ AggregateCore, AggregationType, KeyByLabelValues, Measurement, @@ -379,28 +379,32 @@ impl AccumulatorUpdater for DDSketchAccumulatorUpdater { } // --------------------------------------------------------------------------- -// MultipleSumAccumulatorUpdater +// KeyedSumCountAccumulatorUpdater // --------------------------------------------------------------------------- -pub struct MultipleSumAccumulatorUpdater { - acc: MultipleSumAccumulator, +pub struct KeyedSumCountAccumulatorUpdater { + acc: KeyedSumCountAccumulator, } -impl MultipleSumAccumulatorUpdater { +impl KeyedSumCountAccumulatorUpdater { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { Self { - acc: MultipleSumAccumulator::new(), + acc: KeyedSumCountAccumulator::for_family(family), } } } -impl Default for MultipleSumAccumulatorUpdater { +impl Default for KeyedSumCountAccumulatorUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { +impl AccumulatorUpdater for KeyedSumCountAccumulatorUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -415,7 +419,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleSumAccumulator::new(); + self.acc = KeyedSumCountAccumulator::for_family(self.acc.family.clone()); } fn is_keyed(&self) -> bool { @@ -423,7 +427,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() + std::mem::size_of::() + self.acc.sums.len() * (std::mem::size_of::() + 16) } } @@ -1026,7 +1030,7 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { tracing::warn!( @@ -1043,9 +1047,12 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { Box::new(SumAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), true) => { - Box::new(MultipleSumAccumulatorUpdater::new()) + (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { + Box::new(KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Sum)) } + (SummaryFamilyType::ExactAggregate(ExactKind::Count, _), true) => Box::new( + KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Count), + ), // Direction comes off the family itself now. It used to be read // back out of `aggregation_sub_type` because Planner had one @@ -1362,7 +1369,7 @@ mod tests { #[test] fn test_multiple_sum_updater() { - let mut updater = MultipleSumAccumulatorUpdater::new(); + let mut updater = KeyedSumCountAccumulatorUpdater::new(); assert!(updater.is_keyed()); let key_a = KeyByLabelValues::new_with_labels(vec!["a".to_string()]); @@ -1372,7 +1379,7 @@ mod tests { updater.update_keyed(&key_b, 2.0, 2000); let acc = updater.take_accumulator(); - assert_eq!(acc.type_name(), "MultipleSumAccumulator"); + assert_eq!(acc.type_name(), "KeyedSumCountAccumulator"); } #[test] diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs similarity index 71% rename from data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs index 55b27b76..c39d5583 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs @@ -7,19 +7,32 @@ use serde_json::Value; use std::collections::HashMap; use asap_types::Statistic; +use planner_types::post_asap::ExactKind; + +fn sum_family() -> ExactKind { + ExactKind::Sum +} /// Accumulator that maintains separate sum values for multiple keys /// Allows querying sums for specific label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleSumAccumulator { +pub struct KeyedSumCountAccumulator { + #[serde(default = "sum_family")] + pub family: ExactKind, pub sums: HashMap, #[serde(default)] pub counts: HashMap, } -impl MultipleSumAccumulator { +impl KeyedSumCountAccumulator { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { + assert!(matches!(family, ExactKind::Sum | ExactKind::Count)); Self { + family, sums: HashMap::new(), counts: HashMap::new(), } @@ -69,7 +82,16 @@ impl MultipleSumAccumulator { counts.insert(key, count); } } - Ok(Self { sums, counts }) + let family = match data.get("family").and_then(Value::as_str) { + None | Some("Sum") => ExactKind::Sum, + Some("Count") => ExactKind::Count, + _ => return Err("Invalid keyed additive family".into()), + }; + Ok(Self { + family, + sums, + counts, + }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -133,11 +155,11 @@ impl MultipleSumAccumulator { let count_bytes = num_entries .checked_mul(8) .ok_or("Count section too large")?; - if remaining != 0 && remaining != count_bytes { + if remaining != 0 && remaining != count_bytes && remaining != count_bytes + 1 { return Err("Invalid count section length".into()); } let mut counts = HashMap::new(); - if remaining != 0 { + if count_bytes != 0 && remaining >= count_bytes { for key in keys { let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); offset += 8; @@ -146,17 +168,30 @@ impl MultipleSumAccumulator { } } } - Ok(Self { sums, counts }) + let family = if remaining == count_bytes + 1 { + match buffer[offset] { + 0 => ExactKind::Sum, + 1 => ExactKind::Count, + _ => return Err("Invalid keyed additive family tag".into()), + } + } else { + ExactKind::Sum + }; + Ok(Self { + family, + sums, + counts, + }) } } -impl Default for MultipleSumAccumulator { +impl Default for KeyedSumCountAccumulator { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleSumAccumulator { +impl SerializableToSink for KeyedSumCountAccumulator { fn serialize_to_json(&self) -> Value { let mut sums_obj = serde_json::Map::new(); for (key, sum) in &self.sums { @@ -175,6 +210,7 @@ impl SerializableToSink for MultipleSumAccumulator { } serde_json::json!({ + "family": if self.family == ExactKind::Count { "Count" } else { "Sum" }, "sums": sums_obj, "counts": counts_obj }) @@ -211,17 +247,23 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + buffer.push(if self.family == ExactKind::Count { + 1 + } else { + 0 + }); + buffer } } -impl AggregateCore for MultipleSumAccumulator { +impl AggregateCore for KeyedSumCountAccumulator { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleSumAccumulator" + "KeyedSumCountAccumulator" } fn as_any(&self) -> &dyn std::any::Any { @@ -236,20 +278,20 @@ impl AggregateCore for MultipleSumAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleSumAccumulator + // Check if other is also a KeyedSumCountAccumulator if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleSumAccumulator with {}", + "Cannot merge KeyedSumCountAccumulator with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleSumAccumulator + // Downcast to KeyedSumCountAccumulator let other_multiple_sum = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleSumAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedSumCountAccumulator")?; // Use the existing merge_accumulators method let merged = Self::merge_accumulators(vec![self.clone(), other_multiple_sum.clone()])?; @@ -258,7 +300,11 @@ impl AggregateCore for MultipleSumAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleSum + if self.family == ExactKind::Count { + AggregationType::Count + } else { + AggregationType::Sum + } } fn approx_memory_bytes(&self) -> usize { @@ -281,35 +327,35 @@ impl AggregateCore for MultipleSumAccumulator { use crate::storage_engines::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() - .ok_or("Key required for MultipleSumAccumulator")?; + .ok_or("Key required for KeyedSumCountAccumulator")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleSumAccumulator { +impl MultipleSubpopulationAggregate for KeyedSumCountAccumulator { fn query( &self, statistic: Statistic, key: &KeyByLabelValues, _query_kwargs: Option<&HashMap>, ) -> Result> { - match statistic { - Statistic::Sum => self - .sums - .get(key) - .copied() - .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), - Statistic::Count => self + match (&self.family, statistic) { + (ExactKind::Sum, Statistic::Sum) => self.sums.get(key).copied().ok_or_else(|| { + "Key not found in KeyedSumCountAccumulator" + .to_string() + .into() + }), + (ExactKind::Count, Statistic::Count) => self .counts .get(key) .map(|count| *count as f64) .ok_or_else(|| { - "Sample count unavailable in MultipleSumAccumulator" + "Sample count unavailable in KeyedSumCountAccumulator" .to_string() .into() }), _ => Err( - format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), + format!("Unsupported statistic in KeyedSumCountAccumulator: {statistic:?}").into(), ), } } @@ -319,15 +365,19 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { } } -impl MergeableAccumulator for MultipleSumAccumulator { +impl MergeableAccumulator for KeyedSumCountAccumulator { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleSumAccumulator::new(); + let family = accumulators[0].family.clone(); + if accumulators.iter().any(|acc| acc.family != family) { + return Err("Cannot merge different keyed additive families".into()); + } + let mut result = KeyedSumCountAccumulator::for_family(family); for acc in accumulators { for key in acc.sums.keys() { @@ -366,14 +416,14 @@ mod tests { use super::*; #[test] - fn test_multiple_sum_accumulator_creation() { - let acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_creation() { + let acc = KeyedSumCountAccumulator::new(); assert!(acc.sums.is_empty()); } #[test] - fn test_multiple_sum_accumulator_update() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_update() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -390,25 +440,35 @@ mod tests { #[test] fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); - let mut first = MultipleSumAccumulator::new(); + let mut first = KeyedSumCountAccumulator::for_family(ExactKind::Count); first.update(key.clone(), 10.0); first.update(key.clone(), 20.0); - let mut second = MultipleSumAccumulator::new(); + let mut second = KeyedSumCountAccumulator::for_family(ExactKind::Count); second.update(key.clone(), 7.0); - let merged = MultipleSumAccumulator::merge_accumulators(vec![first, second]).unwrap(); + let merged = KeyedSumCountAccumulator::merge_accumulators(vec![first, second]).unwrap(); for acc in [ merged.clone(), - MultipleSumAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), - MultipleSumAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), ] { - assert_eq!(acc.query(Statistic::Sum, &key, None).unwrap(), 37.0); + assert_eq!(acc.family, ExactKind::Count); + assert!(acc.query(Statistic::Sum, &key, None).is_err()); assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); } } #[test] - fn test_multiple_sum_accumulator_query() { - let mut acc = MultipleSumAccumulator::new(); + fn keyed_additive_merge_rejects_different_planner_families() { + assert!(KeyedSumCountAccumulator::merge_accumulators(vec![ + KeyedSumCountAccumulator::for_family(ExactKind::Sum), + KeyedSumCountAccumulator::for_family(ExactKind::Count), + ]) + .is_err()); + } + + #[test] + fn test_keyed_sum_count_accumulator_query() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -427,8 +487,8 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_get_keys() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_get_keys() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -444,9 +504,9 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_merge() { - let mut acc1 = MultipleSumAccumulator::new(); - let mut acc2 = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_merge() { + let mut acc1 = KeyedSumCountAccumulator::new(); + let mut acc2 = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -457,15 +517,15 @@ mod tests { acc2.add_sum(key1.clone(), 5.0); // Same key, different accumulator - let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.sums.get(&key1), Some(&15.0)); // Should be merged assert_eq!(merged.sums.get(&key2), Some(&20.0)); // Should be preserved } #[test] - fn test_multiple_sum_accumulator_serialization() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_serialization() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -473,18 +533,18 @@ mod tests { // Test JSON serialization let json = acc.serialize_to_json(); - let deserialized = MultipleSumAccumulator::deserialize_from_json(&json).unwrap(); + let deserialized = KeyedSumCountAccumulator::deserialize_from_json(&json).unwrap(); assert_eq!(deserialized.sums.get(&key), Some(&42.5)); // Test byte serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = MultipleSumAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedSumCountAccumulator::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.sums.get(&key), Some(&42.5)); } #[test] fn test_trait_object() { - let mut acc = MultipleSumAccumulator::new(); + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -493,6 +553,6 @@ mod tests { let trait_obj: Box = Box::new(acc); // Test type name through trait object - assert_eq!(trait_obj.type_name(), "MultipleSumAccumulator"); + assert_eq!(trait_obj.type_name(), "KeyedSumCountAccumulator"); } } diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index af284459..f90cff65 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -8,12 +8,12 @@ pub mod edge_runtime_adapter; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; +pub mod keyed_sum_count_accumulator; pub mod max_accumulator; pub mod min_accumulator; pub mod multiple_increase_accumulator; pub mod multiple_max_accumulator; pub mod multiple_min_accumulator; -pub mod multiple_sum_accumulator; pub mod sketch_envelope_accumulator; pub mod sum_accumulator; pub mod univmon_accumulator; @@ -27,11 +27,11 @@ pub use dd_sketch_accumulator::*; pub use hll_sketch_accumulator::*; pub use hydra_kll_accumulator::*; pub use increase_accumulator::*; +pub use keyed_sum_count_accumulator::*; pub use max_accumulator::*; pub use min_accumulator::*; pub use multiple_increase_accumulator::*; pub use multiple_max_accumulator::*; pub use multiple_min_accumulator::*; -pub use multiple_sum_accumulator::*; pub use sketch_envelope_accumulator::*; pub use sum_accumulator::*; diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 2c48006f..16ecaa5e 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1497,7 +1497,7 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { /// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted /// from the series' **aggregated_labels** — these are the labels that become /// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a MultipleSumAccumulator's HashMap). This matches the Arroyo SQL +/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL /// pattern: `udf(concat_ws(';', aggregated_labels), value)`. pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, @@ -1792,7 +1792,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::storage_engines::types::StreamingConfig; @@ -2452,7 +2452,7 @@ mod tests { #[test] fn test_keyed_accumulator_aggregated_labels() { // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside MultipleSumAccumulator) + // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) let config = make_agg_config_full( 3, "cpu", @@ -2504,10 +2504,10 @@ mod tests { let (_output, acc) = &captured[0]; let ms_acc = acc .as_any() - .downcast_ref::() - .expect("should be MultipleSumAccumulator"); + .downcast_ref::() + .expect("should be KeyedSumCountAccumulator"); - // The MultipleSumAccumulator should have two internal keys: "A" and "B" + // The KeyedSumCountAccumulator should have two internal keys: "A" and "B" assert_eq!(ms_acc.sums.len(), 2, "two host keys inside one accumulator"); let mut found_a = false; diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 0721c8a0..b067a0ac 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; -use asap_types::AggregationType; +use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; use crate::query_engines::EngineError; @@ -51,65 +51,40 @@ impl ResolvedMaterialization<'_> { matches!( &self.summary.operator, SummaryOperator::Configured { - aggregation_type: AggregationType::Sum - | AggregationType::Count - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::Rate - | AggregationType::MultipleIncrease - | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleMin - | AggregationType::MultipleMax, + family: SummaryFamilyType::ExactAggregate(..), .. } ) } fn supports(&self, node: &QueryPlanNode) -> bool { - let SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - .. - } = &self.summary.operator - else { + let SummaryOperator::Configured { family, .. } = &self.summary.operator else { // Partial legacy descriptors cannot attest a configured capability. return false; }; - use AggregationType::*; match node { - QueryPlanNode::ExactReadout { readout, .. } => match readout { - ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Count, - ExactReadout::Increase => matches!(aggregation_type, Increase | MultipleIncrease), - ExactReadout::Rate => *aggregation_type == Rate, - // Direction is the family now -- no `aggregation_sub_type` - // cross-check, and a minimum summary can no longer be - // offered up for a maximum readout. - ExactReadout::Min => matches!(aggregation_type, Min | MultipleMin), - ExactReadout::Max => matches!(aggregation_type, Max | MultipleMax), - }, + QueryPlanNode::ExactReadout { readout, .. } => family == &readout.planner_family(), QueryPlanNode::SummaryEstimate { query, .. } => match query { QueryReadout::Quantile { q } => { q.is_finite() && (0.0..=1.0).contains(q) - && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) + } + QueryReadout::Cardinality => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon)) } - QueryReadout::Cardinality => matches!(aggregation_type, HLL | UnivMon), QueryReadout::FrequencyL2 | QueryReadout::FrequencyEntropy => { - *aggregation_type == UnivMon + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) } - QueryReadout::PointCount { value: None, .. } if *aggregation_type == UnivMon => { + QueryReadout::PointCount { value: None, .. } if matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) => { true } - QueryReadout::PointCount { .. } => matches!( - aggregation_type, - CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap - ), - QueryReadout::TopK { .. } => matches!( - aggregation_type, - CountMinSketchWithHeap | CountSketchWithHeap - ), + QueryReadout::PointCount { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap)) + } + QueryReadout::TopK { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + } }, _ => false, } diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 97458531..b3ebde4b 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -65,8 +65,8 @@ use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryNode, + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, }; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; @@ -203,13 +203,13 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match agg_type { - AggregationType::Sum | AggregationType::MultipleSum => asap_types::Statistic::Sum, - AggregationType::Count => asap_types::Statistic::Count, - AggregationType::Increase | AggregationType::MultipleIncrease => { + let stat = match agg_type.planner_exact_family()? { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => asap_types::Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => asap_types::Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { asap_types::Statistic::Increase } - AggregationType::Rate => asap_types::Statistic::Rate, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -237,38 +237,25 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Count) => { - asap_types::Statistic::Count - } - ( - asap_types::query_plan::ExactReadout::Sum, - AggregationType::Sum | AggregationType::MultipleSum, - ) => asap_types::Statistic::Sum, - ( - asap_types::query_plan::ExactReadout::Increase, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Increase, - (asap_types::query_plan::ExactReadout::Rate, AggregationType::Rate) => { - asap_types::Statistic::Rate - } - ( - asap_types::query_plan::ExactReadout::Min, - AggregationType::Min | AggregationType::MultipleMin, - ) => asap_types::Statistic::Min, - ( - asap_types::query_plan::ExactReadout::Max, - AggregationType::Max | AggregationType::MultipleMax, - ) => asap_types::Statistic::Max, - _ => return None, + if agg_type.planner_exact_family().as_ref() != Some(&readout.planner_family()) { + return None; + } + let stat = match readout { + asap_types::query_plan::ExactReadout::Count => asap_types::Statistic::Count, + asap_types::query_plan::ExactReadout::Sum => asap_types::Statistic::Sum, + asap_types::query_plan::ExactReadout::Increase => asap_types::Statistic::Increase, + asap_types::query_plan::ExactReadout::Rate => asap_types::Statistic::Rate, + asap_types::query_plan::ExactReadout::Min => asap_types::Statistic::Min, + asap_types::query_plan::ExactReadout::Max => asap_types::Statistic::Max, }; // Temporal exact summaries are the hot path for long-window // dashboards. Merge their concrete, fixed-size states in one batch // instead of allocating a boxed trait object for every pane. if matches!( - agg_type, - AggregationType::Increase | AggregationType::Rate | AggregationType::MultipleIncrease + readout, + asap_types::query_plan::ExactReadout::Increase + | asap_types::query_plan::ExactReadout::Rate ) { let accumulators = entries .iter() @@ -285,11 +272,7 @@ impl GroupState { ]); return merged.query_statistic(stat, key, &query_kwargs).ok(); } - if matches!( - agg_type, - AggregationType::Min | AggregationType::MultipleMin - ) && readout == asap_types::query_plan::ExactReadout::Min - { + if readout == asap_types::query_plan::ExactReadout::Min { return entries .iter() .flat_map(|windows| windows.values()) @@ -302,11 +285,7 @@ impl GroupState { .into_iter() .reduce(f64::min); } - if matches!( - agg_type, - AggregationType::Max | AggregationType::MultipleMax - ) && readout == asap_types::query_plan::ExactReadout::Max - { + if readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() .flat_map(|windows| windows.values()) @@ -333,9 +312,6 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == asap_types::query_plan::ExactReadout::Count { - return merged.aux_stats().count.map(|count| count as f64); - } merged.query_statistic(stat, key, &query_kwargs).ok() } @@ -589,11 +565,13 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + let exact_family = agg_type.planner_exact_family(); if matches!( - agg_type, - AggregationType::Increase - | AggregationType::Rate - | AggregationType::MultipleIncrease + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Increase | ExactKind::Rate, + _ + )) ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -609,12 +587,12 @@ impl QueryExecutionContext<'_> { )); } } - if let Some((reduction, is_min)) = match agg_type { - AggregationType::Min | AggregationType::MultipleMin => Some(( + if let Some((reduction, is_min)) = match exact_family.as_ref() { + Some(SummaryFamilyType::ExactAggregate(ExactKind::Min, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Min, true, )), - AggregationType::Max | AggregationType::MultipleMax => Some(( + Some(SummaryFamilyType::ExactAggregate(ExactKind::Max, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Max, false, )), @@ -666,10 +644,11 @@ impl QueryExecutionContext<'_> { // counter and extrema state. Additive pane summaries must // remain contiguous because a missing pane is not zero. if matches!( - agg_type, - AggregationType::Sum - | AggregationType::Count - | AggregationType::MultipleSum + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count, + _ + )) ) { check_panes(windows.keys().copied().collect())?; } @@ -914,9 +893,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } ( - GroupState::ExactAgg { entries, .. }, - GroupState::ExactAgg { entries: more, .. }, + GroupState::ExactAgg { entries, agg_type }, + GroupState::ExactAgg { + entries: more, + agg_type: incoming, + }, ) => { + if agg_type.planner_exact_family() != incoming.planner_exact_family() { + return Err(SummaryExecutorError::UnsupportedFamily); + } entries.extend(more); } // `find_candidates`'s exact-match contract never produces a @@ -1265,21 +1250,12 @@ fn summary_family_matches_sketch( /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (family, agg_type), - ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - AggregationType::Sum | AggregationType::MultipleSum, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Count, ExactParams::Count), - AggregationType::Count, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - AggregationType::Increase | AggregationType::MultipleIncrease, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate), - AggregationType::Rate, + family, + SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count | ExactKind::Increase | ExactKind::Rate, + _ ) - ) + ) && agg_type.planner_exact_family().as_ref() == Some(family) } /// Project a full label-values map down to the requested `by` columns -- @@ -1449,6 +1425,58 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + #[test] + fn keyed_count_state_follows_planner_family_and_query_readout() { + use crate::precompute_engine::operators::KeyedSumCountAccumulator; + use asap_types::query_plan::ExactReadout; + + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut payload = KeyedSumCountAccumulator::for_family(ExactKind::Count); + payload.update(key.clone(), 10.0); + payload.update(key.clone(), 20.0); + let state = GroupState::ExactAgg { + entries: vec![Rc::new(BTreeMap::from([( + 60_000, + Arc::new(payload) as Arc, + )]))], + agg_type: AggregationType::Count, + }; + assert_eq!( + state.exact_value_for(ExactReadout::Count, &Some(key.clone()), 0, 60_000), + Some(2.0) + ); + assert_eq!( + state.exact_value_for(ExactReadout::Sum, &Some(key), 0, 60_000), + None + ); + } + + #[test] + fn state_merge_rejects_different_planner_families() { + let index = SketchStore::new(); + let context = QueryExecutionContext { + index: &index, + t0_ms: 0, + t1_ms: 60_000, + is_cumulative: true, + allowed_materializations: None, + }; + let states = vec![ + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Rate, + }, + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Increase, + }, + ]; + assert!(matches!( + context.merge_states(states), + Err(SummaryExecutorError::UnsupportedFamily) + )); + } + #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { let binding = asap_types::query_plan::MaterializationBinding { diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e0c9ba55..ad15c141 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -92,8 +92,8 @@ fn reconstruct_exact_agg( bytes: &[u8], ) -> Option> { use crate::precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleIncreaseAccumulator, - MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, + MultipleIncreaseAccumulator, SumAccumulator, }; use crate::storage_engines::types::AggregateCore; match type_name { @@ -109,9 +109,11 @@ fn reconstruct_exact_agg( "MaxAccumulator" => MaxAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), - "MultipleSumAccumulator" => MultipleSumAccumulator::deserialize_from_bytes(bytes) - .ok() - .map(|a| Box::new(a) as Box), + "KeyedSumCountAccumulator" | "MultipleSumAccumulator" => { + KeyedSumCountAccumulator::deserialize_from_bytes(bytes) + .ok() + .map(|a| Box::new(a) as Box) + } "MultipleIncreaseAccumulator" => MultipleIncreaseAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b56408a2..a10cd3d1 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,4 +1,4 @@ -use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{KeyedSumCountAccumulator, SumAccumulator}; #[cfg(test)] use crate::storage_engines::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, @@ -18,7 +18,7 @@ fn test_single_subpopulation_interface() { #[test] fn test_multiple_subpopulation_interface() { // Multiple accumulator - matches Python behavior exactly - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let mut key = KeyByLabelValues::new(); key.insert("web".to_string()); @@ -43,7 +43,7 @@ fn test_interface_prevents_misuse() { let single_acc: Box = Box::new(SumAccumulator::with_sum(42.0)); let multi_acc: Box = - Box::new(MultipleSumAccumulator::new()); + Box::new(KeyedSumCountAccumulator::new()); // ✅ These work - correct usage let _result1 = single_acc.query(Statistic::Sum, None); @@ -68,7 +68,7 @@ fn test_python_alignment() { // Python: multiple_accumulator.query(Statistic.SUM, key) // Rust: multiple_accumulator.query(Statistic::Sum, &key) - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new(); multi_acc.add_sum(key.clone(), 100.0); let multi_trait: Box = Box::new(multi_acc); diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 2522bb02..d7b3b9ce 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -197,9 +197,13 @@ The registry holds weak references, so retiring the final SID also releases its descriptors. `SketchInstanceMetadata` remains the registration and persistence compatibility DTO while older sidecars are read. -The implemented `SummaryDescriptor` currently contains one `SummaryOperator`, -one derived `FidelityGuarantee`, and a numeric state-schema version. The -implemented `DataDescriptor` contains typed source and value projections, a +The implemented `SummaryDescriptor` contains one `SummaryOperator`, +one derived `FidelityGuarantee`, and a numeric state-schema version. A configured +operator carries Planner's `SummaryFamilyType` as its semantic identity. Its +backend aggregation type and parameters describe the state codec and update +implementation; keyed grouping remains in the Data Descriptor. Descriptor +validation rejects a configured exact family that disagrees with its storage +type. The implemented `DataDescriptor` contains typed source and value projections, a canonical population filter, typed grouping columns and versioned observation semantics. The shared contract now also defines `SummaryInstance`, `ObservedSummaryInventory`, placement, completeness, From a842c9fd35be118009d1e1b5412c1bd687eeaeb4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 16:14:38 +0000 Subject: [PATCH 14/16] refactor: execute precompute from validated post-ASAP DAGs --- control_plane/src/clickhouse.rs | 4 +- control_plane/src/emit/backend_wire.rs | 2 +- control_plane/src/emit/mod.rs | 2 +- control_plane/src/physical/backend_stage.rs | 2 +- control_plane/src/physical/compiler.rs | 68 +-- control_plane/src/physical/pane_reuse.rs | 5 +- control_plane/src/physical/post_asap/lower.rs | 47 +- control_plane/src/physical/post_asap/tests.rs | 32 +- .../src/physical/runtime_capability.rs | 113 ++-- control_plane/src/workload.rs | 86 +-- crates/asap_types/src/accumulator_spec.rs | 184 ++---- crates/asap_types/src/aggregation_config.rs | 109 ++-- crates/asap_types/src/aggregation_type.rs | 91 ++- crates/asap_types/src/key_by_label_names.rs | 2 +- crates/asap_types/src/monitor_spec.rs | 2 +- crates/asap_types/src/policy_fingerprint.rs | 26 +- crates/asap_types/src/policy_registry.rs | 22 +- crates/asap_types/src/precompute_plan.rs | 21 +- crates/asap_types/src/query_plan.rs | 16 + crates/asap_types/src/routing_index.rs | 19 +- crates/asap_types/src/sds.rs | 72 ++- crates/asap_types/src/summary_catalog.rs | 2 +- data_plane/benches/sketch_db.rs | 4 +- data_plane/src/drivers/ingest/otel.rs | 35 +- .../drivers/ingest/prometheus_remote_write.rs | 17 +- data_plane/src/drivers/query/servers/http.rs | 428 +------------- data_plane/src/lib.rs | 8 +- .../precompute_engine/accumulator_factory.rs | 538 +++++++++++++----- .../src/precompute_engine/erp_observer.rs | 6 +- .../src/precompute_engine/ingest_handler.rs | 22 +- .../precompute_engine/maintenance_runtime.rs | 13 +- data_plane/src/precompute_engine/mod.rs | 1 + .../operators/exact_accumulator.rs | 327 +++++++++++ ..._accumulator.rs => keyed_counter_state.rs} | 85 ++- ..._max_accumulator.rs => keyed_max_state.rs} | 69 ++- ..._min_accumulator.rs => keyed_min_state.rs} | 69 ++- ...ator.rs => keyed_sum_count_accumulator.rs} | 268 +++++++-- .../src/precompute_engine/operators/mod.rs | 17 +- .../operators/sum_accumulator.rs | 22 +- .../src/precompute_engine/output_sink.rs | 8 +- data_plane/src/precompute_engine/raw_dag.rs | 295 ++++++++++ .../src/precompute_engine/series_router.rs | 8 +- .../src/precompute_engine/window_manager.rs | 2 +- data_plane/src/precompute_engine/worker.rs | 450 ++++++++++----- .../asap_query_engine/catalog_resolver.rs | 56 +- .../asap_query_engine/exact_subqueries.rs | 4 +- .../asap_query_engine/post_asap_readout.rs | 4 +- .../asap_query_engine/summary_executor.rs | 194 ++++--- data_plane/src/query_engines/query_result.rs | 2 +- .../src/storage_engines/sketch_db/accuracy.rs | 45 +- .../storage_engines/sketch_db/backfill/mod.rs | 8 +- .../sketch_db/backfill/processor.rs | 61 +- .../sketch_db/backfill/raw_sample_reader.rs | 2 +- .../sketch_db/backfill/service.rs | 8 +- .../sketch_db/backfill/window_builder.rs | 43 +- .../src/storage_engines/sketch_db/data/mod.rs | 8 +- .../storage_engines/sketch_db/index/mod.rs | 126 +++- .../sketch_db/lifecycle/eviction.rs | 6 +- .../sketch_db/lifecycle/reconcile.rs | 14 +- .../src/storage_engines/sketch_db/mod.rs | 12 +- .../types/hot_reload_config.rs | 6 +- data_plane/src/storage_engines/types/mod.rs | 2 +- .../types/precomputed_output.rs | 4 +- .../storage_engines/types/streaming_config.rs | 298 +++------- .../accuracy_empirical_validation_tests.rs | 6 +- .../tests/test_utilities/engine_factories.rs | 36 +- data_plane/src/tests/trait_design_tests.rs | 8 +- data_plane/src/utils/file_io.rs | 7 +- .../asapquery_compatibility_process_e2e.rs | 9 +- ...e2e_controller_plans_and_backend_serves.rs | 12 +- docs/design_docs/precompute-dag-execution.md | 24 + .../summary-catalog-sds-architecture.md | 10 +- 72 files changed, 2739 insertions(+), 1895 deletions(-) create mode 100644 data_plane/src/precompute_engine/operators/exact_accumulator.rs rename data_plane/src/precompute_engine/operators/{multiple_increase_accumulator.rs => keyed_counter_state.rs} (86%) rename data_plane/src/precompute_engine/operators/{multiple_max_accumulator.rs => keyed_max_state.rs} (81%) rename data_plane/src/precompute_engine/operators/{multiple_min_accumulator.rs => keyed_min_state.rs} (81%) rename data_plane/src/precompute_engine/operators/{multiple_sum_accumulator.rs => keyed_sum_count_accumulator.rs} (50%) create mode 100644 data_plane/src/precompute_engine/raw_dag.rs create mode 100644 docs/design_docs/precompute-dag-execution.md diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f1e1b1df..cbf69121 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -378,7 +378,7 @@ fn materialize_selected_sql( let aggregation = BackendAggregation { aggregation_id: String::new(), metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: crate::physical::compiler::physical_materialization_family(family), + family: family.clone(), window_secs, spatial_filter: String::new(), grouping: grouping.names(), @@ -591,7 +591,7 @@ fn bind_selected_node( .. } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = crate::physical::compiler::physical_materialization_family(family); + let expected = family.clone(); let selected = select_materialization( &request.precompute_plan.materializations, &table_ref, diff --git a/control_plane/src/emit/backend_wire.rs b/control_plane/src/emit/backend_wire.rs index 829d1be4..ff294e8c 100644 --- a/control_plane/src/emit/backend_wire.rs +++ b/control_plane/src/emit/backend_wire.rs @@ -5,7 +5,7 @@ //! * the storage-routing table, which maps each metric's materialized summary //! families to the query shapes the ASAP tier serves natively versus the //! ones that belong to the archive; -//! * the aggregation and readout JSON the backend's `AggregationConfig` +//! * the aggregation and readout JSON the backend's `PrecomputeMaterialization` //! parser consumes. //! //! `backend_plan::from_stage_config` reuses [`build_backend_aggregation_json`] diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c2ede7ee..d9d13f75 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1,7 +1,7 @@ //! Backend-facing emission for a compiled physical plan. //! //! * [`backend_wire`] builds the storage-routing table and the aggregation / -//! readout JSON the backend's `AggregationConfig` parser consumes. +//! readout JSON the backend's `PrecomputeMaterialization` parser consumes. //! * [`monitor`] carries the CDM monitor declarations. pub mod backend_wire; diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs index a9a64f6a..c3fc78f0 100644 --- a/control_plane/src/physical/backend_stage.rs +++ b/control_plane/src/physical/backend_stage.rs @@ -35,7 +35,7 @@ pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, /// Source metric the aggregation runs over. Required by the backend's - /// `AggregationConfig` parser. + /// `PrecomputeMaterialization` parser. pub metric_name: String, /// Planner-owned committed summary identity. Sketch entries carry a /// validated `SketchKind` (category + algorithm + params); exact entries diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 747fc171..1987994c 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1262,10 +1262,7 @@ impl PhysicalPlanCompiler { .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - // Rate is a readout over the same reset-aware counter state - // as Increase. Keep that semantic distinction in QueryPlan, - // while the physical store binds both to Increase state. - let physical_family = physical_materialization_family(&selected.family); + let physical_family = selected.family.clone(); let physical_algorithm = match &physical_family { SummaryFamilyType::ExactAggregate(kind, _) => { format!("{kind:?}").to_ascii_lowercase() @@ -1704,7 +1701,7 @@ impl PhysicalPlanCompiler { .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? .family; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != physical_materialization_family(node_family) + if materialization_family != *node_family || window_ms == 0 || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 @@ -2828,13 +2825,11 @@ fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), A::DDSketch => 64 * 1024, A::Sum + | A::Count | A::Increase + | A::Rate | A::Min | A::Max - | A::MultipleSum - | A::MultipleIncrease - | A::MultipleMin - | A::MultipleMax | A::SingleSubpopulation | A::MultipleSubpopulation => 256, } @@ -2852,7 +2847,7 @@ fn retained_partition_count( if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( materialization.aggregation_type, - A::Increase | A::MultipleIncrease | A::Min | A::Max | A::MultipleMin | A::MultipleMax + A::Increase | A::Rate | A::Min | A::Max ) || !materialization.grouping_labels.names().is_empty() { @@ -3099,7 +3094,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -fn raw_time_series_input_contract( +pub fn raw_time_series_input_contract( expr: &QueryExpr, exact: bool, ) -> Result<(String, Option, String), String> { @@ -3270,7 +3265,7 @@ fn physical_aggregation( BackendAggregation { aggregation_id, metric_name: selected.metric.clone(), - family: physical_materialization_family(&selected.family), + family: selected.family.clone(), window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected @@ -3680,26 +3675,6 @@ fn collect_selected_materializations( Ok(selected) } -pub(crate) fn physical_materialization_family(family: &SummaryFamilyType) -> SummaryFamilyType { - match family { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) => { - // The SummaryStore Sum accumulator retains the observation count - // alongside its sum. Both logical states can share this producer. - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ) - } - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Rate, _) => { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, - ) - } - _ => family.clone(), - } -} - fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { @@ -4540,8 +4515,7 @@ pub(crate) mod tests { .find(|materialization| { matches!( materialization.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Rate ) }) .expect("reset-aware exact counter"); @@ -5069,8 +5043,7 @@ pub(crate) mod tests { .iter() .all(|m| !matches!( m.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Increase | asap_types::AggregationType::Rate ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); @@ -5706,7 +5679,7 @@ pub(crate) mod tests { } #[test] - fn rate_and_increase_share_physical_counter_state() { + fn rate_and_increase_keep_planner_families_distinct() { let mut workload = request("rate", "rate(m[1m])"); workload .queries @@ -5715,10 +5688,14 @@ pub(crate) mod tests { .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert_eq!(collector.materializations[0].algorithm, "increase"); + let algorithms: std::collections::BTreeSet<_> = collector + .materializations + .iter() + .map(|materialization| materialization.algorithm.as_str()) + .collect(); + assert_eq!(algorithms, ["increase", "rate"].into()); } } @@ -5738,8 +5715,7 @@ pub(crate) mod tests { } #[test] - fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { - // Both dashboard roots use one packed raw accumulator, with explicit readouts. + fn exact_dashboard_preserves_distinct_sum_and_count_producers() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5755,7 +5731,7 @@ pub(crate) mod tests { entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); let bundle = PhysicalPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { assert!( @@ -5765,7 +5741,7 @@ pub(crate) mod tests { )), "{entry:?}" ); - assert_eq!(entry.materialization_bindings().len(), 1); + assert!(!entry.materialization_bindings().is_empty()); } assert!(bundle .query_plan @@ -7479,8 +7455,8 @@ pub(crate) mod tests { assert_eq!( materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, + planner_types::post_asap::ExactKind::Rate, + planner_types::post_asap::ExactParams::Rate, ) ); } diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index b76cdbf0..e1e29a4d 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -34,10 +34,7 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!( - m.aggregation_type, - AggregationType::Sum | AggregationType::MultipleSum - ) + || !matches!(m.aggregation_type, AggregationType::Sum) { continue; } diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index 5d168c89..87729ac6 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -8,14 +8,6 @@ //! variants when a binding rule fires; everything else stays inside //! `Logical(…)`." //! -//! Two node shapes are rewritten *before* selecting a candidate, because -//! the upstream strategy can produce summaries this deployment's data plane -//! doesn't (or, deliberately, shouldn't) serve — not something the -//! `CostModel` hook can reach, since the decision of *whether* to call -//! into `rank_candidates`/`size_params` at all is made before the -//! `CostModel` is ever consulted. See each helper's docs for the specific -//! reason. -//! //! `AggIntent::Extension` (the `Frequency` point-query) needs no such //! pre-pass anymore: `ControlPlaneCostModel::realize_extension`/ //! `readout_extension` (ASAPController#150) now realize it as a real @@ -143,41 +135,8 @@ fn bind_recursive( )) } - _ => { - let rewritten = rewrite_rate_to_increase(expr); - let node = crate::planner_selection::select_summary(&rewritten, cost_model)?; - Ok(PostAsapPlan::Summary(node)) - } - } -} - -/// Rewrite Rate to Increase along the aggregate spine traversed by Planner. -/// This deployment computes rate by dividing the Increase readout by window -/// seconds, rather than storing a separate Rate accumulator. -fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs - .iter() - .map(|intent| { - if matches!(intent, AggIntent::Rate) { - AggIntent::Increase - } else { - intent.clone() - } - }) - .collect(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(rewrite_rate_to_increase(child)), - }, - other => other.clone(), + _ => Ok(PostAsapPlan::Summary( + crate::planner_selection::select_summary(expr, cost_model)?, + )), } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index 3f12dec6..ad255841 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -519,13 +519,9 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { /// `ONLY_SPATIAL` — `sum by (host) (m)`. /// Control plane path: `Aggregate{Sum, by=[host]}` over a bare `Scan`. /// -/// The old locally-defined `AggregationType::MultipleSum` (keyed vs -/// unkeyed sum) identity no longer exists at the L4 IR level — -/// `SummaryKind::Sum` covers both; the keyed/unkeyed distinction now -/// lives on `SummaryAgg::by` (non-empty ⇒ the old "MultipleSum" shape), -/// per `emit::mod.rs`'s exact-accumulator classification notes. +/// Family remains Sum; the reduction carries the grouping columns. #[test] -fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { +fn phase_b_pattern_only_spatial_aggregate_binds_to_grouped_sum() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), // service column measures: vec![AggIntent::Sum { col: None }], @@ -546,7 +542,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { assert_eq!( reduction.group_keys().map(|k| k.keys()), Some(&[1][..]), - "keyed sum must carry the group-by column (the MultipleSum-equivalent signal)" + "Sum reduction must retain the group-by column" ); } other => panic!("expected SummaryAgg(Sum, by=[1]), got {other:?}"), @@ -556,14 +552,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { } /// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// `bind_query_expr` (not `implement_tree` directly) rewrites -/// `AggIntent::Rate` to `AggIntent::Increase` before binding (see -/// `lower.rs`'s `rewrite_rate_to_increase` — this deployment's data -/// plane has no Rate accumulator). The old -/// `AggregationType::MultipleIncrease` identity is now -/// `SummaryKind::Increase` with a non-empty `by`. +/// Planner preserves the Rate family and the `by` reduction independently. #[test] -fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { +fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), measures: vec![AggIntent::Rate], @@ -579,11 +570,11 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { } => { assert_eq!( family, - &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } - other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), + other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), }, other => panic!("expected Committed(Summary(_)), got {other:?}"), } @@ -716,12 +707,9 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { ); } -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Control plane path: `Aggregate{Rate}` over -/// `Window` → `bind_query_expr` rewrites `Rate` to `Increase` and binds an -/// exact accumulator (`SummaryAgg{Increase}`) — no approximate summary -/// family. Both paths produce a single non-summary streaming row; the L5 -/// emitter is the one that picks the actual MultipleIncrease processor. +/// A Rate query keeps Planner's exact Rate family through binding. The +/// physical emitter chooses the runtime processor without changing that +/// family identity. #[test] fn phase_b_e2e_rate_falls_through_to_logical() { let bound = pipeline_l1_to_l4( diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 5b9f407c..7ebe2f03 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -101,7 +101,7 @@ pub enum Capability { /// * Sum-over-time requires archive execution because cumulative samples /// cannot be reconstructed from delta state alone. /// -/// Rate and Increase require the Increase capability; plain sum requires Sum. +/// Rate and Increase have distinct exact-family capabilities. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -275,8 +275,7 @@ impl Capability { /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` /// has no wildcard concept of its own; family-matching subsumes it. - /// `ExactAgg` is intentionally NOT routed through this path — see - /// [`multi_pop_satisfies_single`]'s doc for why. + /// Exact families require identity; keyed layout is checked separately. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { @@ -313,22 +312,9 @@ impl Capability { SketchAlgorithm::CmsWithHeap, ) } - // Exact-aggregation family: the agg_type must match - // exactly OR be the single-pop ⇆ multi-pop equivalent. A - // `MultipleSum` policy can serve a `Sum` query by - // re-aggregating across keys; the `find_matching_policies` - // group_by ⊆ policy_grouping_labels check is what - // ultimately decides whether the re-aggregation is - // semantically valid. The reverse direction (single-pop - // serving multi-pop) is NOT allowed — the single-pop - // policy has lost the key dimension and can't recover it. - // - // Exact counter summaries are a distinct state contract. A sum of - // cumulative sample values cannot reconstruct reset correction or - // Prometheus boundary extrapolation. - (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have || multi_pop_satisfies_single(*req, *have) - } + // Exact family identity must match. Grouping compatibility is + // checked separately by population routing. + (Capability::ExactAgg(req), Capability::ExactAgg(have)) => req == have, _ => false, } } @@ -348,30 +334,12 @@ fn sketch_algorithms_compatible( sketch_family_satisfied(required, available) } -/// True when `available` is the multi-population equivalent of -/// `required`'s single-population variant — i.e. a `MultipleSum` -/// policy can serve a `Sum` query (via re-aggregation across keys), -/// `MultipleIncrease` can serve `Increase`, `MultipleMax` can -/// serve `Max`. Asymmetric: this returns `false` for the reverse -/// direction (single-pop can't recover keys that have been collapsed -/// away). -fn multi_pop_satisfies_single(required: AggregationType, available: AggregationType) -> bool { - matches!( - (required, available), - (AggregationType::Sum, AggregationType::MultipleSum) - | (AggregationType::Increase, AggregationType::MultipleIncrease) - | (AggregationType::Min, AggregationType::MultipleMin) - | (AggregationType::Max, AggregationType::MultipleMax) - ) -} - // ── AggIntent → Capability bridge ──────────────────────────────────────────── #[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` for intents that have no ASAP-tier sketch -/// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent -/// — see [`AggIntent::archive_only`]). +/// answer it. Returns `None` when no deployed ASAP-tier capability can +/// satisfy the intent. /// /// This is a runtime routing requirement, not a summary-selection rule. /// ASAPPlanner owns legal implementations and candidate enumeration; this @@ -395,15 +363,17 @@ pub fn capability_for(intent: &AggIntent) -> Option { } match intent { AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Count { accuracy } if is_exact(accuracy) => { + Some(Capability::ExactAgg(AggregationType::Count)) + } // Direction is part of the capability: a stored minimum cannot // answer `max_over_time` and vice versa, so these must not // collapse onto one `ExactAgg` the way they did while Planner // had a single `MinMax` accumulator. AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase | AggIntent::Rate => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } + AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), + AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::QuantileApprox(None)) } @@ -532,18 +502,14 @@ mod tests { } #[test] - fn capability_for_count_exact_routes_to_archive() { - // `count_over_time` lowers to `Count{accuracy:Exact}`. The - // PR #200/#201 follow-up briefly routed this to - // `ExactAgg(Sum)`, but the data plane has no count - // accumulator — `SumAccumulator` returns its `sum` for both - // `Statistic::Sum` and `Statistic::Count`, so the result was - // sum-of-values, not sample-count. Reverted to `None` (archive - // routing) until a real `SumCountAccumulator` lands. + fn capability_for_count_exact_preserves_count_family() { let intent = AggIntent::Count { accuracy: AccuracyTarget::Exact, }; - assert_eq!(capability_for(&intent), None); + assert_eq!( + capability_for(&intent), + Some(Capability::ExactAgg(AggregationType::Count)) + ); } #[test] @@ -593,16 +559,16 @@ mod tests { assert!(!Capability::ExactAgg(AggregationType::Max) .is_satisfied_by(&Capability::ExactAgg(AggregationType::Min))); assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleMax))); + .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); } #[test] - fn capability_for_rate_increase_route_to_exact_agg_increase() { - // PR-6 follow-up: Rate and Increase route to ASAP-tier - // ExactAgg(Increase) — the counter-reset-aware exact precompute. - // Pre-follow-up this returned `None`. + fn capability_for_rate_and_increase_preserves_family() { let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!(capability_for(&AggIntent::Rate), exact_inc); + assert_eq!( + capability_for(&AggIntent::Rate), + Some(Capability::ExactAgg(AggregationType::Rate)) + ); assert_eq!(capability_for(&AggIntent::Increase), exact_inc); } @@ -849,10 +815,6 @@ mod tests { AggregationType::Min, AggregationType::Max, AggregationType::DatasketchesKLL, - AggregationType::MultipleSum, - AggregationType::MultipleIncrease, - AggregationType::MultipleMin, - AggregationType::MultipleMax, AggregationType::HydraKLL, AggregationType::CountMinSketch, AggregationType::CountMinSketchWithHeap, @@ -873,21 +835,16 @@ mod tests { fn sum_family_cannot_impersonate_exact_counter_state() { let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); - assert!( - !required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum)) - ); + let required_multi = Capability::ExactAgg(AggregationType::Increase); + assert!(!required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } #[test] fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { - // Same single/multi-population direction as multi_pop_satisfies_single: - // a single-pop available (Sum) can't serve a multi-pop required - // capability (MultipleIncrease) -- it already lost the per-key - // breakdown a multi-pop caller needs. - let required = Capability::ExactAgg(AggregationType::MultipleIncrease); + // A different exact family cannot supply counter state. + let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } @@ -905,30 +862,26 @@ mod tests { // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] - fn exact_agg_routing_covers_sum_rate_increase_only() { - // `Capability::ExactAgg` routing covers the three intents the - // data plane has a real accumulator for: `Sum` (SumAccumulator) - // and `Rate` / `Increase` (IncreaseAccumulator). + fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { assert_eq!( capability_for(&AggIntent::Sum { col: None }), Some(Capability::ExactAgg(AggregationType::Sum)) ); assert_eq!( capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Increase)) + Some(Capability::ExactAgg(AggregationType::Rate)) ); assert_eq!( capability_for(&AggIntent::Increase), Some(Capability::ExactAgg(AggregationType::Increase)) ); - // `Count{Exact}` (count_over_time) and `Avg` both need a real - // count accumulator that doesn't exist yet — they route to - // archive until `SumCountAccumulator` lands. + // Exact count follows the Planner Count family; Avg still needs + // its own composition contract. assert_eq!( capability_for(&AggIntent::Count { accuracy: AccuracyTarget::Exact, }), - None + Some(Capability::ExactAgg(AggregationType::Count)) ); assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); } diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 6bb7fc32..d6b74cd4 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -18,8 +18,7 @@ use planner_types::pre_asap::AggIntent; /// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` /// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): /// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Sum`] -/// (rate binds to ExactAgg(Sum)-shaped capability) +/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] /// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] /// /// Before this enum: the `WorkloadStore` was keyed by metric name alone @@ -30,7 +29,7 @@ use planner_types::pre_asap::AggIntent; /// shape). /// /// After: the store is keyed by `(metric, role)` so each shape gets its -/// own plan, its own `AggregationConfig` on the backend's streaming +/// own plan, its own `PrecomputeMaterialization` on the backend's streaming /// config, and its own routing-connector pipeline. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -39,15 +38,15 @@ pub enum AggRole { /// or workload entries with `sketch_family_override: DDSketch | KLL`. /// Routes to a quantile-shaped sketch (DDSketch / KLL). Quantile, - /// Bare counter selector, `sum(...)`, `sum_over_time(...)`, - /// `rate(...)`, `increase(...)`. All bind to ExactAgg(Sum)-shaped - /// capability on the data plane; the streaming-config emits an - /// `aggregation_type: Sum` rather than a sketch. + /// Bare selector or Sum-shaped exact aggregation. Sum, + /// Reset-aware per-second counter rate. + Rate, + /// Reset-aware counter increase over the selected window. + Increase, /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to a Sum-as-count - /// exact-aggregation. + /// to HLL when a sketch is appropriate, otherwise to exact Count. Count, /// `topk(...)`, or workload entries with /// `sketch_family_override: CountSketch | CountMinSketch`. Routes @@ -70,6 +69,8 @@ impl AggRole { match self { AggRole::Quantile => "quantile", AggRole::Sum => "sum", + AggRole::Rate => "rate", + AggRole::Increase => "increase", AggRole::Count => "count", AggRole::Topk => "topk", AggRole::Other => "other", @@ -97,15 +98,16 @@ impl std::fmt::Display for AggRole { /// through the same canonical pipeline the live serving path uses /// (`query_parser::parse_query_expr_canonical` → /// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent (the one bound to the data-plane capability) is matched: +/// intent is matched, except that a Sum wrapping a counter function +/// keeps the inner Rate or Increase role: /// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] /// * [`AggIntent::TopK`] → [`AggRole::Topk`] /// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the /// windowed-Count-as-Frequency extension /// (`intent_algebra::as_frequency`) → [`AggRole::Count`] /// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → [`AggRole::Sum`] -/// * Anything else recognised but not one of the four shapes above +/// → their respective roles +/// * Anything else recognised but not one of the listed shapes above /// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → /// [`AggRole::Other`]. /// * Bare metric selector (no `Aggregate` node at all) → @@ -125,11 +127,7 @@ impl std::fmt::Display for AggRole { /// semantics, whichever the lowerer picks). The Sum-shaped /// alternative is rare in practice; users who want it write /// `sum_over_time(count(...))` which classifies as Sum. -/// * `rate` / `irate` / `increase` — Sum. `irate` folds onto -/// `AggIntent::Rate` at L3 same as `rate`; both bind to -/// ExactAgg(Increase) on the data plane (see -/// `data_plane/src/precompute_engine/ingest_handler.rs`'s handling -/// of `AggKind::ExactAgg { Increase }`). +/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { // 1. `sketch_family_override` wins. if let Some(family) = entry.sketch_family_override.as_ref() { @@ -167,11 +165,30 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { if crate::planner_selection::as_frequency(outer).is_some() { return AggRole::Count; } + // A spatial `sum by (...)` around a counter function still requires the + // counter family's state; using Sum as the registration key would let it + // overwrite a bare Sum workload for the same metric. + if matches!(outer, AggIntent::Sum { .. }) { + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Rate)) + { + return AggRole::Rate; + } + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Increase)) + { + return AggRole::Increase; + } + } match outer { AggIntent::Quantile { .. } => AggRole::Quantile, AggIntent::TopK { .. } => AggRole::Topk, AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } | AggIntent::Rate | AggIntent::Increase => AggRole::Sum, + AggIntent::Sum { .. } => AggRole::Sum, + AggIntent::Rate => AggRole::Rate, + AggIntent::Increase => AggRole::Increase, _ => AggRole::Other, } } @@ -970,13 +987,7 @@ mod tests { #[test] fn agg_role_sum_query_strings() { - for q in [ - "sum by (zone) (m)", - "sum_over_time(m[5m])", - "rate(m[5m])", - "increase(m[5m])", - "sum by (zone) (rate(m[5m]))", - ] { + for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { assert_eq!( derive_agg_role(&entry("m", Some(q), None)), AggRole::Sum, @@ -985,6 +996,18 @@ mod tests { } } + #[test] + fn counter_functions_have_distinct_workload_roles() { + for (query, expected) in [ + ("rate(m[5m])", AggRole::Rate), + ("sum by (zone) (rate(m[5m]))", AggRole::Rate), + ("increase(m[5m])", AggRole::Increase), + ("sum by (zone) (increase(m[5m]))", AggRole::Increase), + ] { + assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); + } + } + #[test] fn agg_role_count_query_strings() { for q in [ @@ -1095,7 +1118,7 @@ mod tests { } #[test] - fn three_synthetic_http_requests_total_entries_classify_to_two_distinct_roles() { + fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { // Synthetic mirror of `deploy/configs/mvp-workload.yaml` // entries 2/3/4 — proves `derive_agg_role` produces distinct // roles for the three http_requests_total shapes. Pre-B2 the @@ -1120,15 +1143,8 @@ mod tests { ), ]; let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Sum, AggRole::Count]); - // The store distinguishes Sum vs Count keys, so two of the - // three entries (the two Sum-shaped ones) still collide - // under (metric, role). That's the documented behaviour — - // two YAML entries with the SAME (metric, role) overwrite, - // which is the legitimate "operator updated their workload" - // path. The fix scope is collisions across DIFFERENT shapes, - // not idempotent re-registers. + assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); + assert_eq!(distinct.len(), 3); } } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 482ef6d6..bdf5949b 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,68 +1,12 @@ -//! Typed accumulator dispatch derived from legacy streaming config. +//! Validate stored materialization descriptors against Planner summary families. //! -//! The semantic identity is ASAPPlanner's [`SummaryFamilyType`]. This module -//! only adds the backend execution concern of keyed versus unkeyed state and -//! adapts the stable legacy wire fields into that canonical representation. -//! -//! ## This is an additive representation, not a replacement (yet) -//! -//! `AggregationConfig` keeps its `aggregation_type` / `aggregation_sub_type` -//! / `parameters` fields untouched. Two hard constraints ruled out full -//! removal in this pass: -//! -//! 1. **`PolicyFingerprint` hash stability.** [`crate::policy_fingerprint`] -//! hashes `aggregation_type` / `aggregation_sub_type` / `parameters` -//! directly, and its own module doc is explicit that the byte layout -//! it produces is a stability *contract* ("Don't reorder fields... -//! any such change invalidates every deployed fingerprint and forces -//! a cold-start rebuild"). Changing what feeds that hash — even by -//! routing it through an equivalent typed shape — risks producing a -//! different byte sequence for the same logical policy, which strands -//! on-disk sids after a deploy. `policy_fingerprint.rs` is -//! deliberately **not touched** by this module; it keeps reading the -//! original three fields, unchanged. -//! 2. **Consumer fan-out.** `AggregationType` is read by ~40 files across -//! `data_plane` and `asap_types` — persistence (`sid_metadata.json` -//! round-trip), query-time capability matching -//! (`capability_matching.rs`, unrelated to accumulator dispatch), -//! the query engine, reconciliation, index maintenance — not just -//! `accumulator_factory.rs` (the single highest-risk consumer, and -//! the one this module targets). Migrating all of them in one PR was -//! judged too large to land and review safely; that's tracked as -//! follow-up, not done here. -//! -//! So: `AccumulatorSpec` is *computed from* `AggregationConfig`'s -//! existing fields via [`AggregationConfig::accumulator_spec`], and -//! consumed by `data_plane::precompute_engine::accumulator_factory` -//! instead of the raw fields. The wire format (`aggregationType` / -//! `aggregationSubType` / `parameters` JSON/YAML keys) is completely -//! unaffected — nothing here changes how `AggregationConfig::from_yaml` -//! / `from_json` parse or how `serialize_to_json` emits. -//! -//! Backend-specific execution details remain deliberately separate: -//! -//! - **Min/max direction.** Direction is part of the family now, not a -//! string riding alongside it: `AggregationType::{Min, Max}` (and the -//! keyed `{MultipleMin, MultipleMax}`) map to `ExactKind::Min` and -//! `ExactKind::Max` respectively — upstream still spells its -//! maximum accumulator `MinMax`, but it is a maximum. Nothing reads -//! `AggregationConfig::aggregation_sub_type` for the direction any -//! more, so a min state can no longer content-address onto a max one. -//! - **HydraKLL's `(row, col)` tiling.** `SketchParams::Kll` carries -//! only `k` — upstream has no concept of the CMS-like grid-of-KLL-cells -//! layout `HydraKllSketchAccumulator` uses to parallelize a keyed KLL -//! across many populations. `accumulator_factory.rs` calls -//! [`cms_params`] directly for keyed KLL execution -//! arm, same extraction the plain CMS arms use, because `w`/`d` are -//! genuinely the same wire keys for both. -//! - **Top-k ranking mode (`weight_mode`).** Not a sketch structural -//! parameter — a data_plane-only "what to accumulate" axis -//! (`accumulator_factory::TopkWeight`) with no upstream equivalent. -//! Stays a raw-`parameters`-reading helper in `accumulator_factory.rs`. +//! This projection supports catalog identity and imported state metadata. It is +//! not an execution program. Raw and maintenance execution dispatch directly +//! on the selected post-ASAP DAG payload; the descriptor must agree with it. use serde_json::Value; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; use planner_types::post_asap::{ @@ -75,8 +19,8 @@ use planner_types::post_asap::{ /// accumulator to run (`kind`), with what tuning (`params`), and /// whether it's keyed by a group-by label set (`grouping`). /// -/// Computed on demand from an [`AggregationConfig`] via -/// [`AggregationConfig::accumulator_spec`] — not stored on the config +/// Computed on demand from an [`PrecomputeMaterialization`] via +/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config /// itself, so there is exactly one source of truth for the fields that /// feed [`crate::policy_fingerprint::PolicyFingerprint`]. #[derive(Debug, Clone, PartialEq)] @@ -85,10 +29,7 @@ pub struct AccumulatorSpec { /// validated `SketchKind` (category + algorithm + params), following the /// ASAP-aware-mapping vocabulary. pub family: SummaryFamilyType, - /// `Some(labels)` for a keyed (multi-population) accumulator, - /// `None` for a single-population one. This is the axis - /// `AggregationType` wrongly folded into identity (`Sum` vs - /// `MultipleSum`) — here it's a sibling field instead. + /// Physical keyed-state layout, independent of semantic family. pub grouping: Option, } @@ -96,8 +37,8 @@ pub struct AccumulatorSpec { /// /// This is execution semantics, separate from the summary family: the same /// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Legacy streaming artifacts still encode the rule in -/// `parameters`; callers use [`AggregationConfig::sample_update_rule`] so the +/// counter deltas. Stored descriptors encode the rule in +/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the /// runtime does not branch on ad-hoc strings. #[derive(Debug, Clone, Copy, PartialEq)] pub enum SampleUpdateRule { @@ -134,7 +75,7 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) } -impl AggregationConfig { +impl PrecomputeMaterialization { pub fn sample_update_rule(&self) -> SampleUpdateRule { let scale = self .parameters @@ -157,23 +98,14 @@ impl AggregationConfig { } } -/// Why [`AggregationConfig::accumulator_spec`] couldn't resolve a config -/// into an [`AccumulatorSpec`]. Each variant matches one of the three -/// distinct fallback paths `accumulator_factory::create_accumulator_updater` -/// took pre-Step-5 — preserved verbatim (including which default -/// updater and which warning text each one produced) so this refactor -/// changes *how* the dispatch is expressed, not what it does for any -/// input. +/// A storage descriptor cannot be resolved to a supported Planner family. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccumulatorSpecError { /// `aggregation_type` was `SingleSubpopulation` with an /// `aggregation_sub_type` string not in the recognized alias list. - /// Pre-Step-5 this defaulted to `SumAccumulatorUpdater`. UnknownSingleSubpopulationSubType(String), /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. Pre-Step-5 this defaulted - /// to `MultipleSumAccumulatorUpdater` (note: a *different* default - /// than the `SingleSubpopulation` case). + /// unrecognized `aggregation_sub_type`. UnknownMultipleSubpopulationSubType(String), /// `aggregation_type` itself has no accumulator-dispatch mapping. /// Also returned for an invalid HLL precision. A resolved family identifies @@ -185,36 +117,24 @@ impl std::fmt::Display for AccumulatorSpecError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::UnknownSingleSubpopulationSubType(s) => { - write!( - f, - "Unknown SingleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown SingleSubpopulation sub_type '{s}'") } Self::UnknownMultipleSubpopulationSubType(s) => { - write!( - f, - "Unknown MultipleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") } - Self::UnmappedAggregationType(t) => write!( - f, - "Unknown aggregation_type '{t:?}', defaulting to SingleSubpopulation Sum" - ), + Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), } } } impl std::error::Error for AccumulatorSpecError {} -impl AggregationConfig { +impl PrecomputeMaterialization { /// Resolve this config's `(aggregation_type, aggregation_sub_type, /// parameters)` triple into a typed [`AccumulatorSpec`]. /// - /// Mirrors `accumulator_factory::create_accumulator_updater`'s - /// pre-Step-5 dispatch exactly — same sub_type alias lists, same - /// numeric defaults, same three fallback paths (see - /// [`AccumulatorSpecError`]) — just re-expressed as data instead of - /// as a 14-arm match baked into the accumulator constructor. + /// Unsupported descriptors return an error; this projection never chooses + /// a fallback family and cannot authorize DAG execution. pub fn accumulator_spec(&self) -> Result { use AggregationType::*; @@ -227,21 +147,9 @@ impl AggregationConfig { ) }; let (family, keyed) = match self.aggregation_type { - Sum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - Increase => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - Min => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - Max => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, + Sum | Count | Increase | Rate | Min | Max => ( + self.aggregation_type.planner_exact_family().unwrap(), + !self.aggregated_labels.is_empty(), ), DatasketchesKLL => ( independent_sketch( @@ -252,22 +160,6 @@ impl AggregationConfig { ), false, ), - MultipleSum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - MultipleIncrease => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - MultipleMin => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - MultipleMax => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), HydraKLL => { let k = kll_k_param(self) as u32; ( @@ -497,7 +389,7 @@ impl AggregationConfig { /// Extract the KLL `k` parameter. Capital `"K"` takes precedence over /// lowercase `"k"` to match the convention used by the top-level /// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &AggregationConfig) -> u16 { +pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { config .parameters .get("K") @@ -513,7 +405,7 @@ pub fn kll_k_param(config: &AggregationConfig) -> u16 { /// matches what the control plane's `sketch_params_to_json` emits and /// what `sketch_config_to_params` uses for OTLP policy_fp content /// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { +pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { let row_num = config .parameters .get("d") @@ -530,7 +422,7 @@ pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { /// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` /// from `parameters`; defaults to 20 (the heap holds the top-k /// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &AggregationConfig) -> usize { +pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { config .parameters .get("heap_size") @@ -545,7 +437,7 @@ pub fn heap_size_param(config: &AggregationConfig) -> usize { /// Pull `relativeAccuracy` (or canonical aliases) out of a /// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, /// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { +pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { let parsed = param_f64(config, "relativeAccuracy") .or_else(|| param_f64(config, "relative_accuracy")) .or_else(|| param_f64(config, "alpha")) @@ -561,7 +453,7 @@ pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { } } -fn param_f64(config: &AggregationConfig, key: &str) -> Option { +fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { config.parameters.get(key).and_then(Value::as_f64) } @@ -599,8 +491,8 @@ mod tests { sub_type: &str, params: HashMap, grouping_labels: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), params, @@ -630,13 +522,9 @@ mod tests { } #[test] - fn multiple_sum_is_keyed_sum() { - let cfg = make_config( - AggregationType::MultipleSum, - "", - HashMap::new(), - vec!["zone"], - ); + fn keyed_layout_preserves_sum_family() { + let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); + cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); let spec = cfg.accumulator_spec().expect("resolves"); assert_exact(&spec, ExactKind::Sum); assert_eq!( @@ -862,16 +750,16 @@ mod tests { assert_eq!( AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown SingleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown MultipleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL', defaulting to SingleSubpopulation Sum" + "Unknown aggregation_type 'HLL'" ); } @@ -905,7 +793,7 @@ mod tests { // ---- PolicyFingerprint stability guard --------------------------- /// `accumulator_spec()` must be a pure, additional *read* of - /// `AggregationConfig` — it must not change what + /// `PrecomputeMaterialization` — it must not change what /// `PolicyFingerprint::from_config` hashes. This locks in a fixed /// fingerprint for a fixed config as a tripwire: if this test ever /// needs its expected constant updated, `policy_fingerprint.rs` diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 92dfd41d..f4fe8699 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -87,8 +87,9 @@ impl WindowMaterializationLayout { } } -/// Per-aggregation policy with content-derived [`PolicyFingerprint`] identity. -/// An `aggregationId` field in input YAML is ignored for compatibility. +/// Physical materialization metadata with content-derived identity. +/// This descriptor cannot authorize execution: the enclosing PrecomputePlan +/// must bind it to a compatible Planner DAG producer. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputeMaterialization { pub aggregation_type: AggregationType, @@ -196,10 +197,6 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} -/// Compatibility name for legacy streaming-config and precompute call sites. -/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. -pub type AggregationConfig = PrecomputeMaterialization; - impl PrecomputeMaterialization { pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection @@ -338,7 +335,7 @@ impl PrecomputeMaterialization { /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s - /// `HashMap` keys). **Always** equal to + /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. pub fn policy_fp_u64(&self) -> u64 { @@ -818,12 +815,18 @@ mod tests { /// SAME config as a fixture without it. #[test] fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = - AggregationConfig::from_yaml_data(&sample_yaml(true), None, QueryLanguage::PromQl) - .expect("parse ok"); - let without = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse ok"); + let with = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(true), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); + let without = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); assert_eq!( with.policy_fingerprint(), without.policy_fingerprint(), @@ -834,10 +837,18 @@ mod tests { /// Round-tripping the same content yields the same fingerprint. #[test] fn fingerprint_is_deterministic_per_content() { - let a = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse a"); - let b = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse b"); + let a = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse a"); + let b = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse b"); assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); assert_ne!( a.policy_fingerprint().as_u64(), @@ -862,37 +873,44 @@ mod tests { ] { yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); let config = - AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) + .unwrap(); assert_eq!(config.window_layout, layout); let mut wire = config.serialize_to_json(); wire["groupingLabels"] = serde_json::to_value(&config.grouping_labels).unwrap(); wire["aggregatedLabels"] = serde_json::to_value(&config.aggregated_labels.labels).unwrap(); wire["rollupLabels"] = serde_json::to_value(&config.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&wire).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); assert_eq!(decoded.window_layout, layout); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); wire["window_layout"] = wire["windowLayout"].clone(); - assert!(AggregationConfig::deserialize_from_json(&wire).is_err()); + assert!(PrecomputeMaterialization::deserialize_from_json(&wire).is_err()); } yaml.as_mapping_mut() .unwrap() .remove(serde_yaml::Value::from("windowLayout")); - let legacy = AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + let legacy = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); assert_eq!( legacy.window_layout, WindowMaterializationLayout::Pane { pane_secs: 10 } ); yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!(AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err()); + assert!( + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() + ); } #[test] fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let mut epoch = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let unknown = epoch.policy_fingerprint(); epoch.pane_origin_ms = Some(7_000); let planned = epoch.policy_fingerprint(); @@ -910,13 +928,13 @@ mod tests { .as_object_mut() .unwrap() .insert("paneOriginMs".into(), origin); - let decoded: AggregationConfig = serde_json::from_value(derived.clone()).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(derived.clone()).unwrap(); assert_eq!(decoded.pane_origin_ms, Some(7_000)); let mut legacy = derived; legacy.as_object_mut().unwrap().remove("paneOriginMs"); assert_eq!( - serde_json::from_value::(legacy) + serde_json::from_value::(legacy) .expect("decode legacy wire") .pane_origin_ms, None @@ -926,18 +944,24 @@ mod tests { /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. #[test] fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); } /// PR 5: `serialize_to_json` no longer emits `aggregationId`. #[test] fn serialize_to_json_omits_aggregation_id() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let json = cfg.serialize_to_json(); assert!( json.get("aggregationId").is_none(), @@ -949,9 +973,12 @@ mod tests { fn typed_projection_roundtrips_and_legacy_column_keeps_identity() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .unwrap(); + let mut config = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .unwrap(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -960,7 +987,7 @@ mod tests { let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); - let decoded: AggregationConfig = serde_json::from_value(legacy).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(legacy).unwrap(); assert_eq!(decoded.policy_fingerprint(), column_identity); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), @@ -977,8 +1004,8 @@ mod tests { "rollup": config.rollup_labels.serialize_to_json(), }); assert!(wire.get("valueColumn").is_none()); - let json = AggregationConfig::deserialize_from_json(&wire).unwrap(); - let yaml = AggregationConfig::from_yaml_data( + let json = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); + let yaml = PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(&wire).unwrap(), None, QueryLanguage::ClickHouseSql, @@ -994,8 +1021,8 @@ mod tests { ); let mut conflicting = wire; conflicting["valueColumn"] = serde_json::json!("other_column"); - assert!(AggregationConfig::deserialize_from_json(&conflicting).is_err()); - assert!(AggregationConfig::from_yaml_data( + assert!(PrecomputeMaterialization::deserialize_from_json(&conflicting).is_err()); + assert!(PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(conflicting).unwrap(), None, QueryLanguage::ClickHouseSql diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ccdcbec0..647f7604 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -14,15 +14,13 @@ use std::str::FromStr; pub enum AggregationType { // ---------- single-population (non-keyed) ---------- Sum, + Count, Increase, + Rate, Min, Max, DatasketchesKLL, // ---------- multi-population (keyed) ---------- - MultipleSum, - MultipleIncrease, - MultipleMin, - MultipleMax, HydraKLL, CountMinSketch, CountMinSketchWithHeap, @@ -38,17 +36,31 @@ pub enum AggregationType { } impl AggregationType { + /// Adapt a storage/processor tag to Planner's exact family. Keyed storage + /// changes the payload layout, not the semantic family. + pub fn planner_exact_family(self) -> Option { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + _ => return None, + }; + Some(SummaryFamilyType::ExactAggregate(kind, params)) + } + pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", + AggregationType::Count => "Count", AggregationType::Increase => "Increase", + AggregationType::Rate => "Rate", AggregationType::Min => "Min", AggregationType::Max => "Max", AggregationType::DatasketchesKLL => "DatasketchesKLL", - AggregationType::MultipleSum => "MultipleSum", - AggregationType::MultipleIncrease => "MultipleIncrease", - AggregationType::MultipleMin => "MultipleMin", - AggregationType::MultipleMax => "MultipleMax", AggregationType::HydraKLL => "HydraKLL", AggregationType::CountMinSketch => "CountMinSketch", AggregationType::CountMinSketchWithHeap => "CountMinSketchWithHeap", @@ -67,10 +79,6 @@ impl AggregationType { matches!( self, AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax | AggregationType::CountMinSketch | AggregationType::CountMinSketchWithHeap | AggregationType::CountSketch @@ -93,14 +101,12 @@ impl FromStr for AggregationType { match s { // Canonical names "Sum" => Ok(AggregationType::Sum), + "Count" => Ok(AggregationType::Count), "Increase" => Ok(AggregationType::Increase), + "Rate" => Ok(AggregationType::Rate), "Min" => Ok(AggregationType::Min), "Max" => Ok(AggregationType::Max), "DatasketchesKLL" => Ok(AggregationType::DatasketchesKLL), - "MultipleSum" => Ok(AggregationType::MultipleSum), - "MultipleIncrease" => Ok(AggregationType::MultipleIncrease), - "MultipleMin" => Ok(AggregationType::MultipleMin), - "MultipleMax" => Ok(AggregationType::MultipleMax), "HydraKLL" => Ok(AggregationType::HydraKLL), "CountMinSketch" => Ok(AggregationType::CountMinSketch), "CountMinSketchWithHeap" => Ok(AggregationType::CountMinSketchWithHeap), @@ -121,12 +127,6 @@ impl FromStr for AggregationType { "DatasketchesKLLAccumulator" | "KLL" | "kll" | "datasketches_kll" => { Ok(AggregationType::DatasketchesKLL) } - "MultipleSumAccumulator" | "multiple_sum" => Ok(AggregationType::MultipleSum), - "MultipleIncreaseAccumulator" | "multiple_increase" => { - Ok(AggregationType::MultipleIncrease) - } - "MultipleMinAccumulator" | "multiple_min" => Ok(AggregationType::MultipleMin), - "MultipleMaxAccumulator" | "multiple_max" => Ok(AggregationType::MultipleMax), "HydraKllSketchAccumulator" | "hydra_kll" => Ok(AggregationType::HydraKLL), "CountMinSketchAccumulator" | "CMS" | "cms" | "count_min_sketch" => { Ok(AggregationType::CountMinSketch) @@ -149,7 +149,7 @@ impl FromStr for AggregationType { | "MultipleMinMaxAccumulator" | "multiple_min_max" => Err(format!( "Retired aggregation type: '{s}' -- min and max are separate types now, \ - use 'Min'/'Max' (or 'MultipleMin'/'MultipleMax')" + use 'Min'/'Max'" )), _ => Err(format!("Unknown aggregation type: '{s}'")), } @@ -168,3 +168,48 @@ impl<'de> Deserialize<'de> for AggregationType { s.parse().map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + /// Removed layout tags cannot be installed as semantic families. + #[test] + fn rejects_keyed_family_aliases() { + for name in [ + "MultipleSum", + "MultipleIncrease", + "MultipleMin", + "MultipleMax", + ] { + assert!(name.parse::().is_err(), "{name}"); + } + } + + #[test] + fn storage_layout_tags_do_not_create_planner_families() { + for (storage, expected) in [ + (AggregationType::Sum, ExactKind::Sum), + (AggregationType::Count, ExactKind::Count), + (AggregationType::Increase, ExactKind::Increase), + (AggregationType::Rate, ExactKind::Rate), + ] { + let family = storage.planner_exact_family().unwrap(); + assert!( + matches!(family, SummaryFamilyType::ExactAggregate(kind, _) if kind == expected) + ); + } + assert_eq!( + AggregationType::Rate.planner_exact_family(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Rate, + ExactParams::Rate + )) + ); + assert_ne!( + AggregationType::Rate.planner_exact_family(), + AggregationType::Increase.planner_exact_family() + ); + } +} diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index deb7fe3f..5cd902b7 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -2,7 +2,7 @@ //! //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` -//! (`AggregationConfig::grouping_labels`, `PolicyFingerprint`, +//! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, //! `PolicyRegistry`, `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index 535ec3ef..a22352be 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -44,7 +44,7 @@ impl MonitorFunctional { /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both -/// sides can reach — same reasoning as `AggregationConfig`/`PolicyFingerprint`. +/// sides can reach — same reasoning as `PrecomputeMaterialization`/`PolicyFingerprint`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct MonitorSpec { pub agg_id: u64, diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index ca7353aa..b68e385c 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -4,7 +4,7 @@ //! the controller-allocated `aggregation_id: u64`. Where `aggregation_id` //! is a counter the control plane mints and ships in the streaming-config //! YAML, `PolicyFingerprint` is derived deterministically from the -//! `AggregationConfig`'s content — so two control planes producing the +//! `PrecomputeMaterialization`'s content — so two control planes producing the //! same policy independently produce the same fingerprint, and the data //! plane can index without a separate id allocation. //! @@ -14,7 +14,7 @@ //! grouping_labels, aggregated_labels, rollup_labels, window_size, //! slide_interval, window_type, pane_origin_ms, spatial_filter_normalized)` //! -//! The hash includes **every** field of `AggregationConfig` that +//! The hash includes **every** field of `PrecomputeMaterialization` that //! determines what the policy does — sketch / exact-agg shape, //! group-by + rollup layout, window cadence, spatial filter. Two //! configs that compare equal on these dimensions produce the same @@ -50,9 +50,9 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use xxhash_rust::xxh64::xxh64; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; -/// Stable, content-addressed handle for an `AggregationConfig`. +/// Stable, content-addressed handle for an `PrecomputeMaterialization`. /// /// Wrap a `u64` so callers can't accidentally swap a `PolicyFingerprint` /// with an `aggregation_id` — they're both u64-shaped but they index @@ -75,14 +75,14 @@ impl PolicyFingerprint { } impl PolicyFingerprint { - /// Compute the fingerprint of an [`AggregationConfig`]. + /// Compute the fingerprint of an [`PrecomputeMaterialization`]. /// /// Hash inputs are concatenated with `\0` byte separators and /// canonicalized so that map/iteration order can't affect the /// outcome. Parameter values are rendered via `serde_json::to_string` /// for nested-shape determinism (matches the existing /// `parameters_canonical` form used in `AggKind::ExactAgg`). - pub fn from_config(cfg: &AggregationConfig) -> Self { + pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { let mut buf: Vec = Vec::with_capacity(512); if !cfg.population_key_encoding.is_legacy() { @@ -265,8 +265,8 @@ mod tests { group_by: Vec<&str>, window_size: u64, spatial_filter: &str, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -298,7 +298,7 @@ mod tests { ); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); let mut canonical = legacy.clone(); @@ -306,7 +306,7 @@ mod tests { assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); use crate::traits::SerializableToSink; let mut sink = canonical.serialize_to_json(); @@ -315,7 +315,7 @@ mod tests { sink["aggregatedLabels"] = serde_json::to_value(&canonical.aggregated_labels.labels).unwrap(); sink["rollupLabels"] = serde_json::to_value(&canonical.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&sink).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&sink).unwrap(); assert_eq!( decoded.population_key_encoding, canonical.population_key_encoding @@ -462,7 +462,7 @@ mod tests { ); } - /// Pre-PR-5 the `aggregation_id` field on `AggregationConfig` was + /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was /// excluded from the fingerprint hash. PR 5 deletes the field /// entirely — identity *is* the fingerprint — so this is now /// vacuously true. Kept as a doc-comment anchor; no runtime test @@ -525,7 +525,7 @@ mod tests { fn spatial_filter_canonicalization_drives_fingerprint() { // Two filters that differ only in matcher ordering produce the // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `AggregationConfig::new` (via + // canonicalization step in `PrecomputeMaterialization::new` (via // `normalize_spatial_filter`) sorts matchers by key. let a = cfg( "http_lat", diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 4b4f9e95..dde5e5cc 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -1,7 +1,7 @@ //! Content-addressed policy registry. //! -//! Derived view over a collection of `AggregationConfig`s that maps -//! [`PolicyFingerprint`] → [`AggregationConfig`]. This is the +//! Derived view over a collection of `PrecomputeMaterialization`s that maps +//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated //! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` //! carries (see `data_plane::storage_engines::types::streaming_config`'s @@ -20,7 +20,7 @@ //! //! ## Identity invariants //! -//! Two `AggregationConfig`s that produce the same `PolicyFingerprint` +//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* //! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write @@ -30,13 +30,13 @@ use std::collections::HashMap; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; /// Content-addressed lookup table for active aggregation policies. #[derive(Debug, Clone, Default)] pub struct PolicyRegistry { - policies: HashMap, + policies: HashMap, } impl PolicyRegistry { @@ -46,7 +46,7 @@ impl PolicyRegistry { /// them. pub fn from_configs(configs: I) -> Self where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); for cfg in configs { @@ -63,7 +63,7 @@ impl PolicyRegistry { /// surfacing. pub fn from_configs_with_collisions(configs: I) -> (Self, usize) where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); let mut collisions = 0usize; @@ -77,12 +77,12 @@ impl PolicyRegistry { } /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.policies.get(&fp) } /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.policies.iter() } @@ -110,11 +110,11 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(_id: u64, metric: &str) -> AggregationConfig { + fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is derived from // content. Kept as a parameter so existing call sites in the // tests below don't churn. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 746649fc..ae8b0ad0 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -101,11 +101,10 @@ pub struct PlanEnvelope { pub capability_snapshot_id: String, } -/// Backend-side materialization projection consumed by the streaming -/// precompute engine. This is deliberately config-driven: it contains no -/// PromQL string or ad-hoc scheduler job. The aggregation definitions are -/// emitted to `/api/v1/streaming-config`, where the runtime matches incoming -/// series, maintains windows, and writes content-addressed materializations. +/// DAG-format precompute installation. Planner node payloads and dependency +/// edges define execution; materializations attach storage/window placement. +/// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths +/// execute through the maintenance DAG scheduler at stored-state frontiers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -155,6 +154,8 @@ pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, ExactAccumulatorV1, + /// Persisted backend state with explicit Planner family and population layout. + PlannerExactAccumulatorV1, ExactCounterAccumulatorV2, } @@ -904,8 +905,14 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate, _, - ) => vec![StateEncoding::ExactCounterAccumulatorV2], - SummaryFamilyType::ExactAggregate(..) => vec![StateEncoding::ExactAccumulatorV1], + ) => vec![ + StateEncoding::ExactCounterAccumulatorV2, + StateEncoding::PlannerExactAccumulatorV1, + ], + SummaryFamilyType::ExactAggregate(..) => vec![ + StateEncoding::ExactAccumulatorV1, + StateEncoding::PlannerExactAccumulatorV1, + ], SummaryFamilyType::Sketch(kind, _) if matches!( kind.algorithm(), diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 5bdd2605..2ebb6404 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -637,6 +637,22 @@ pub enum ExactReadout { Max, } +impl ExactReadout { + /// Planner family required by this installed DAG readout node. + pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + }; + SummaryFamilyType::ExactAggregate(kind, params) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryReadout { diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 4e0dd515..40a90fe3 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `AggregationConfig`s, so it represents planned policy +//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -32,7 +32,7 @@ use std::collections::{BTreeSet, HashMap}; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; use crate::policy_registry::PolicyRegistry; @@ -62,7 +62,7 @@ impl RoutingIndex { /// Tier 1 — exact fingerprint lookup. Delegates to the underlying /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.registry.get(fp) } @@ -136,8 +136,8 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(metric: &str) -> AggregationConfig { - AggregationConfig::new( + fn cfg(metric: &str) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -172,7 +172,7 @@ mod tests { fn multiple_policies_for_the_same_metric_all_bucket_together() { // Same metric, distinct group-by shapes -> distinct fingerprints, // same bucket. - let a = AggregationConfig::new( + let a = PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -220,8 +220,9 @@ mod tests { #[test] fn len_and_is_empty_match_registry() { - let idx = - RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let idx = RoutingIndex::build(PolicyRegistry::from_configs( + Vec::::new(), + )); assert!(idx.is_empty()); assert_eq!(idx.len(), 0); @@ -234,7 +235,7 @@ mod tests { fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { let mut parameters = StdHashMap::new(); parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::DDSketch, String::new(), parameters, diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index c0621614..218cd7a2 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -383,6 +383,9 @@ pub enum SummaryOperator { /// Complete planner materialization configuration, including heap/Hydra /// dimensions and readout/update subtype. Never equal to a legacy projection. Configured { + /// Planner-selected semantic family; grouping and pane layout live in + /// the data descriptor and summary definition, respectively. + family: planner_types::post_asap::SummaryFamilyType, aggregation_type: AggregationType, aggregation_sub_type: String, parameters: BTreeMap, @@ -598,13 +601,48 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if let SummaryOperator::Configured { + family, + aggregation_type, + .. + } = &operator + { + if let Some(expected) = aggregation_type.planner_exact_family() { + if family != &expected { + return Err(SdsError( + "configured storage type disagrees with Planner family".into(), + )); + } + } else { + use AggregationType as A; + let expected = match aggregation_type { + A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), + A::CountMinSketch => Some(SketchAlgorithm::Cms), + A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), + A::CountSketch => Some(SketchAlgorithm::CountSketch), + A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), + A::DDSketch => Some(SketchAlgorithm::DDSketch), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let Some(expected) = expected { + if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) + { + return Err(SdsError( + "configured sketch storage disagrees with Planner family".into(), + )); + } + } + } + } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), )); } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); + let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); Ok(Self { id, operator, @@ -640,6 +678,10 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { + family: config + .accumulator_spec() + .map_err(|error| SdsError(error.to_string()))? + .family, aggregation_type: config.aggregation_type, aggregation_sub_type: config.aggregation_sub_type.clone(), parameters: config @@ -667,11 +709,8 @@ impl FidelityGuarantee { matches!( (aggregation_type, self), (A::UnivMon, UnivMonFrequency { .. }) - | ( - A::Sum | A::MultipleSum | A::Min | A::Max | A::MultipleMin | A::MultipleMax, - Exact - ) - | (A::Increase | A::MultipleIncrease, ExactCounter { .. }) + | (A::Sum | A::Count | A::Min | A::Max, Exact) + | (A::Increase | A::Rate, ExactCounter { .. }) | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) | (A::DDSketch, DdSketchRelativeError { .. }) | (A::HLL, HllCardinalityError { .. }) @@ -1444,6 +1483,7 @@ mod tests { .is_err()); assert!(SummaryDescriptor::new( SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::new(), @@ -1468,6 +1508,24 @@ mod tests { ) .is_err()); } + + #[test] + fn configured_descriptor_rejects_family_storage_disagreement() { + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + family: AggregationType::Rate.planner_exact_family().unwrap(), + aggregation_type: AggregationType::Increase, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, + 2, + ) + .is_err()); + } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); @@ -1524,11 +1582,13 @@ mod tests { #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), }; let b = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 4d1ac062..223bdf37 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -13,7 +13,7 @@ use crate::PolicyFingerprint; use crate::WindowMaterializationLayout; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 2; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 3; /// Stable materialization identity binds operator and population descriptors. /// Concrete intervals, groups and completeness belong to runtime instances. diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 01162475..11cbfa4a 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -395,13 +395,13 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; use std::collections::HashMap; - let cfg = AggregationConfig::new( + let cfg = PrecomputeMaterialization::new( AT::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 8de15bfd..3d8e0acb 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -582,7 +582,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t } /// Resolve the bucket sid (and `policy_fp`) for a single data point -/// against a single matching `AggregationConfig`. +/// against a single matching `PrecomputeMaterialization`. /// /// B7.6 — sid is the bucket identity in the precompute engine; this /// helper folds `(config, grouping-label-values)` into a single u64 via @@ -602,7 +602,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t /// their separate wire-level identity protocol. fn resolve_bucket_sid_for_agg_config( ingest_state: &Arc, - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, point_labels: &HashMap, captured_generation: Option<&asap_types::sds::CatalogGeneration>, ) -> Result<(u64, asap_types::PolicyFingerprint), String> { @@ -1783,15 +1783,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Detection is independent of the legacy dual-write // (it only drives the routed/unconfigured accounting), // so we walk it whether or not the worker push fires. - let matching_configs: Vec<&asap_types::aggregation_config::AggregationConfig> = - agg_configs - .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) - .collect(); + let matching_configs: Vec< + &asap_types::aggregation_config::PrecomputeMaterialization, + > = agg_configs + .values() + .filter(|config| { + config.metric == canonical_name + || config.spatial_filter_normalized == canonical_name + || config.spatial_filter == canonical_name + }) + .collect(); let matched_any = !matching_configs.is_empty(); // CQ-2 — only pay the worker push (and the per-config @@ -1869,7 +1870,7 @@ async fn route_modified_otlp_sketches_to_precompute( routed += 1; } else { // CQ-6 — a decoded sketch that matched no - // AggregationConfig in the running streaming config. + // PrecomputeMaterialization in the running streaming config. ingest_state .observability .dropped_unconfigured @@ -1914,7 +1915,7 @@ async fn route_modified_otlp_sketches_to_precompute( /// `AggregationType`. Inverse direction is in /// `sketch_algorithm_for` above. Used by /// [`derive_sketch_policy_fp`] to find the policy whose -/// `AggregationConfig.aggregation_type` matches a freshly-ingested +/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested /// sketch. /// /// `Any` is a control-plane analysis-time wildcard — it doesn't @@ -3466,7 +3467,7 @@ mod policy_fp_lookup_tests { fn sketch_config_to_params_uses_canonical_keys() { // The param-name vocabulary must match what the control plane // writes in streaming-config YAML (see - // `asap_types::aggregation_config::AggregationConfig::from_yaml_data`). + // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). // Drift surfaces as `find_policy_by_content` missing matches. let dd = sketch_config_to_params(&SketchConfig::DDSketch { relative_accuracy: 0.01, @@ -4564,7 +4565,7 @@ mod sid_bucketing_tests { metric::Data, number_data_point::Value as NumberValue, Gauge as PbGauge, Metric as PbMetric, NumberDataPoint, ResourceMetrics, ScopeMetrics, }; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -4581,8 +4582,8 @@ mod sid_bucketing_tests { } } - fn sum_agg_config(metric: &str, grouping: &[&str]) -> AggregationConfig { - AggregationConfig::new( + fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index b6d3bfd7..cd847bc0 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -717,11 +717,9 @@ fn route_messages( && matches!( config.aggregation_type, asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + | asap_types::AggregationType::Rate | asap_types::AggregationType::Min | asap_types::AggregationType::Max - | asap_types::AggregationType::MultipleMin - | asap_types::AggregationType::MultipleMax )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -1040,8 +1038,8 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; - let aggregation = AggregationConfig { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; + let aggregation = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -1165,10 +1163,10 @@ mod tests { #[test] fn global_topk_cms_routes_once_while_counters_remain_per_series() { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let config = - |aggregation_type, grouping: Vec, aggregated: Vec| AggregationConfig { + let config = |aggregation_type, grouping: Vec, aggregated: Vec| { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -1203,7 +1201,8 @@ mod tests { table_timestamp_column: None, partitioning: None, value_source_column: None, - }; + } + }; let cms = config( AggregationType::CountMinSketchWithHeap, vec![], diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 382ec0b4..56f60a0b 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1004,9 +1004,9 @@ fn metric_has_exact_agg_sum_sid( cap, Capability::ExactAgg( AggregationType::Sum - | AggregationType::MultipleSum + | AggregationType::Count | AggregationType::Increase - | AggregationType::MultipleIncrease + | AggregationType::Rate ) ) { return true; @@ -2847,100 +2847,18 @@ mod tests { /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and /// the GET snapshot emission. #[tokio::test] - async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; - let client = Client::new(); - - // Initial GET: empty config, 0 entries. - let initial = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .send() - .await - .expect("GET failed"); - assert!(initial.status().is_success()); - let initial_body: serde_json::Value = initial.json().await.unwrap(); - assert_eq!(initial_body["aggregation_count"], 0); - - // POST a new config with two aggregation_ids. The YAML shape - // matches what `StreamingConfig::from_yaml_data` parses — see - // `asap-common/dependencies/rs/asap_types/src/streaming_config.rs`. - let new_config_yaml = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 102 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host, region] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 120 - windowType: tumbling - spatialFilter: '' -"#; - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(new_config_yaml.to_string()) - .send() - .await - .expect("POST failed"); - let post_status = post_resp.status(); - let post_body: serde_json::Value = post_resp.json().await.unwrap(); - assert!( - post_status.is_success(), - "POST returned {post_status}: {post_body}" - ); - assert_eq!(post_body["status"], "success"); - assert_eq!(post_body["new_aggregation_count"], 2); - // PR 5: the YAML's `aggregationId` fields are silently - // dropped — `agg_ids_added` carries fingerprint u64s. - let added = post_body["agg_ids_added"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(added.len(), 2, "exactly two distinct aggs were added"); - assert!(added.iter().all(|id| *id != 0), "fingerprints are non-zero"); - - // GET again: should reflect the two new ids. - let after = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) + async fn flat_streaming_config_is_rejected_without_mutating_active_state() { + let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; + let before = handle.snapshot(); + let response = Client::new() + .post(format!("http://127.0.0.1:{port}/api/v1/streaming-config")) + .body("aggregations: [{aggregationType: Sum, metric: m}]") .send() .await - .expect("GET after swap failed"); - assert!(after.status().is_success()); - let after_body: serde_json::Value = after.json().await.unwrap(); - assert_eq!(after_body["aggregation_count"], 2); - - // The underlying StreamingConfigHandle handle (cloned into - // the server at setup) also reflects the swap — proving that - // downstream consumers that re-snapshot would see the new - // state. PR 5: the map is keyed on fingerprints, so just - // assert the entry count. - let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert!(Arc::ptr_eq(&before, &handle.snapshot())); } #[tokio::test] @@ -2983,7 +2901,7 @@ aggregations: .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), reqwest::StatusCode::GONE); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "error"); } @@ -3048,192 +2966,6 @@ aggregations: }); } - #[tokio::test] - async fn test_streaming_config_swap_drives_sid_reconcile() { - // Schema retirement final cut: the swap handler now drives a - // single sid-level reconcile (no `SchemaRegistry`). Sids that - // already exist in the catalog and whose content signature - // does not appear in the new config get force-retired; the - // response surfaces them under `sids_retired`. There is no - // `sids_added` — sids are minted lazily by the ingest path, - // not by the swap handler. - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::sketch_db::AggStatus; - - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let summary_store = Arc::new(SketchStore::new()); - // Pre-register two Active sids whose signatures match the - // first config below; only sid 1 will survive the second - // swap. - register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); - register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); - let server_port = setup_test_server_with_hot_reload_and_sketch_index( - hot_reload.clone(), - summary_store.clone(), - ) - .await; - let client = Client::new(); - - // POST a config whose signatures cover both pre-registered - // sids. Nothing should retire. - let yaml_two = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 202 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_two.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - let retired_ids = body["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert!( - retired_ids.is_empty(), - "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", - ); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Active - ); - - // Swap to a config that drops `mem_usage`. Sid 2's signature - // is now orphaned; the handler must force-retire it. - let yaml_one = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp2 = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .expect("POST failed"); - let body2: serde_json::Value = resp2.json().await.unwrap(); - let retired = body2["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(retired, vec![2u64]); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Retired - ); - } - - #[tokio::test] - async fn test_streaming_config_swap_response_shape_with_empty_catalog() { - // With no registered sids, the swap still works — it just - // produces an empty `sids_retired` array. The `agg_ids_added` - // / `agg_ids_removed` / `new_aggregation_count` fields are - // driven purely by the diff of the two configs and are - // independent of the sid catalog. - // - // PR 5: the YAML's `aggregationId: 42` is silently dropped at - // parse time — the backend identity is content-addressed via - // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 - // in the HTTP response is the fingerprint's `as_u64()` form, - // NOT the literal `42` the YAML once spelled out. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; - let client = Client::new(); - - let yaml = r#" -aggregations: - - aggregationType: Sum - aggregationSubType: '' - metric: m - labels: - grouping: [] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - assert_eq!(body["new_aggregation_count"], 1); - let added = body["agg_ids_added"].as_array().expect("array"); - assert_eq!(added.len(), 1, "exactly one agg was added"); - assert_ne!( - added[0].as_u64().unwrap(), - 0, - "agg id is not the 0 sentinel" - ); - assert_eq!(body["agg_ids_removed"], serde_json::json!([])); - // No pre-registered sids → nothing to retire. - assert_eq!(body["sids_retired"].as_array().unwrap().len(), 0); - } - #[tokio::test] async fn test_get_schemas_returns_active_and_retired_sids_with_status_filter() { // Schema retirement final cut: `/api/v1/db/schemas` now @@ -3253,29 +2985,12 @@ aggregations: .await; let client = Client::new(); - // Retire sid 2 by pushing a config covering only `m1`. - let yaml_one = r#" -aggregations: - - aggregationId: 1 - aggregationType: Sum - aggregationSubType: '' - metric: m1 - labels: { grouping: [], rollup: [], aggregated: [] } - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .unwrap(); - assert!(resp.status().is_success()); + assert!(summary_store + .force_retire( + 2, + crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION + ) + .is_some()); // GET /api/v1/db/schemas (no filter = all). let resp = client @@ -3512,7 +3227,7 @@ aggregations: registry: Arc, active_agg_ids: &[u64], ) -> (u16, std::collections::HashMap) { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -3534,7 +3249,7 @@ aggregations: let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -5617,96 +5332,11 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res (StatusCode::OK, axum::Json(body)).into_response() } -async fn handle_post_streaming_config( - State(state): State, - body: axum::body::Bytes, -) -> axum::response::Response { - use axum::http::StatusCode; +async fn handle_post_streaming_config() -> axum::response::Response { use axum::response::IntoResponse; - use std::collections::HashSet; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - - let yaml_text = match std::str::from_utf8(&body) { - Ok(s) => s, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("request body is not valid UTF-8: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let yaml_value: serde_yaml::Value = match serde_yaml::from_str(yaml_text) { - Ok(v) => v, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("YAML parse error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let new_config = - match crate::storage_engines::types::StreamingConfig::from_yaml_data(&yaml_value) { - Ok(c) => c, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("StreamingConfig build error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - - let new_ids: HashSet = new_config - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let added: Vec = new_ids.difference(&old_ids).copied().collect(); - let removed: Vec = old_ids.difference(&new_ids).copied().collect(); - - if !removed.is_empty() { - warn!( - "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ - precompute worker groups for these ids will continue with their \ - construction-time config until they close naturally (phase 1 \ - limitation; see StreamingConfigHandle module doc)", - removed - ); - } - - // Schema retirement final cut: the sid catalog is the only - // lifecycle registry. The legacy per-`agg_id` `SchemaRegistry` is - // gone, so the swap handler now drives a single sid-level - // reconcile (`reconcile_from_streaming_config`) which force-retires - // any sid whose content signature no longer appears in the new - // config. There is no "added" set: sids are minted lazily at the - // first ingest write under the new config (see - // `SketchStore::ingest_precompute_for_agg_config`). - let snap = handle.snapshot(); - let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.summary_store.as_ref(), - snap.as_ref(), - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - - let body = serde_json::json!({ - "status": "success", - "agg_ids_added": added, - "agg_ids_removed": removed, - "new_aggregation_count": new_ids.len(), - "sids_retired": sid_summary.retired}); - (StatusCode::OK, axum::Json(body)).into_response() + (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ + "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" + }))).into_response() } pub use asap_types::plan_publication::PhysicalPlanInstallRequest; @@ -5772,7 +5402,7 @@ pub fn validate_and_build_runtime_plan( } } } - let runtime_materializations = request + let _runtime_materializations = request .precompute_plan .runtime_materializations() .map_err(|error| format!("PrecomputePlan validation error: {error}"))?; @@ -5787,8 +5417,10 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = - crate::storage_engines::types::StreamingConfig::new(runtime_materializations); + let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; let typed_fps: BTreeSet<_> = streaming_config .materializations_by_policy_fingerprint .keys() diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2721bc68..8c3ac483 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -37,13 +37,13 @@ pub mod utils; // Re-export commonly used types to avoid glob import conflicts pub use storage_engines::types::{ - AggregateCore, AggregationConfig, KeyByLabelValues, Measurement, MergeableAccumulator, - MultipleSubpopulationAggregate, PrecomputedOutput, SerializableToSink, - SingleSubpopulationAggregate, + AggregateCore, KeyByLabelValues, Measurement, MergeableAccumulator, + MultipleSubpopulationAggregate, PrecomputeMaterialization, PrecomputedOutput, + SerializableToSink, SingleSubpopulationAggregate, }; pub use precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; pub use storage_engines::StoreResult; diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 43a779fc..9f94c8c9 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,30 +1,19 @@ use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, - HydraKllSketchAccumulator, IncreaseAccumulator, MaxAccumulator, MinAccumulator, - MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, - MultipleSumAccumulator, SumAccumulator, + HydraKllSketchAccumulator, IncreaseAccumulator, KeyedCounterState, KeyedMaxState, + KeyedMinState, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::{ AggregateCore, AggregationType, KeyByLabelValues, Measurement, }; -use asap_types::aggregation_config::AggregationConfig; -// Step 5 (sketch-identity unification, see -// scratchpad/artifacts/enum-unification-plan.md): dispatch below is -// driven by `AccumulatorSpec` (SummaryFamilyType + typed family parameters + -// keyed-axis grouping) instead of raw `AggregationType` + -// `aggregation_sub_type` string matching. Numeric params come straight -// off the committed family's typed params (no HashMap lookups) except -// `cms_params`, kept as a raw-`parameters` read for the one case Planner's -// family parameters have no field for: HydraKLL's `(row, col)` tiling grid (see -// `asap_types::accumulator_spec`'s module doc for why). `cms_params` -// now lives there — the only place that still needs the other three -// former local helpers (`kll_k_param`, `heap_size_param`, -// `ddsketch_alpha_param`) is that module's own `AccumulatorSpec` -// construction, so they aren't re-imported here. +use asap_types::aggregation_config::PrecomputeMaterialization; +// Production dispatch consumes Planner SummaryAgg payloads directly. The +// config adapter below is compiled only for isolated historical kernel tests. use super::operators::hll_sketch_accumulator::HllSketchAccumulator; use super::operators::univmon_accumulator::UnivMonAccumulator; -use asap_types::accumulator_spec::{cms_params, AccumulatorSpecError}; +#[cfg(test)] +use asap_types::accumulator_spec::cms_params; use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; /// Generate the two boilerplate clone-based `AccumulatorUpdater` methods @@ -379,28 +368,32 @@ impl AccumulatorUpdater for DDSketchAccumulatorUpdater { } // --------------------------------------------------------------------------- -// MultipleSumAccumulatorUpdater +// KeyedSumCountAccumulatorUpdater // --------------------------------------------------------------------------- -pub struct MultipleSumAccumulatorUpdater { - acc: MultipleSumAccumulator, +pub struct KeyedSumCountAccumulatorUpdater { + acc: KeyedSumCountAccumulator, } -impl MultipleSumAccumulatorUpdater { +impl KeyedSumCountAccumulatorUpdater { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { Self { - acc: MultipleSumAccumulator::new(), + acc: KeyedSumCountAccumulator::for_family(family), } } } -impl Default for MultipleSumAccumulatorUpdater { +impl Default for KeyedSumCountAccumulatorUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { +impl AccumulatorUpdater for KeyedSumCountAccumulatorUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -415,7 +408,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleSumAccumulator::new(); + self.acc = KeyedSumCountAccumulator::for_family(self.acc.family.clone()); } fn is_keyed(&self) -> bool { @@ -423,13 +416,13 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() - + self.acc.sums.len() * (std::mem::size_of::() + 8) + std::mem::size_of::() + + self.acc.sums.len() * (std::mem::size_of::() + 16) } } // --------------------------------------------------------------------------- -// MultipleMinAccumulatorUpdater / MultipleMaxAccumulatorUpdater +// KeyedMinStateUpdater / KeyedMaxStateUpdater // --------------------------------------------------------------------------- macro_rules! multiple_extremum_updater { @@ -475,32 +468,32 @@ macro_rules! multiple_extremum_updater { }; } -multiple_extremum_updater!(MultipleMinAccumulatorUpdater, MultipleMinAccumulator); -multiple_extremum_updater!(MultipleMaxAccumulatorUpdater, MultipleMaxAccumulator); +multiple_extremum_updater!(KeyedMinStateUpdater, KeyedMinState); +multiple_extremum_updater!(KeyedMaxStateUpdater, KeyedMaxState); // --------------------------------------------------------------------------- -// MultipleIncreaseAccumulatorUpdater +// KeyedCounterStateUpdater // --------------------------------------------------------------------------- -pub struct MultipleIncreaseAccumulatorUpdater { - acc: MultipleIncreaseAccumulator, +pub struct KeyedCounterStateUpdater { + acc: KeyedCounterState, } -impl MultipleIncreaseAccumulatorUpdater { +impl KeyedCounterStateUpdater { pub fn new() -> Self { Self { - acc: MultipleIncreaseAccumulator::new(), + acc: KeyedCounterState::new(), } } } -impl Default for MultipleIncreaseAccumulatorUpdater { +impl Default for KeyedCounterStateUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { +impl AccumulatorUpdater for KeyedCounterStateUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -528,7 +521,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleIncreaseAccumulator::new(); + self.acc = KeyedCounterState::new(); } fn is_keyed(&self) -> bool { @@ -536,7 +529,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() + std::mem::size_of::() + self.acc.increases.len() * (std::mem::size_of::() + std::mem::size_of::()) @@ -899,27 +892,20 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &AggregationConfig) -> bool { - matches!( - config.aggregation_type, - AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax - | AggregationType::CountMinSketch - | AggregationType::CountMinSketchWithHeap - | AggregationType::CountSketch - | AggregationType::CountSketchWithHeap - | AggregationType::HydraKLL - ) +pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { + config + .accumulator_spec() + .expect("valid fixture") + .grouping + .is_some() } /// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. /// /// * `value` / `sum`: sum values per key (default). /// * `count` / `frequency` / `freq`: count occurrences per key. -fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { +#[cfg(test)] +fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { match config.sample_update_rule() { asap_types::SampleUpdateRule::Count => TopkWeight::Count, asap_types::SampleUpdateRule::Value { .. } @@ -927,7 +913,8 @@ fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { } } -fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { +#[cfg(test)] +fn topk_weight_scale_param(config: &PrecomputeMaterialization) -> f64 { match config.sample_update_rule() { asap_types::SampleUpdateRule::Value { scale } => scale, asap_types::SampleUpdateRule::CounterDelta { scale } => scale, @@ -943,6 +930,7 @@ fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { /// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with /// `SketchParams::Kll`, so the /// other arm is unreachable from a `spec` this module builds itself. +#[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { // Lossless: `accumulator_spec()` only ever stores a value that @@ -955,12 +943,12 @@ fn kll_k(params: &SketchParams) -> u16 { } } -/// Read `(width, depth)` out of `SketchParams::Cms` or `::CountSketch` +/// Read `(rows = depth, columns = width)` out of `SketchParams::Cms` or `::CountSketch` /// — same shape, different variant per bare-sketch identity. fn cms_dims(params: &SketchParams) -> (usize, usize) { match params { SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { - (*width as usize, *depth as usize) + (*depth as usize, *width as usize) } other => unreachable!( "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" @@ -968,7 +956,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { } } -/// Read `(width, depth, heap_size)` out of `SketchParams::CmsWithHeap` +/// Read `(rows = depth, columns = width, heap_size)` out of `SketchParams::CmsWithHeap` /// or `::CountSketchWithHeap`. fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { match params { @@ -981,7 +969,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { width, depth, heap_size, - } => (*width as usize, *depth as usize, *heap_size as usize), + } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), @@ -989,6 +977,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { } /// Read the DDSketch relative-accuracy `alpha` out of `SketchParams::DDSketch`. +#[cfg(test)] fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, @@ -998,54 +987,28 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { } } -/// Create an appropriate `AccumulatorUpdater` from an `AggregationConfig`. -/// -/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryFamilyType` identity -/// plus the keyed/unkeyed `grouping` axis — instead of the pre-Step-5 -/// `AggregationType` + `aggregation_sub_type` string combo. See -/// `asap_types::accumulator_spec`'s module doc for why min/max direction, -/// HydraKLL's `(row, col)` tiling, and top-k `weight_mode` still read -/// `config` directly rather than going through Planner family parameters — -/// none of those three have a field in the Planner-owned types. -pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let spec = match config.accumulator_spec() { - Ok(spec) => spec, - // Three fallback paths, preserved verbatim from the pre-Step-5 - // dispatch: same warning text, same default updater per case - // (Single- and MultipleSubpopulation default to *different* - // updaters — see `AccumulatorSpecError`'s doc). - Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown SingleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(SumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown MultipleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(MultipleSumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnmappedAggregationType(other)) => { - tracing::warn!( - "Unknown aggregation_type '{:?}', defaulting to SingleSubpopulation Sum", - other - ); - return Box::new(SumAccumulatorUpdater::new()); - } - }; +/// Construct isolated payload fixtures for kernel/storage unit tests. +/// Production execution requires a validated Planner DAG program. +#[cfg(test)] +pub fn create_fixture_accumulator( + config: &PrecomputeMaterialization, +) -> Box { + let spec = config + .accumulator_spec() + .expect("invalid isolated kernel fixture"); let keyed = spec.grouping.is_some(); match (&spec.family, keyed) { - (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), false) => { Box::new(SumAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { - Box::new(MultipleSumAccumulatorUpdater::new()) + Box::new(KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Sum)) } + (SummaryFamilyType::ExactAggregate(ExactKind::Count, _), true) => Box::new( + KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Count), + ), // Direction comes off the family itself now. It used to be read // back out of `aggregation_sub_type` because Planner had one @@ -1056,20 +1019,20 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - Box::new(MultipleMinAccumulatorUpdater::new()) + Box::new(KeyedMinStateUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), false) => { Box::new(MaxAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), true) => { - Box::new(MultipleMaxAccumulatorUpdater::new()) + Box::new(KeyedMaxStateUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), false) => { Box::new(IncreaseAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { - Box::new(MultipleIncreaseAccumulatorUpdater::new()) + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), true) => { + Box::new(KeyedCounterStateUpdater::new()) } (SummaryFamilyType::Sketch(kind, _), false) @@ -1187,14 +1150,8 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - tracing::warn!( - "SummaryFamilyType {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", - other_family, - keyed - ); - Box::new(SumAccumulatorUpdater::new()) + panic!("unsupported isolated kernel fixture {other_family:?}, keyed={keyed}") } } } @@ -1271,7 +1228,7 @@ mod tests { #[test] fn hll_and_univmon_raw_updates_share_value_identity() { for family in [AggregationType::HLL, AggregationType::UnivMon] { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( family, String::new(), Default::default(), @@ -1288,7 +1245,7 @@ mod tests { None, None, ); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); for value in [0.0, -0.0, 2.0, 2.0, f64::NAN] { updater.update_single(value, 1000); } @@ -1362,7 +1319,7 @@ mod tests { #[test] fn test_multiple_sum_updater() { - let mut updater = MultipleSumAccumulatorUpdater::new(); + let mut updater = KeyedSumCountAccumulatorUpdater::new(); assert!(updater.is_keyed()); let key_a = KeyByLabelValues::new_with_labels(vec!["a".to_string()]); @@ -1372,7 +1329,7 @@ mod tests { updater.update_keyed(&key_b, 2.0, 2000); let acc = updater.take_accumulator(); - assert_eq!(acc.type_name(), "MultipleSumAccumulator"); + assert_eq!(acc.type_name(), "KeyedSumCountAccumulator"); } #[test] @@ -1424,7 +1381,7 @@ mod tests { use std::collections::HashMap; let make_config = |agg_type: AggregationType, sub_type: &str| { - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), HashMap::new(), @@ -1463,18 +1420,15 @@ mod tests { AggregationType::MultipleSubpopulation, "Sum" ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleSum, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleIncrease, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleMax, - "" - ))); + let mut keyed = make_config(AggregationType::Sum, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Increase, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Max, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); assert!(config_is_keyed(&make_config( AggregationType::CountMinSketch, "" @@ -1497,12 +1451,12 @@ mod tests { for (agg_type, sub_type) in &[ (AggregationType::SingleSubpopulation, "Sum"), (AggregationType::MultipleSubpopulation, "Sum"), - (AggregationType::MultipleSum, ""), + (AggregationType::Sum, ""), (AggregationType::DatasketchesKLL, ""), (AggregationType::CountMinSketch, ""), ] { let config = make_config(*agg_type, sub_type); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); assert_eq!( config_is_keyed(&config), updater.is_keyed(), @@ -1518,7 +1472,7 @@ mod tests { use std::collections::HashMap; let mut params = HashMap::new(); params.insert("K".to_string(), serde_json::Value::from(50_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "DatasketchesKLL".to_string(), params, @@ -1535,7 +1489,7 @@ mod tests { None, None, ); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); let acc = updater.snapshot_accumulator(); let kll = acc .as_any() @@ -1555,7 +1509,7 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), serde_json::Value::from(7_u64)); params.insert("w".to_string(), serde_json::Value::from(2048_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), params, @@ -1575,7 +1529,7 @@ mod tests { assert_eq!(super::cms_params(&config), (7, 2048)); // Empty params — defaults `(4, 1000)`. - let empty_config = AggregationConfig::new( + let empty_config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), HashMap::new(), @@ -1601,7 +1555,10 @@ mod tests { /// Build a `*WithHeap` config keyed by group-by label `host`, with the /// given `weight_mode` param (None → default = value-weighted). - fn topk_config(agg_type: AggregationType, weight_mode: Option<&str>) -> AggregationConfig { + fn topk_config( + agg_type: AggregationType, + weight_mode: Option<&str>, + ) -> PrecomputeMaterialization { use std::collections::HashMap; let mut params = HashMap::new(); // Small, deterministic geometry; heap big enough to hold all hosts. @@ -1611,7 +1568,7 @@ mod tests { if let Some(m) = weight_mode { params.insert("weight_mode".to_string(), serde_json::Value::from(m)); } - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -1699,7 +1656,7 @@ mod tests { fn value_weighted_topk_ranks_hosts_by_sum_of_value() { // DEFAULT mode (no weight_mode param) must be value-weighted. let config = topk_config(AggregationType::CountMinSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); assert!(updater.is_keyed()); feed_stream(&mut *updater); @@ -1725,14 +1682,24 @@ mod tests { #[test] fn counter_delta_scale_preserves_sub_unit_membership_weights() { - let mut config = topk_config( - AggregationType::CountMinSketchWithHeap, - Some("counter_delta"), - ); - config - .parameters - .insert("weight_scale".into(), serde_json::json!(1_000_000)); - let mut updater = create_accumulator_updater(&config); + use planner_types::post_asap::{ + EntityIdentity, NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, + }; + let config = topk_config(AggregationType::CountMinSketchWithHeap, None); + let family = config.accumulator_spec().unwrap().family; + let input = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::ResetAwareCounterDelta { + value: planner_types::pre_asap::ColumnRef::SampleValue, + series: EntityIdentity::PromqlLabelSet { excluding: vec![] }, + }, + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::ResetAwareCounterDerivative, + }, + }; + let mut updater = create_planner_accumulator(&family, &input, &Default::default()).unwrap(); updater.update_keyed(&host_key("payment"), 0.004, 1_000); updater.update_keyed(&host_key("order"), 0.002, 1_000); let ranked = ranked_topk(&*updater.take_accumulator()); @@ -1744,7 +1711,7 @@ mod tests { fn count_weighted_topk_still_ranks_by_occurrence_frequency() { // Opt-in frequency-top-k: weight_mode=count must rank by event count. let config = topk_config(AggregationType::CountMinSketchWithHeap, Some("count")); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); @@ -1764,7 +1731,7 @@ mod tests { // (real median-of-signed-rows math) — same value-weighted default // as the CMS-family heap path, but no longer conflated with it. let config = topk_config(AggregationType::CountSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); assert_eq!(acc.type_name(), "CountSketchWithHeapAccumulator"); @@ -1800,3 +1767,278 @@ mod tests { } } } + +#[cfg(test)] +mod planner_family_regression { + use super::*; + use asap_types::{enums::WindowKind, KeyByLabelNames}; + + // Every installed exact producer must retain its family in runtime state. + #[test] + fn exact_state_identity_survives_factory_and_reset() { + for kind in [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Rate, + AggregationType::Increase, + AggregationType::Min, + AggregationType::Max, + ] { + let config = PrecomputeMaterialization::new( + kind, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "metric".into(), + None, + None, + None, + ); + let mut updater = create_planner_accumulator( + &config.accumulator_spec().unwrap().family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &Default::default(), + ) + .unwrap(); + updater.update_single(4.0, 1000); + updater.update_single(7.0, 2000); + assert_eq!(updater.take_accumulator().get_accumulator_type(), kind); + assert_eq!(updater.snapshot_accumulator().get_accumulator_type(), kind); + } + } +} + +/// Construct the kernel declared by a Planner SummaryAgg. No backend config +/// tags participate in this dispatch and unsupported payloads are errors. +pub fn create_planner_accumulator( + family: &SummaryFamilyType, + input: &planner_types::post_asap::SummaryUpdate, + grouping: &planner_types::post_asap::GroupingStrategy, +) -> Result, String> { + use planner_types::post_asap::GroupingStrategy; + if grouping != &GroupingStrategy::PerSubpopulationInstance { + return Err("shared summary grouping requires a supported Planner Hydra kernel".into()); + } + if matches!(family, SummaryFamilyType::ExactAggregate(..)) { + return Ok(Box::new(PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator::new( + family.clone(), + input.item.is_some(), + )?, + })); + } + let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + return Err(format!("unsupported Planner summary family {family:?}")); + }; + if family_grouping != grouping { + return Err("Planner family and operator grouping disagree".into()); + } + // Heap counters use fixed-point storage for fractional counter deltas. + // This encodes the selected update; it does not choose another family. + let weight_scale = if matches!( + input.weight, + planner_types::post_asap::SummaryInputExpr::ResetAwareCounterDelta { .. } + ) { + 1_000_000.0 + } else { + 1.0 + }; + let updater: Box = match (kind.algorithm(), kind.params()) { + (SketchAlgorithm::Kll, SketchParams::Kll { k }) => Box::new(KllAccumulatorUpdater::new( + u16::try_from(*k).map_err(|_| "KLL k exceeds runtime bound")?, + )), + (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Box::new(DDSketchAccumulatorUpdater::new(*alpha)) + } + (SketchAlgorithm::Cms, params @ SketchParams::Cms { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CmsAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CountSketch, params @ SketchParams::CountSketch { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CountSketchAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CmsWithHeap, params @ SketchParams::CmsWithHeap { .. }) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CmsHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + ( + SketchAlgorithm::CountSketchWithHeap, + params @ SketchParams::CountSketchWithHeap { .. }, + ) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CountSketchWithHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => Box::new(HllUpdater { + acc: HllSketchAccumulator::new( + asap_sketchlib::HllVariant::Regular, + u32::from(*precision), + ), + }), + ( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + }, + ) => Box::new(UnivMonUpdater { + acc: UnivMonAccumulator::new( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + *layers as usize, + ) + .map_err(|e| e.to_string())?, + }), + _ => { + return Err(format!( + "unsupported Planner algorithm/parameters: {kind:?}" + )) + } + }; + if updater.is_keyed() != input.item.is_some() + && !asap_types::accumulator_spec::is_unit_sample_frequency(input) + { + return Err("Planner item expression does not match the selected kernel layout".into()); + } + Ok(updater) +} + +struct PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator, +} +impl AccumulatorUpdater for PlannerExactUpdater { + fn update_single(&mut self, value: f64, timestamp: i64) { + self.acc.update(None, value, timestamp); + } + fn update_keyed(&mut self, key: &KeyByLabelValues, value: f64, timestamp: i64) { + self.acc.update(Some(key), value, timestamp); + } + impl_clone_accumulator_methods!(acc); + fn reset(&mut self) { + self.acc = super::operators::exact_accumulator::ExactAccumulator::new( + self.acc.family().clone(), + self.acc.is_keyed(), + ) + .expect("installed exact family"); + } + fn is_keyed(&self) -> bool { + self.acc.is_keyed() + } + fn memory_usage_bytes(&self) -> usize { + self.acc.approx_memory_bytes() + } +} + +#[cfg(test)] +mod planner_parameter_regression { + use super::*; + use planner_types::post_asap::{SketchKind, SummaryInputExpr, SummaryUpdate}; + + // Planner width is the bucket count; depth is the independent hash-row count. + #[test] + fn planner_sketch_dimensions_are_not_transposed() { + for (algorithm, params) in [ + ( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ] { + let family = SummaryFamilyType::Sketch( + SketchKind::new(algorithm.clone(), params), + Default::default(), + ); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + }; + let state = create_planner_accumulator(&family, &update, &Default::default()) + .unwrap() + .snapshot_accumulator(); + let dims = match algorithm { + SketchAlgorithm::Cms => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketch => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows, s.inner.cols) + } + SketchAlgorithm::CmsWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketchWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + _ => unreachable!(), + }; + assert_eq!(dims, (3, 128), "{algorithm:?}"); + } + } +} diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 47593276..9e499adc 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -56,7 +56,7 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::AggregationConfig, + config: &asap_types::PrecomputeMaterialization, timestamp_ms: i64, value: f64, ) { @@ -264,8 +264,8 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { - let config = asap_types::AggregationConfig::new( + fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { + let config = asap_types::PrecomputeMaterialization::new( asap_types::AggregationType::HLL, String::new(), Default::default(), diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index d975eb6a..67940478 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,7 +1,7 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::StreamingConfigHandle; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,11 +28,11 @@ pub struct IngestObservability { /// A full / delta frame failed to decode (or a delta failed to /// apply) and was dropped. pub dropped_decode_fail: AtomicU64, - /// A decoded sketch matched no `AggregationConfig` in the running + /// A decoded sketch matched no `PrecomputeMaterialization` in the running /// streaming config (legacy routing-side bucketing miss). pub dropped_unconfigured: AtomicU64, /// The output sink could not resolve a `policy_fp` to an - /// `AggregationConfig` (registry miss) and skipped the write. + /// `PrecomputeMaterialization` (registry miss) and skipped the write. pub dropped_policy_miss: AtomicU64, /// RES-1 — max number of distinct tumbling windows a per-series /// snapshot base may lag behind the newest observed `window_start` @@ -120,7 +120,7 @@ pub struct IngestState { pub samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64, /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. - /// This replaces the old frozen `Vec>`. + /// This replaces the old frozen `Vec>`. pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, @@ -157,7 +157,7 @@ impl IngestState { /// visible immediately without restart. /// /// Returns the shared `Arc` — no cloning of - /// individual AggregationConfig objects, just an atomic refcount + /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() @@ -247,7 +247,7 @@ impl IngestState { /// ingest sources (e.g. OTLP) can reuse it. pub fn extract_group_key_for( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { extract_group_key(series_key, config) } @@ -261,7 +261,7 @@ impl IngestState { /// [`Self::extract_group_key_for`] does after the round-trip. pub fn extract_group_key_from_labels( labels: &std::collections::HashMap, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map( |name| { @@ -278,7 +278,7 @@ impl IngestState { /// for a given series key and aggregation config. fn extract_group_key( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { let labels = parse_labels_from_series_key(series_key); crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map(|name| { @@ -294,18 +294,18 @@ mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Kept as a parameter to // avoid churning the call sites below. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), std::collections::HashMap::new(), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index e8f416e5..450084f4 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -122,7 +122,7 @@ fn frozen_population_value( struct OperatorAdapter<'a> { binding: &'a BackendExecutableBinding, inputs: MaintenanceInputs<'a>, - configs: &'a [asap_types::aggregation_config::AggregationConfig], + configs: &'a [asap_types::aggregation_config::PrecomputeMaterialization], } impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { @@ -193,7 +193,12 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } finalize_exact(node, inputs) } - ExecutableOperatorPayload::SummaryAgg { family, input, .. } => { + ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } => { let [value] = inputs else { return Err("maintenance SummaryAgg requires exactly one row input".into()); }; @@ -251,7 +256,9 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = super::accumulator_factory::create_accumulator_updater(config); + let mut updater = super::accumulator_factory::create_planner_accumulator( + family, input, grouping, + )?; if updater.is_keyed() { return Err("keyed maintenance accumulator requires an item expression".into()); } diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 068ac13b..3f744870 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod metrics; pub mod multisource_coordinator; pub mod operators; pub mod output_sink; +pub mod raw_dag; pub mod series_buffer; pub mod series_router; pub mod subdag_scheduler; diff --git a/data_plane/src/precompute_engine/operators/exact_accumulator.rs b/data_plane/src/precompute_engine/operators/exact_accumulator.rs new file mode 100644 index 00000000..b7fcead1 --- /dev/null +++ b/data_plane/src/precompute_engine/operators/exact_accumulator.rs @@ -0,0 +1,327 @@ +//! Exact summary state identified by Planner family, independent of keyed layout. +use super::increase_accumulator::IncreaseAccumulator; +use crate::storage_engines::types::{ + AggregateCore, AggregationType, AuxStats, KeyByLabelValues, Measurement, SerializableToSink, +}; +use asap_types::Statistic; +use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +type Error = Box; + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum ScalarState { + Sum(f64), + Count(u64), + Min(Option), + Max(Option), + Counter(Option), +} + +/// Both the family and population layout survive persistence. Sharing counter +/// arithmetic never authorizes a Rate state to answer an Increase readout. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExactAccumulator { + family: SummaryFamilyType, + scalar: ScalarState, + keyed: Option>, +} + +impl ExactAccumulator { + pub fn new(family: SummaryFamilyType, keyed: bool) -> Result { + use ExactKind as K; + use ExactParams as P; + let scalar = match &family { + SummaryFamilyType::ExactAggregate(K::Sum, P::Sum) => ScalarState::Sum(0.0), + SummaryFamilyType::ExactAggregate(K::Count, P::Count) => ScalarState::Count(0), + SummaryFamilyType::ExactAggregate(K::Min, P::Min) => ScalarState::Min(None), + SummaryFamilyType::ExactAggregate(K::Max, P::Max) => ScalarState::Max(None), + SummaryFamilyType::ExactAggregate(K::Rate, P::Rate) + | SummaryFamilyType::ExactAggregate(K::Increase, P::Increase) => { + ScalarState::Counter(None) + } + _ => return Err(format!("unsupported exact Planner family: {family:?}")), + }; + Ok(Self { + family, + scalar, + keyed: keyed.then(HashMap::new), + }) + } + + pub fn family(&self) -> &SummaryFamilyType { + &self.family + } + pub fn is_keyed(&self) -> bool { + self.keyed.is_some() + } + + pub fn update(&mut self, key: Option<&KeyByLabelValues>, value: f64, timestamp: i64) { + let state = match (&mut self.keyed, key) { + (Some(states), Some(key)) => states + .entry(key.clone()) + .or_insert_with(|| self.scalar.clone()), + (None, None) => &mut self.scalar, + _ => panic!("exact update population layout differs from installed DAG"), + }; + match state { + ScalarState::Sum(sum) => *sum += value, + ScalarState::Count(count) => { + *count = count.checked_add(1).expect("exact count overflow") + } + ScalarState::Min(current) => { + *current = Some(current.map_or(value, |old| old.min(value))) + } + ScalarState::Max(current) => { + *current = Some(current.map_or(value, |old| old.max(value))) + } + ScalarState::Counter(current) => match current { + Some(counter) => counter.update(Measurement::new(value), timestamp), + None => { + *current = Some(IncreaseAccumulator::new( + Measurement::new(value), + timestamp, + Measurement::new(value), + timestamp, + )) + } + }, + } + } + + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let state: Self = rmp_serde::from_slice(bytes)?; + let expected = Self::new(state.family.clone(), state.is_keyed())?; + let same_variant = |value: &ScalarState| { + std::mem::discriminant(value) == std::mem::discriminant(&expected.scalar) + }; + if !same_variant(&state.scalar) + || state + .keyed + .as_ref() + .is_some_and(|states| states.values().any(|s| !same_variant(s))) + { + return Err("exact payload differs from declared Planner family".into()); + } + Ok(state) + } + + fn statistic(&self) -> Statistic { + match self.family { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Min, _) => Statistic::Min, + SummaryFamilyType::ExactAggregate(ExactKind::Max, _) => Statistic::Max, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => Statistic::Rate, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => Statistic::Increase, + _ => unreachable!("validated exact family"), + } + } +} + +fn merge_scalar(left: &ScalarState, right: &ScalarState) -> Result { + Ok(match (left, right) { + (ScalarState::Sum(a), ScalarState::Sum(b)) => ScalarState::Sum(a + b), + (ScalarState::Count(a), ScalarState::Count(b)) => { + ScalarState::Count(a.checked_add(*b).ok_or("exact count overflow")?) + } + (ScalarState::Min(a), ScalarState::Min(b)) => { + ScalarState::Min(a.iter().chain(b).copied().reduce(f64::min)) + } + (ScalarState::Max(a), ScalarState::Max(b)) => { + ScalarState::Max(a.iter().chain(b).copied().reduce(f64::max)) + } + (ScalarState::Counter(a), ScalarState::Counter(b)) => { + ScalarState::Counter(match (a, b) { + (Some(a), Some(b)) => Some( + >::merge_accumulators(vec![a.clone(), b.clone()])?, + ), + (a, b) => a.clone().or_else(|| b.clone()), + }) + } + _ => return Err("exact scalar state families differ".into()), + }) +} + +impl SerializableToSink for ExactAccumulator { + fn serialize_to_json(&self) -> serde_json::Value { + serde_json::json!({"family": self.family, "scalar": self.scalar, "keyed": self.keyed.as_ref().map(|m|m.iter().collect::>())}) + } + fn serialize_to_bytes(&self) -> Vec { + rmp_serde::to_vec_named(self).expect("exact state encoding") + } +} + +impl AggregateCore for ExactAccumulator { + fn clone_boxed_core(&self) -> Box { + Box::new(self.clone()) + } + fn type_name(&self) -> &'static str { + "PlannerExactAccumulatorV1" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn merge_with(&self, other: &dyn AggregateCore) -> Result, Error> { + let other = other + .as_any() + .downcast_ref::() + .ok_or("merge requires Planner exact state")?; + if self.family != other.family || self.is_keyed() != other.is_keyed() { + return Err("cannot merge different Planner families or layouts".into()); + } + let mut merged = self.clone(); + if let (Some(target), Some(source)) = (&mut merged.keyed, &other.keyed) { + for (key, state) in source { + let combined = match target.get(key) { + Some(old) => merge_scalar(old, state)?, + None => state.clone(), + }; + target.insert(key.clone(), combined); + } + } else { + merged.scalar = merge_scalar(&self.scalar, &other.scalar)?; + } + Ok(Box::new(merged)) + } + fn get_accumulator_type(&self) -> AggregationType { + match self.statistic() { + Statistic::Sum => AggregationType::Sum, + Statistic::Count => AggregationType::Count, + Statistic::Min => AggregationType::Min, + Statistic::Max => AggregationType::Max, + Statistic::Rate => AggregationType::Rate, + Statistic::Increase => AggregationType::Increase, + _ => unreachable!(), + } + } + fn approx_memory_bytes(&self) -> usize { + std::mem::size_of::() + + self.keyed.as_ref().map_or(0, |m| { + m.keys() + .map(|k| { + std::mem::size_of::() + + k.labels.iter().map(String::len).sum::() + }) + .sum::() + }) + } + fn aux_stats(&self) -> AuxStats { + if self.is_keyed() { + return AuxStats::empty(); + } + match self.scalar { + ScalarState::Sum(value) => AuxStats { + sum: Some(value), + ..AuxStats::empty() + }, + ScalarState::Count(value) => AuxStats { + count: Some(value), + ..AuxStats::empty() + }, + ScalarState::Min(value) => AuxStats { + min: value, + ..AuxStats::empty() + }, + ScalarState::Max(value) => AuxStats { + max: value, + ..AuxStats::empty() + }, + ScalarState::Counter(_) => AuxStats::empty(), + } + } + fn get_keys(&self) -> Option> { + self.keyed.as_ref().map(|m| m.keys().cloned().collect()) + } + fn query_statistic( + &self, + statistic: Statistic, + key: &Option, + kwargs: &HashMap, + ) -> Result { + if statistic != self.statistic() { + return Err("readout differs from Planner exact family".into()); + } + let state = match (&self.keyed, key) { + (Some(states), Some(key)) => states.get(key).ok_or("unknown exact population")?, + (None, None) => &self.scalar, + _ => return Err("readout population differs from installed layout".into()), + }; + match state { + ScalarState::Sum(sum) => Ok(*sum), + ScalarState::Count(count) => Ok(*count as f64), + ScalarState::Min(value) | ScalarState::Max(value) => { + value.ok_or_else(|| "empty exact population".into()) + } + ScalarState::Counter(Some(counter)) => { + counter.query_statistic(statistic, &None, kwargs) + } + ScalarState::Counter(None) => Err("empty counter population".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Identity, population isolation, and readout survive the persisted format. + #[test] + fn exact_families_roundtrip_and_reject_cross_family_operations() { + let families = [ + (ExactKind::Sum, ExactParams::Sum, Statistic::Sum, 16.0), + (ExactKind::Count, ExactParams::Count, Statistic::Count, 3.0), + (ExactKind::Min, ExactParams::Min, Statistic::Min, 2.0), + (ExactKind::Max, ExactParams::Max, Statistic::Max, 8.0), + (ExactKind::Rate, ExactParams::Rate, Statistic::Rate, 3.0), + ( + ExactKind::Increase, + ExactParams::Increase, + Statistic::Increase, + 6.0, + ), + ]; + for keyed in [false, true] { + let key = keyed.then(|| KeyByLabelValues::new_with_labels(vec!["a".into()])); + let mut states = Vec::new(); + for (kind, params, stat, value) in &families { + let mut state = ExactAccumulator::new( + SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), + keyed, + ) + .unwrap(); + for (ts, v) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(key.as_ref(), v, ts); + } + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()).unwrap(); + assert_eq!(restored.family(), state.family()); + assert_eq!( + restored + .query_statistic(*stat, &key, &HashMap::new()) + .unwrap(), + *value + ); + for (_, _, wrong, _) in &families { + if wrong != stat { + assert!(restored + .query_statistic(*wrong, &key, &HashMap::new()) + .is_err()); + } + } + states.push(restored); + } + for (i, a) in states.iter().enumerate() { + for (j, b) in states.iter().enumerate() { + assert_eq!(a.merge_with(b).is_ok(), i == j); + } + } + } + } +} diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs similarity index 86% rename from data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_counter_state.rs index c1d59aa1..b94d2fab 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs @@ -12,11 +12,11 @@ use asap_types::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys /// Allows tracking rate/increase for different label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleIncreaseAccumulator { +pub struct KeyedCounterState { pub increases: HashMap, } -impl MultipleIncreaseAccumulator { +impl KeyedCounterState { pub fn new() -> Self { Self { increases: HashMap::new(), @@ -91,13 +91,13 @@ impl MultipleIncreaseAccumulator { } } -impl Default for MultipleIncreaseAccumulator { +impl Default for KeyedCounterState { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleIncreaseAccumulator { +impl SerializableToSink for KeyedCounterState { fn serialize_to_json(&self) -> Value { let entries: Vec = self .increases @@ -135,13 +135,13 @@ impl SerializableToSink for MultipleIncreaseAccumulator { } } -impl AggregateCore for MultipleIncreaseAccumulator { +impl AggregateCore for KeyedCounterState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleIncreaseAccumulator" + "KeyedCounterState" } fn as_any(&self) -> &dyn std::any::Any { @@ -156,20 +156,20 @@ impl AggregateCore for MultipleIncreaseAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleIncreaseAccumulator + // Check if other is also a KeyedCounterState if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleIncreaseAccumulator with {}", + "Cannot merge KeyedCounterState with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleIncreaseAccumulator + // Downcast to KeyedCounterState let other_multiple_increase = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleIncreaseAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedCounterState")?; // Clone self once, then merge each matching counter with the same // reset-aware, boundary-aware implementation used by the unkeyed path. @@ -189,7 +189,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleIncrease + AggregationType::Increase } fn approx_memory_bytes(&self) -> usize { @@ -210,14 +210,12 @@ impl AggregateCore for MultipleIncreaseAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleIncreaseAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedCounterState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { +impl MultipleSubpopulationAggregate for KeyedCounterState { fn query( &self, statistic: Statistic, @@ -227,7 +225,7 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { let data = self .increases .get(key) - .ok_or_else(|| format!("Key {key} not found in MultipleIncreaseAccumulator"))?; + .ok_or_else(|| format!("Key {key} not found in KeyedCounterState"))?; data.query(statistic, query_kwargs) } @@ -237,15 +235,15 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { } } -impl MergeableAccumulator for MultipleIncreaseAccumulator { +impl MergeableAccumulator for KeyedCounterState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleIncreaseAccumulator::new(); + let mut result = KeyedCounterState::new(); for accumulator in accumulators { for (key, data) in accumulator.increases { @@ -291,14 +289,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_creation() { - let acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_creation() { + let acc = KeyedCounterState::new(); assert!(acc.increases.is_empty()); } #[test] - fn test_multiple_increase_accumulator_update() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_update() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -316,8 +314,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_query() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_query() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -344,14 +342,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_sum_per_key() { - // `sum by (zone) (counter)` reaches MultipleIncreaseAccumulator + fn test_keyed_counter_state_sum_per_key() { + // `sum by (zone) (counter)` reaches KeyedCounterState // only when the ASAP-tier ingest groups multiple series under // a single accumulator (the `Multiple*` variant). In that case // each per-key Sum should be the series' latest cumulative // value; the engine's outer `by` aggregation does the cross-key // grouping. (Issue ProjectASAP/ASAPCollector#46.) - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let east = KeyByLabelValues::new_with_labels(vec!["us-east-1".to_string()]); let west = KeyByLabelValues::new_with_labels(vec!["us-west-2".to_string()]); @@ -369,9 +367,9 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_merge() { - let mut acc1 = MultipleIncreaseAccumulator::new(); - let mut acc2 = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_merge() { + let mut acc1 = KeyedCounterState::new(); + let mut acc2 = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -387,7 +385,7 @@ mod tests { create_test_increase_accumulator_with_time(15.0, 2000, 30.0, 3000), ); // Later time range - let merged = MultipleIncreaseAccumulator::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = KeyedCounterState::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.increases.len(), 2); assert!(merged.increases.contains_key(&key1)); @@ -400,8 +398,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_serialization() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_serialization() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let second_key = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -415,7 +413,7 @@ mod tests { // Test JSON serialization let json_value = acc.serialize_to_json(); - let deserialized = MultipleIncreaseAccumulator::deserialize_from_json(&json_value).unwrap(); + let deserialized = KeyedCounterState::deserialize_from_json(&json_value).unwrap(); assert_eq!(deserialized.increases.len(), 2); let deserialized_acc = deserialized.increases.get(&key).unwrap(); @@ -425,8 +423,7 @@ mod tests { // Test binary serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = - MultipleIncreaseAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedCounterState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.increases.len(), 2); let deserialized_acc_bytes = deserialized_bytes.increases.get(&key).unwrap(); @@ -445,8 +442,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_get_keys() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_get_keys() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let key2 = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -462,7 +459,7 @@ mod tests { #[test] fn test_trait_object() { - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new(); acc.update(key.clone(), create_test_increase_accumulator(10.0, 25.0)); @@ -477,7 +474,7 @@ mod tests { } // #[test] - // fn test_multiple_increase_accumulator_arroyo_deserialization() { + // fn test_keyed_counter_state_arroyo_deserialization() { // // Create test data in Arroyo MessagePack format // // Format: {key: [starting_value, starting_timestamp, last_seen_value, last_seen_timestamp]} // let mut test_data = std::collections::HashMap::new(); @@ -489,7 +486,7 @@ mod tests { // // Test Arroyo deserialization // let deserialized_acc = - // MultipleIncreaseAccumulator::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); + // KeyedCounterState::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); // // Verify the deserialized accumulator has the correct data // assert_eq!(deserialized_acc.increases.len(), 2); diff --git a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_max_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_max_state.rs index 1865d268..4309c04a 100644 --- a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_max_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key maximum over many populations, mergeable by comparison. /// /// The minimum direction is -/// [`MultipleMinAccumulator`](super::multiple_min_accumulator::MultipleMinAccumulator), +/// [`KeyedMinState`](super::keyed_min_state::KeyedMinState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMaxAccumulator { +pub struct KeyedMaxState { pub values: HashMap, } -impl MultipleMaxAccumulator { +impl KeyedMaxState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMaxAccumulator { } } -impl SerializableToSink for MultipleMaxAccumulator { +impl SerializableToSink for KeyedMaxState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMaxAccumulator { } } -impl AggregateCore for MultipleMaxAccumulator { +impl AggregateCore for KeyedMaxState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMaxAccumulator" + "KeyedMaxState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMaxAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMaxAccumulator with {}", + "Cannot merge KeyedMaxState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMaxAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMaxAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMaxState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMaxAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMax + AggregationType::Max } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMaxAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMaxAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMaxState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { +impl MultipleSubpopulationAggregate for KeyedMaxState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMaxAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMaxAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMaxState").into()), + other => Err(format!("Unsupported statistic in KeyedMaxState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { } } -impl MergeableAccumulator for MultipleMaxAccumulator { +impl MergeableAccumulator for KeyedMaxState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMaxAccumulator::new(); + let mut result = KeyedMaxState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_largest_per_key() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Min, &key("a"), None).is_err()); assert!(acc.query(Statistic::Max, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMaxAccumulator::new(); + let mut left = KeyedMaxState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMaxAccumulator::new(); + let mut right = KeyedMaxState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Max, &key("a"), None).unwrap(), 10.0); assert_eq!(merged.query(Statistic::Max, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_min_accumulator::MultipleMinAccumulator; - let mine = MultipleMaxAccumulator::new(); - let theirs = MultipleMinAccumulator::new(); + use super::super::keyed_min_state::KeyedMinState; + let mine = KeyedMaxState::new(); + let theirs = KeyedMinState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMaxAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMaxState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMaxAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMaxState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_min_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_min_state.rs index 00c25040..5be698f5 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_min_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key minimum over many populations, mergeable by comparison. /// /// The maximum direction is -/// [`MultipleMaxAccumulator`](super::multiple_max_accumulator::MultipleMaxAccumulator), +/// [`KeyedMaxState`](super::keyed_max_state::KeyedMaxState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMinAccumulator { +pub struct KeyedMinState { pub values: HashMap, } -impl MultipleMinAccumulator { +impl KeyedMinState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMinAccumulator { } } -impl SerializableToSink for MultipleMinAccumulator { +impl SerializableToSink for KeyedMinState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMinAccumulator { } } -impl AggregateCore for MultipleMinAccumulator { +impl AggregateCore for KeyedMinState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMinAccumulator" + "KeyedMinState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMinAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMinAccumulator with {}", + "Cannot merge KeyedMinState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMinAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMinAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMinState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMinAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMin + AggregationType::Min } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMinAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMinAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMinState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMinAccumulator { +impl MultipleSubpopulationAggregate for KeyedMinState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMinAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMinAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMinState").into()), + other => Err(format!("Unsupported statistic in KeyedMinState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { } } -impl MergeableAccumulator for MultipleMinAccumulator { +impl MergeableAccumulator for KeyedMinState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMinAccumulator::new(); + let mut result = KeyedMinState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_smallest_per_key() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Max, &key("a"), None).is_err()); assert!(acc.query(Statistic::Min, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMinAccumulator::new(); + let mut left = KeyedMinState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMinAccumulator::new(); + let mut right = KeyedMinState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Min, &key("a"), None).unwrap(), 5.0); assert_eq!(merged.query(Statistic::Min, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_max_accumulator::MultipleMaxAccumulator; - let mine = MultipleMinAccumulator::new(); - let theirs = MultipleMaxAccumulator::new(); + use super::super::keyed_max_state::KeyedMaxState; + let mine = KeyedMinState::new(); + let theirs = KeyedMaxState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMinAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMinState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMinAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMinState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs similarity index 50% rename from data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs index 85a9982d..c39d5583 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs @@ -7,26 +7,53 @@ use serde_json::Value; use std::collections::HashMap; use asap_types::Statistic; +use planner_types::post_asap::ExactKind; + +fn sum_family() -> ExactKind { + ExactKind::Sum +} /// Accumulator that maintains separate sum values for multiple keys /// Allows querying sums for specific label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleSumAccumulator { +pub struct KeyedSumCountAccumulator { + #[serde(default = "sum_family")] + pub family: ExactKind, pub sums: HashMap, + #[serde(default)] + pub counts: HashMap, } -impl MultipleSumAccumulator { +impl KeyedSumCountAccumulator { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { + assert!(matches!(family, ExactKind::Sum | ExactKind::Count)); Self { + family, sums: HashMap::new(), + counts: HashMap::new(), } } pub fn update(&mut self, key: KeyByLabelValues, value: f64) { - *self.sums.entry(key).or_insert(0.0) += value; + let is_new = !self.sums.contains_key(&key); + *self.sums.entry(key.clone()).or_insert(0.0) += value; + if let Some(count) = self.counts.get(&key).copied() { + if let Some(next) = count.checked_add(1).filter(|next| *next != u64::MAX) { + self.counts.insert(key, next); + } else { + self.counts.remove(&key); + } + } else if is_new { + self.counts.insert(key, 1); + } } pub fn add_sum(&mut self, key: KeyByLabelValues, sum: f64) { + self.counts.remove(&key); self.sums.insert(key, sum); } @@ -43,7 +70,28 @@ impl MultipleSumAccumulator { sums.insert(key, sum); } - Ok(Self { sums }) + let mut counts = HashMap::new(); + if let Some(counts_data) = data.get("counts").and_then(Value::as_object) { + for (key_str, value) in counts_data { + let key_json: Value = serde_json::from_str(key_str)?; + let key = KeyByLabelValues::deserialize_from_json(&key_json)?; + let count = value.as_u64().ok_or("Invalid count value")?; + if !sums.contains_key(&key) { + return Err("Count key missing from sums".into()); + } + counts.insert(key, count); + } + } + let family = match data.get("family").and_then(Value::as_str) { + None | Some("Sum") => ExactKind::Sum, + Some("Count") => ExactKind::Count, + _ => return Err("Invalid keyed additive family".into()), + }; + Ok(Self { + family, + sums, + counts, + }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -62,6 +110,7 @@ impl MultipleSumAccumulator { offset += 4; let mut sums = HashMap::new(); + let mut keys = Vec::new(); for _ in 0..num_entries { // Read key length and data @@ -99,20 +148,50 @@ impl MultipleSumAccumulator { ]); offset += 8; + keys.push(key.clone()); sums.insert(key, sum); } - - Ok(Self { sums }) + let remaining = buffer.len() - offset; + let count_bytes = num_entries + .checked_mul(8) + .ok_or("Count section too large")?; + if remaining != 0 && remaining != count_bytes && remaining != count_bytes + 1 { + return Err("Invalid count section length".into()); + } + let mut counts = HashMap::new(); + if count_bytes != 0 && remaining >= count_bytes { + for key in keys { + let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); + offset += 8; + if count != u64::MAX { + counts.insert(key, count); + } + } + } + let family = if remaining == count_bytes + 1 { + match buffer[offset] { + 0 => ExactKind::Sum, + 1 => ExactKind::Count, + _ => return Err("Invalid keyed additive family tag".into()), + } + } else { + ExactKind::Sum + }; + Ok(Self { + family, + sums, + counts, + }) } } -impl Default for MultipleSumAccumulator { +impl Default for KeyedSumCountAccumulator { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleSumAccumulator { +impl SerializableToSink for KeyedSumCountAccumulator { fn serialize_to_json(&self) -> Value { let mut sums_obj = serde_json::Map::new(); for (key, sum) in &self.sums { @@ -124,8 +203,16 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + let mut counts_obj = serde_json::Map::new(); + for (key, count) in &self.counts { + let key_str = serde_json::to_string(&key.serialize_to_json()).unwrap(); + counts_obj.insert(key_str, Value::from(*count)); + } + serde_json::json!({ - "sums": sums_obj + "family": if self.family == ExactKind::Count { "Count" } else { "Sum" }, + "sums": sums_obj, + "counts": counts_obj }) } @@ -136,7 +223,9 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&(self.sums.len() as u32).to_le_bytes()); // Write each key-value pair + let mut ordered_keys = Vec::with_capacity(self.sums.len()); for (key, sum) in &self.sums { + ordered_keys.push(key); let key_bytes = key.serialize_to_bytes(); // Write key length and data @@ -147,17 +236,34 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&sum.to_le_bytes()); } + for key in ordered_keys { + buffer.extend_from_slice( + &self + .counts + .get(key) + .copied() + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + } + + buffer.push(if self.family == ExactKind::Count { + 1 + } else { + 0 + }); + buffer } } -impl AggregateCore for MultipleSumAccumulator { +impl AggregateCore for KeyedSumCountAccumulator { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleSumAccumulator" + "KeyedSumCountAccumulator" } fn as_any(&self) -> &dyn std::any::Any { @@ -172,20 +278,20 @@ impl AggregateCore for MultipleSumAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleSumAccumulator + // Check if other is also a KeyedSumCountAccumulator if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleSumAccumulator with {}", + "Cannot merge KeyedSumCountAccumulator with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleSumAccumulator + // Downcast to KeyedSumCountAccumulator let other_multiple_sum = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleSumAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedSumCountAccumulator")?; // Use the existing merge_accumulators method let merged = Self::merge_accumulators(vec![self.clone(), other_multiple_sum.clone()])?; @@ -194,13 +300,17 @@ impl AggregateCore for MultipleSumAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleSum + if self.family == ExactKind::Count { + AggregationType::Count + } else { + AggregationType::Sum + } } fn approx_memory_bytes(&self) -> usize { // HashMap. Label strings dominate; use a // conservative per-entry estimate plus HashMap overhead. - const BYTES_PER_ENTRY: usize = 96; + const BYTES_PER_ENTRY: usize = 112; std::mem::size_of::() + self.sums.len() * BYTES_PER_ENTRY } @@ -217,26 +327,35 @@ impl AggregateCore for MultipleSumAccumulator { use crate::storage_engines::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() - .ok_or("Key required for MultipleSumAccumulator")?; + .ok_or("Key required for KeyedSumCountAccumulator")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleSumAccumulator { +impl MultipleSubpopulationAggregate for KeyedSumCountAccumulator { fn query( &self, statistic: Statistic, key: &KeyByLabelValues, _query_kwargs: Option<&HashMap>, ) -> Result> { - match statistic { - Statistic::Sum | Statistic::Count => self - .sums + match (&self.family, statistic) { + (ExactKind::Sum, Statistic::Sum) => self.sums.get(key).copied().ok_or_else(|| { + "Key not found in KeyedSumCountAccumulator" + .to_string() + .into() + }), + (ExactKind::Count, Statistic::Count) => self + .counts .get(key) - .copied() - .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), + .map(|count| *count as f64) + .ok_or_else(|| { + "Sample count unavailable in KeyedSumCountAccumulator" + .to_string() + .into() + }), _ => Err( - format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), + format!("Unsupported statistic in KeyedSumCountAccumulator: {statistic:?}").into(), ), } } @@ -246,17 +365,41 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { } } -impl MergeableAccumulator for MultipleSumAccumulator { +impl MergeableAccumulator for KeyedSumCountAccumulator { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleSumAccumulator::new(); + let family = accumulators[0].family.clone(); + if accumulators.iter().any(|acc| acc.family != family) { + return Err("Cannot merge different keyed additive families".into()); + } + let mut result = KeyedSumCountAccumulator::for_family(family); for acc in accumulators { + for key in acc.sums.keys() { + match ( + result.counts.get(key).copied(), + acc.counts.get(key).copied(), + ) { + (None, Some(count)) if !result.sums.contains_key(key) => { + result.counts.insert(key.clone(), count); + } + (Some(existing), Some(count)) => { + if let Some(total) = existing.checked_add(count) { + result.counts.insert(key.clone(), total); + } else { + result.counts.remove(key); + } + } + _ => { + result.counts.remove(key); + } + } + } for (key, sum) in acc.sums { *result.sums.entry(key).or_insert(0.0) += sum; } @@ -273,14 +416,14 @@ mod tests { use super::*; #[test] - fn test_multiple_sum_accumulator_creation() { - let acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_creation() { + let acc = KeyedSumCountAccumulator::new(); assert!(acc.sums.is_empty()); } #[test] - fn test_multiple_sum_accumulator_update() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_update() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -295,8 +438,37 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_query() { - let mut acc = MultipleSumAccumulator::new(); + fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut first = KeyedSumCountAccumulator::for_family(ExactKind::Count); + first.update(key.clone(), 10.0); + first.update(key.clone(), 20.0); + let mut second = KeyedSumCountAccumulator::for_family(ExactKind::Count); + second.update(key.clone(), 7.0); + let merged = KeyedSumCountAccumulator::merge_accumulators(vec![first, second]).unwrap(); + for acc in [ + merged.clone(), + KeyedSumCountAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + ] { + assert_eq!(acc.family, ExactKind::Count); + assert!(acc.query(Statistic::Sum, &key, None).is_err()); + assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); + } + } + + #[test] + fn keyed_additive_merge_rejects_different_planner_families() { + assert!(KeyedSumCountAccumulator::merge_accumulators(vec![ + KeyedSumCountAccumulator::for_family(ExactKind::Sum), + KeyedSumCountAccumulator::for_family(ExactKind::Count), + ]) + .is_err()); + } + + #[test] + fn test_keyed_sum_count_accumulator_query() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -315,8 +487,8 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_get_keys() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_get_keys() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -332,9 +504,9 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_merge() { - let mut acc1 = MultipleSumAccumulator::new(); - let mut acc2 = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_merge() { + let mut acc1 = KeyedSumCountAccumulator::new(); + let mut acc2 = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -345,15 +517,15 @@ mod tests { acc2.add_sum(key1.clone(), 5.0); // Same key, different accumulator - let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.sums.get(&key1), Some(&15.0)); // Should be merged assert_eq!(merged.sums.get(&key2), Some(&20.0)); // Should be preserved } #[test] - fn test_multiple_sum_accumulator_serialization() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_serialization() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -361,18 +533,18 @@ mod tests { // Test JSON serialization let json = acc.serialize_to_json(); - let deserialized = MultipleSumAccumulator::deserialize_from_json(&json).unwrap(); + let deserialized = KeyedSumCountAccumulator::deserialize_from_json(&json).unwrap(); assert_eq!(deserialized.sums.get(&key), Some(&42.5)); // Test byte serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = MultipleSumAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedSumCountAccumulator::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.sums.get(&key), Some(&42.5)); } #[test] fn test_trait_object() { - let mut acc = MultipleSumAccumulator::new(); + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -381,6 +553,6 @@ mod tests { let trait_obj: Box = Box::new(acc); // Test type name through trait object - assert_eq!(trait_obj.type_name(), "MultipleSumAccumulator"); + assert_eq!(trait_obj.type_name(), "KeyedSumCountAccumulator"); } } diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index af284459..51bdb47a 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -5,15 +5,16 @@ pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; pub mod edge_runtime_adapter; +pub mod exact_accumulator; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; +pub mod keyed_counter_state; +pub mod keyed_max_state; +pub mod keyed_min_state; +pub mod keyed_sum_count_accumulator; pub mod max_accumulator; pub mod min_accumulator; -pub mod multiple_increase_accumulator; -pub mod multiple_max_accumulator; -pub mod multiple_min_accumulator; -pub mod multiple_sum_accumulator; pub mod sketch_envelope_accumulator; pub mod sum_accumulator; pub mod univmon_accumulator; @@ -27,11 +28,11 @@ pub use dd_sketch_accumulator::*; pub use hll_sketch_accumulator::*; pub use hydra_kll_accumulator::*; pub use increase_accumulator::*; +pub use keyed_counter_state::*; +pub use keyed_max_state::*; +pub use keyed_min_state::*; +pub use keyed_sum_count_accumulator::*; pub use max_accumulator::*; pub use min_accumulator::*; -pub use multiple_increase_accumulator::*; -pub use multiple_max_accumulator::*; -pub use multiple_min_accumulator::*; -pub use multiple_sum_accumulator::*; pub use sketch_envelope_accumulator::*; pub use sum_accumulator::*; diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 2cbe66fe..d5ff3b02 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -197,7 +197,11 @@ impl SingleSubpopulationAggregate for SumAccumulator { } match statistic { - Statistic::Sum | Statistic::Count => Ok(self.sum), + Statistic::Sum => Ok(self.sum), + Statistic::Count => self + .observation_count + .map(|count| count as f64) + .ok_or_else(|| "sample count is unavailable for this Sum payload".into()), _ => Err(format!("Unsupported statistic in SumAccumulator: {statistic:?}").into()), } } @@ -308,10 +312,7 @@ mod tests { crate::SingleSubpopulationAggregate::query(&acc, Statistic::Sum, None).unwrap(), 42.0 ); - assert_eq!( - crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), - 42.0 - ); + assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).is_err()); assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Min, None).is_err()); // SumAccumulator is a single subpopulation accumulator, doesn't need key-based queries @@ -321,6 +322,17 @@ mod tests { ); } + #[test] + fn count_readout_uses_observation_count_not_sum() { + let mut acc = SumAccumulator::new(); + acc.update(10.0); + acc.update(20.0); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), + 2.0 + ); + } + #[test] fn test_sum_accumulator_merge() { let acc1 = SumAccumulator::with_sum(10.0); diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index a710e57a..f9583d60 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -112,7 +112,7 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `AggregationConfig` via + /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via /// `PolicyRegistry::get(output.policy_fp)`. The legacy /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` /// is the only identity handle on `PrecomputedOutput`. Outputs @@ -338,7 +338,7 @@ mod tests { use crate::precompute_engine::operators::{DDSketchAccumulator, SumAccumulator}; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -384,11 +384,11 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> AggregationConfig { + fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers obtain the id // via `config.policy_fp_u64()`. - AggregationConfig { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs new file mode 100644 index 00000000..ee76c8b4 --- /dev/null +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -0,0 +1,295 @@ +//! Bind raw ingestion to a selected Planner producer and its raw dependency edge. +use super::accumulator_factory::{create_planner_accumulator, AccumulatorUpdater}; +use crate::storage_engines::types::KeyByLabelValues; +use asap_types::{executable_plan::BackendNodeBinding, PrecomputeMaterialization}; +use planner_types::post_asap::{ + EdgeRole, ExecutableOperatorPayload, GroupingStrategy, PostAsapNodeId, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; +use planner_types::pre_asap::{ColumnRef, QueryExpr, Source}; +use std::collections::HashMap; + +/// A validated executable projection; semantics come from the installed node. +/// The retained node ID makes failures attributable to the selected DAG. +#[derive(Debug, Clone)] +pub struct RawDagProgram { + pub node: PostAsapNodeId, + pub family: SummaryFamilyType, + pub input: SummaryUpdate, + pub grouping: GroupingStrategy, + pub reduction: planner_types::pre_asap::Reduction, + projected_column: Option, +} + +impl RawDagProgram { + pub fn from_plan( + plan: &asap_types::precompute_plan::PrecomputePlan, + config: &PrecomputeMaterialization, + ) -> Result { + let mut selected: Option = None; + for installed in plan.executable_dags.values() { + installed.validate()?; + let dag = installed.document.decode()?; + for node in &dag.nodes { + if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { summary_definition }) if summary_definition.fingerprint() == config.policy_fingerprint()) + { + continue; + } + let ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + reduction, + } = &node.payload + else { + return Err( + "raw materialization binding must identify a Planner SummaryAgg".into(), + ); + }; + if config.derived_input.is_some() { + return Err("derived producer must execute through maintenance DAG".into()); + } + let incoming: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node.id).collect(); + let [edge] = incoming.as_slice() else { + return Err("raw SummaryAgg must have exactly one DAG input".into()); + }; + if edge.role != EdgeRole::Input { + return Err("raw SummaryAgg input edge has wrong role".into()); + } + let source = dag + .nodes + .iter() + .find(|n| n.id == edge.producer) + .ok_or("missing raw DAG input")?; + let ExecutableOperatorPayload::Fallback { expression } = &source.payload else { + return Err("raw producer requires an executable source input; maintenance edges cannot be bypassed".into()); + }; + let scan = match expression { + QueryExpr::TimeRange { child, .. } => child.as_ref(), + source => source, + }; + match scan { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + .. + } if metric == &config.metric => { + let (metric, window, filter) = + control_plane::physical::compiler::raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + )?; + if metric != config.metric + || window.is_some_and(|seconds| seconds != config.window_size) + || asap_types::utils::normalize_spatial_filter(&filter) + != config.spatial_filter_normalized + { + return Err( + "raw DAG source filter/window differs from physical binding".into(), + ); + } + } + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if config.table_name.as_ref() == Some(table_ref) => { + return Err( + "raw table execution requires a validated table scan executor".into(), + ); + } + _ => return Err("raw DAG input does not match installed source routing".into()), + } + if let planner_types::pre_asap::Reduction::Reduce(keys) = reduction { + if keys.is_without() { + return Err( + "raw without reduction requires explicit dynamic population routing" + .into(), + ); + } + let names = keys + .keys() + .iter() + .map(|id| { + source + .output_schema + .fields + .get(*id) + .map(|f| f.name.clone()) + .ok_or("missing reduction column") + }) + .collect::, _>>()?; + if names != config.grouping_labels.names() { + return Err("DAG reduction differs from physical population binding".into()); + } + } + if let SummaryFamilyType::ExactAggregate(kind, _) = family { + if input.item.is_some() { + return Err( + "raw exact populations must follow Planner reduction, not an item map" + .into(), + ); + } + if !matches!(input.weight, SummaryInputExpr::Column(_)) + && !(matches!(kind, planner_types::post_asap::ExactKind::Count) + && input.weight == SummaryInputExpr::Constant(1.0)) + { + return Err("raw exact update differs from stored source projection".into()); + } + } + if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { + return Err( + "materialization storage family differs from selected Planner node".into(), + ); + } + // The stored descriptor must name the same update semantics; its + // content identity cannot be reused for an unrelated DAG program. + let update_matches = match (&input.weight, config.sample_update_rule()) { + ( + SummaryInputExpr::Column(_), + asap_types::SampleUpdateRule::Value { scale }, + ) => scale == 1.0, + (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { + *value == 1.0 + } + ( + SummaryInputExpr::ResetAwareCounterDelta { .. }, + asap_types::SampleUpdateRule::CounterDelta { scale }, + ) => scale == 1_000_000.0, + _ => { + asap_types::accumulator_spec::is_unit_sample_frequency(input) + || (matches!( + family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ) + ) && input.weight == SummaryInputExpr::Constant(1.0)) + } + }; + if !update_matches { + return Err("DAG update differs from stored summary identity".into()); + } + let program = Self { + node: node.id, + family: family.clone(), + input: input.clone(), + grouping: grouping.clone(), + reduction: reduction.clone(), + projected_column: config + .effective_value_projection() + .column() + .map(str::to_owned), + }; + program.validate()?; + if let Some(old) = &selected { + if old.family != program.family + || old.input != program.input + || old.grouping != program.grouping + || old.reduction != program.reduction + { + return Err( + "one stored definition is bound to incompatible Planner producers" + .into(), + ); + } + } else { + selected = Some(program); + } + } + } + selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) + } + + pub fn updater(&self) -> Result, String> { + create_planner_accumulator(&self.family, &self.input, &self.grouping) + } + + fn validate(&self) -> Result<(), String> { + match &self.input.weight { + SummaryInputExpr::Column(ColumnRef::SampleValue) | SummaryInputExpr::Constant(_) => {} + SummaryInputExpr::Column( + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }, + ) if self.projected_column.as_ref() == Some(name) => {} + SummaryInputExpr::ResetAwareCounterDelta { + value: ColumnRef::SampleValue, + series: planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + } if excluding.is_empty() => {} + _ => return Err("raw DAG weight expression is unsupported".into()), + } + fn item(expr: &SummaryInputExpr) -> bool { + match expr { + SummaryInputExpr::Column(ColumnRef::Named(_) | ColumnRef::SampleValue) => true, + SummaryInputExpr::Tuple(items) => items.iter().all(item), + SummaryInputExpr::EntityIdentity( + planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + ) => excluding.is_empty(), + _ => false, + } + } + if self.input.item.as_ref().is_some_and(|e| !item(e)) { + return Err("raw DAG item expression is unsupported".into()); + } + self.updater().map(|_| ()) + } + + pub fn uses_counter_delta(&self) -> bool { + matches!( + self.input.weight, + SummaryInputExpr::ResetAwareCounterDelta { .. } + ) + } + + pub fn apply( + &self, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + ) -> Result<(), String> { + let weight = match &self.input.weight { + SummaryInputExpr::Constant(c) => *c, + // The worker retains one previous value per series across pane rotation. + SummaryInputExpr::Column(_) | SummaryInputExpr::ResetAwareCounterDelta { .. } => value, + _ => return Err("unsupported raw weight expression".into()), + }; + let scalar_frequency = asap_types::accumulator_spec::is_unit_sample_frequency(&self.input) + && !updater.is_keyed(); + let weight = if scalar_frequency { value } else { weight }; + updater.validate_single_input(weight)?; + if updater.is_keyed() { + let labels = super::worker::parse_labels_from_series_key(series); + fn eval( + expr: &SummaryInputExpr, + series: &str, + value: f64, + labels: &HashMap<&str, &str>, + ) -> Result, String> { + Ok(match expr { + SummaryInputExpr::EntityIdentity(_) => vec![series.to_owned()], + SummaryInputExpr::Column(ColumnRef::SampleValue) => vec![value.to_string()], + SummaryInputExpr::Column(ColumnRef::Named(name)) => vec![labels + .get(name.as_str()) + .map(|s| super::worker::decode_label_value(s).into_owned()) + .ok_or_else(|| format!("missing DAG item column {name}"))?], + SummaryInputExpr::Tuple(items) => items + .iter() + .map(|i| eval(i, series, value, labels)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + _ => return Err("unsupported raw item expression".into()), + }) + } + let item = self + .input + .item + .as_ref() + .ok_or("keyed DAG kernel requires an explicit item")?; + let key = KeyByLabelValues::new_with_labels(eval(item, series, value, &labels)?); + updater.update_keyed(&key, weight, timestamp); + } else { + updater.update_single(weight, timestamp); + } + Ok(()) + } +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 751f47e5..01af6231 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -20,7 +20,7 @@ use xxhash_rust::xxh64::xxh64; /// hashing or pane lookup. `group_key` and `policy_fp` still travel /// alongside the sid: `group_key` is consumed at emit-time to render the /// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `AggregationConfig` from the hot-reload snapshot (window +/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window /// shape, late-data policy, etc.). Together they let the worker key state /// by sid without losing the data the legacy `(agg_id, group_key)` shape /// carried. @@ -50,8 +50,8 @@ pub enum WorkerMessage { /// `(metric, attrs_fingerprint, agg_kind_canonical)` — see /// `SeriesIdResolver::resolve`. Worker keys `group_states` on this. sid: u64, - /// Source `AggregationConfig` fingerprint. Worker looks up its - /// `AggregationConfig` (window size, sketch kind/config, late + /// Source `PrecomputeMaterialization` fingerprint. Worker looks up its + /// `PrecomputeMaterialization` (window size, sketch kind/config, late /// data policy, etc.) via `snap.get_aggregation_config(policy_fp.as_u64())`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons (e.g. "constant"). @@ -78,7 +78,7 @@ pub enum WorkerMessage { AccumulatorInput { /// Registry-allocated bucket identity; see `GroupSamples::sid`. sid: u64, - /// Source `AggregationConfig` fingerprint; see + /// Source `PrecomputeMaterialization` fingerprint; see /// `GroupSamples::policy_fp`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons, matching the diff --git a/data_plane/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs index afccd016..e1214fd9 100644 --- a/data_plane/src/precompute_engine/window_manager.rs +++ b/data_plane/src/precompute_engine/window_manager.rs @@ -18,7 +18,7 @@ pub struct WindowManager { impl WindowManager { /// Create a new WindowManager. /// - /// `window_size_secs` and `slide_interval_secs` come from `AggregationConfig` + /// `window_size_secs` and `slide_interval_secs` come from `PrecomputeMaterialization` /// (which stores them in seconds). They are converted to milliseconds internally. pub fn new(window_size_secs: u64, slide_interval_secs: u64) -> Self { Self::with_origin(window_size_secs, slide_interval_secs, None) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 2c48006f..c6aac060 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,6 +1,6 @@ -use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, -}; +#[cfg(test)] +use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; +use crate::precompute_engine::accumulator_factory::AccumulatorUpdater; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::group_key::GroupKey; use crate::precompute_engine::metrics::record_late_input; @@ -11,7 +11,7 @@ use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; @@ -37,10 +37,11 @@ use tracing::{debug, debug_span, info, warn}; /// producing one output per (sid, window) — exactly like Arroyo's /// `GROUP BY window, key`. struct GroupState { + program: Option>, series_id: u64, catalog_generation: Option>, input_revisions: BTreeMap>, - config: Arc, + config: Arc, /// Source policy fingerprint that minted this sid. Held so /// `evict_orphaned_groups` can check liveness against the streaming /// config snapshot (a sid stays alive only while its source policy is @@ -411,7 +412,7 @@ impl Worker { /// /// B7.6 — buckets are now keyed by `sid` (a single u64) rather than /// `(agg_id, group_key)`. `policy_fp` is the source config's - /// fingerprint, used to fetch the `AggregationConfig` from the + /// fingerprint, used to fetch the `PrecomputeMaterialization` from the /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// @@ -425,12 +426,16 @@ impl Worker { sid: u64, policy_fp: PolicyFingerprint, group_key: &Arc, - ) -> Option<&mut GroupState> { + ) -> Result, String> { if !self.group_states.contains_key(&sid) { let snap = self.hot_reload.snapshot(); - let cfg = snap.get_aggregation_config(policy_fp.as_u64())?; + let Some(cfg) = snap.get_aggregation_config(policy_fp.as_u64()) else { + return Ok(None); + }; + let program = snap.raw_programs.get(&policy_fp.as_u64()).cloned(); let config = Arc::new(cfg.clone()); let gs = GroupState { + program, series_id: sid, catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), @@ -454,7 +459,7 @@ impl Worker { self.group_count .store(self.group_states.len(), Ordering::Relaxed); } - self.group_states.get_mut(&sid) + Ok(self.group_states.get_mut(&sid)) } /// Process a batch of samples for a specific sid bucket. @@ -462,7 +467,7 @@ impl Worker { /// /// This is the core of the Arroyo-equivalent GROUP BY logic. /// B7.6 — buckets are keyed by `sid`; `policy_fp` is the source - /// `AggregationConfig` fingerprint used to resolve the bucket's + /// `PrecomputeMaterialization` fingerprint used to resolve the bucket's /// config on first sight; `group_key` is held on the resulting /// `GroupState` for emit-time label rendering. pub fn process_group_samples( @@ -479,7 +484,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -489,6 +494,10 @@ impl Worker { return Ok(()); } let state = self.group_states.get_mut(&sid).unwrap(); + #[cfg(not(test))] + if state.program.is_none() { + return Err("raw precompute requires an installed post-ASAP DAG producer".into()); + } // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. @@ -537,12 +546,19 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = - if let SampleUpdateRule::CounterDelta { .. } = state.config.sample_update_rule() { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), + ) { + reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) + } else { + Some(*val) + }; for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -589,9 +605,14 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } + if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), ) { if let Some(input) = state.input_revisions.get_mut(&bucket_start) { Arc::make_mut(input).first_revision = 0; @@ -600,8 +621,16 @@ impl Worker { continue; } record_late_input("append_correction", "raw_sample"); - let mut updater = create_accumulator_updater(&state.config); - apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + let mut updater = + installed_updater(state.program.as_deref(), &state.config)?; + apply_installed_sample( + state.program.as_deref(), + &mut *updater, + series_key, + *val, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { @@ -646,12 +675,21 @@ impl Worker { // only closes an idle pane, not a long-running bulk ingest whose // records share one event timestamp. state.touch_pane(bucket_start, now_ms); - let updater = state - .active_panes - .entry(bucket_start) - .or_insert_with(|| create_accumulator_updater(&state.config)); + if let std::collections::btree_map::Entry::Vacant(entry) = + state.active_panes.entry(bucket_start) + { + entry.insert(installed_updater(state.program.as_deref(), &state.config)?); + } + let updater = state.active_panes.get_mut(&bucket_start).unwrap(); if let Some(value) = value { - apply_sample(&mut **updater, series_key, value, *ts, &state.config); + apply_installed_sample( + state.program.as_deref(), + &mut **updater, + series_key, + value, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { observer.observe( @@ -767,7 +805,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -1349,7 +1387,10 @@ pub fn extract_metric_name(series_key: &str) -> &str { /// aggregation config's `grouping_labels`. /// /// The series key format is: `metric_name{label1="val1",label2="val2",...}` -pub fn extract_key_from_series(series_key: &str, config: &AggregationConfig) -> KeyByLabelValues { +pub fn extract_key_from_series( + series_key: &str, + config: &PrecomputeMaterialization, +) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1492,19 +1533,61 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { std::borrow::Cow::Owned(out) } +fn installed_updater( + program: Option<&super::raw_dag::RawDagProgram>, + config: &PrecomputeMaterialization, +) -> Result, String> { + if let Some(program) = program { + return program.updater(); + } + #[cfg(test)] + { + Ok(create_fixture_accumulator(config)) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + +fn apply_installed_sample( + program: Option<&super::raw_dag::RawDagProgram>, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + config: &PrecomputeMaterialization, +) -> Result<(), String> { + if let Some(program) = program { + return program.apply(updater, series, value, timestamp); + } + #[cfg(test)] + { + apply_sample(updater, series, value, timestamp, config); + Ok(()) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + /// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. /// /// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted /// from the series' **aggregated_labels** — these are the labels that become /// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a MultipleSumAccumulator's HashMap). This matches the Arroyo SQL +/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL /// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +#[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) { if updater.is_keyed() { // Planner's PromQL Top-K item is the series identity. When no @@ -1529,7 +1612,7 @@ pub(crate) fn apply_sample( /// Convert a cumulative counter sample into a non-negative, reset-aware /// increment. Only the immediately preceding sample per series is retained; /// pane rotation therefore cannot lose the boundary increment. -fn reset_aware_counter_delta( +pub(crate) fn reset_aware_counter_delta( previous: &mut HashMap, series_key: &str, value: f64, @@ -1560,7 +1643,7 @@ fn reset_aware_counter_delta( /// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. fn extract_aggregated_key_from_series( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1792,7 +1875,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::storage_engines::types::StreamingConfig; @@ -1808,7 +1891,7 @@ mod tests { window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { make_agg_config_full( id, metric, @@ -1831,7 +1914,7 @@ mod tests { slide_secs: u64, grouping: Vec<&str>, aggregated: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers below build the // streaming-config map by reading `config.policy_fp_u64()` @@ -1841,7 +1924,7 @@ mod tests { } else { WindowKind::Sliding }; - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, agg_sub_type.to_string(), HashMap::new(), @@ -1861,7 +1944,7 @@ mod tests { } fn make_worker( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1871,7 +1954,7 @@ mod tests { } fn make_worker_with_lateness( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1900,12 +1983,12 @@ mod tests { } /// Build a fresh `StreamingConfigHandle` from a map of agg_id - /// → AggregationConfig. Worker::new takes this handle instead of - /// the old `HashMap>`. Tests use this + /// → PrecomputeMaterialization. Worker::new takes this handle instead of + /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( - configs: HashMap, + configs: HashMap, ) -> crate::storage_engines::types::StreamingConfigHandle { crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), @@ -1991,7 +2074,7 @@ mod tests { assert_eq!(output.start_timestamp as i64, *ts); assert_eq!(output.end_timestamp as i64, *ts); // Raw mode emits PolicyFingerprint::UNSET (no source - // AggregationConfig in the raw-mode fast path). The sink + // PrecomputeMaterialization in the raw-mode fast path). The sink // drops UNSET outputs with a warn — verified separately // via integration tests. assert!(output.policy_fp.is_unset()); @@ -2452,7 +2535,7 @@ mod tests { #[test] fn test_keyed_accumulator_aggregated_labels() { // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside MultipleSumAccumulator) + // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) let config = make_agg_config_full( 3, "cpu", @@ -2504,10 +2587,10 @@ mod tests { let (_output, acc) = &captured[0]; let ms_acc = acc .as_any() - .downcast_ref::() - .expect("should be MultipleSumAccumulator"); + .downcast_ref::() + .expect("should be KeyedSumCountAccumulator"); - // The MultipleSumAccumulator should have two internal keys: "A" and "B" + // The KeyedSumCountAccumulator should have two internal keys: "A" and "B" assert_eq!(ms_acc.sums.len(), 2, "two host keys inside one accumulator"); let mut found_a = false; @@ -2680,100 +2763,15 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn test_worker_from_streaming_config_yaml() { - let yaml = r#" -aggregations: -- aggregationType: SingleSubpopulation - aggregationSubType: Sum - labels: - grouping: [] - rollup: [] - aggregated: [] - metric: requests_total - parameters: {} - tumblingWindowSize: 10 - windowSize: 10 - windowType: tumbling - slideInterval: 0 - spatialFilter: '' -"#; - - let data: serde_yaml::Value = serde_yaml::from_str(yaml).expect("valid YAML"); - let streaming_config = - StreamingConfig::from_yaml_data(&data).expect("valid streaming config"); - - // PR 5: the streaming-config key is the policy fingerprint. - let agg_id = *streaming_config - .materializations() - .keys() - .next() - .expect("one agg"); - assert!(streaming_config.contains(agg_id)); - - let agg_configs = streaming_config.materializations().clone(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - let pf = PolicyFingerprint(agg_id); - let sid = 1_u64; - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(1_000, 3.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(5_000, 4.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(9_000, 5.0)]), - ) - .unwrap(); - assert_eq!(sink.len(), 0); - - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(10_000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - - let (output, acc) = &captured[0]; - let _ = agg_id; - assert!(!output.policy_fp.is_unset()); - assert_eq!(output.start_timestamp, 0); - assert_eq!(output.end_timestamp, 10_000); - - let sum_acc = acc - .as_any() - .downcast_ref::() - .expect("should be SumAccumulator"); - assert!( - (sum_acc.sum - 12.0).abs() < 1e-10, - "sum should be 3+4+5=12, got {}", - sum_acc.sum - ); + fn test_worker_rejects_flat_streaming_config_yaml() { + let data = + serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); + assert!(StreamingConfig::from_yaml_data(&data).is_err()); } #[test] fn test_extract_key_from_series() { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), @@ -3413,7 +3411,7 @@ aggregations: /// Build a worker with explicit wall-clock closure grace values. fn make_worker_with_wall_clock_policy( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, late_data_policy: LateDataPolicy, idle_grace_period_ms: i64, @@ -4278,3 +4276,179 @@ aggregations: ); } } + +#[cfg(test)] +mod dag_execution_tests { + use super::*; + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::precompute_engine::output_sink::CapturingOutputSink; + use crate::storage_engines::types::StreamingConfig; + use asap_types::query_plan::ExactReadout; + + fn plan(query: &str) -> control_plane::physical::compiler::CompiledPhysicalPlan { + let mut json: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut item = json["query_workload"]["repeating_queries"][0].clone(); + item["query"] = query.into(); + json["query_workload"]["repeating_queries"] = serde_json::json!([item]); + let snapshot = serde_json::from_value(json).unwrap(); + crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap() + } + + // A selected producer must govern updates, persisted family, and query readout. + #[test] + fn installed_dag_ingestion_persistence_and_readout() { + for (query, readout, answer) in [ + ( + "sum_over_time(asap_demo_gauge[5s])", + ExactReadout::Sum, + 54.0, + ), + ( + "count_over_time(asap_demo_gauge[5s])", + ExactReadout::Count, + 5.0, + ), + ("min_over_time(asap_demo_gauge[5s])", ExactReadout::Min, 3.0), + ( + "max_over_time(asap_demo_gauge[5s])", + ExactReadout::Max, + 20.0, + ), + ("rate(asap_demo_counter_total[5s])", ExactReadout::Rate, 5.5), + ( + "increase(asap_demo_counter_total[5s])", + ExactReadout::Increase, + 27.5, + ), + ] { + let plan = plan(query); + let config = plan + .precompute_plan + .materializations + .first() + .expect("ASAP producer required") + .clone(); + let fp = config.policy_fingerprint(); + let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); + let doc = serde_json::to_value(&streaming).unwrap(); + assert!(doc.get("aggregation_configs").is_none()); + let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let sink = Arc::new(CapturingOutputSink::new()); + let (_tx, rx) = mpsc::channel(8); + let mut worker = Worker::new( + 0, + rx, + sink.clone(), + StreamingConfigHandle::new(streaming), + WorkerRuntimeConfig { + max_buffer_per_series: 100, + allowed_lateness_ms: 10_000, + pass_raw_samples: false, + raw_mode_aggregation_id: 0, + late_data_policy: LateDataPolicy::Drop, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + }, + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicI64::new(0)), + ); + worker + .process_group_samples( + 1, + fp, + &Arc::new(GroupKey::new([])), + [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, v)| (config.metric.clone(), t, v)) + .collect(), + ) + .unwrap(); + worker.force_close_all().unwrap(); + let mut states = BTreeMap::new(); + for (output, state) in sink.drain() { + let select = if matches!( + config.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) { + output.start_timestamp == 0 && output.end_timestamp == 5000 + } else { + output.end_timestamp <= 5000 + }; + if select { + assert_eq!( + state.get_accumulator_type().planner_exact_family(), + Some(readout.planner_family()) + ); + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()) + .unwrap(); + states.insert( + output.end_timestamp as i64, + Arc::new(restored) as Arc, + ); + } + } + assert!(!states.is_empty(), "{query}: no stored states"); + let group = + crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { + entries: vec![std::rc::Rc::new(states)], + agg_type: config.aggregation_type, + }; + assert_eq!( + group.exact_value_for(readout, &None, 0, 5000), + Some(answer), + "{query}" + ); + } + } + + // A flat config and a DAG whose producer no longer matches its binding cannot install. + #[test] + fn execution_requires_matching_dag_producer() { + assert!(serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err()); + let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; + plan.executable_dags.clear(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("DAG producer")); + } + // Changing a raw update must not silently reuse the original summary identity. + #[test] + fn altered_dag_update_cannot_reuse_a_stored_definition() { + let mut plan = plan("sum_over_time(asap_demo_gauge[5s])").precompute_plan; + let installed = plan.executable_dags.values_mut().next().unwrap(); + let mut dag = installed.document.decode().unwrap(); + for node in &mut dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + input, .. + } = &mut node.payload + { + input.weight = planner_types::post_asap::SummaryInputExpr::Constant(99.0); + } + } + installed.document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( + installed.document.query_id.clone(), + &dag, + ) + .unwrap(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("update")); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index ba07c0b9..b067a0ac 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; -use asap_types::AggregationType; +use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; use crate::query_engines::EngineError; @@ -51,64 +51,40 @@ impl ResolvedMaterialization<'_> { matches!( &self.summary.operator, SummaryOperator::Configured { - aggregation_type: AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease - | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleMin - | AggregationType::MultipleMax, + family: SummaryFamilyType::ExactAggregate(..), .. } ) } fn supports(&self, node: &QueryPlanNode) -> bool { - let SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - .. - } = &self.summary.operator - else { + let SummaryOperator::Configured { family, .. } = &self.summary.operator else { // Partial legacy descriptors cannot attest a configured capability. return false; }; - use AggregationType::*; match node { - QueryPlanNode::ExactReadout { readout, .. } => match readout { - ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Sum, - ExactReadout::Increase | ExactReadout::Rate => { - matches!(aggregation_type, Increase | MultipleIncrease) - } - // Direction is the family now -- no `aggregation_sub_type` - // cross-check, and a minimum summary can no longer be - // offered up for a maximum readout. - ExactReadout::Min => matches!(aggregation_type, Min | MultipleMin), - ExactReadout::Max => matches!(aggregation_type, Max | MultipleMax), - }, + QueryPlanNode::ExactReadout { readout, .. } => family == &readout.planner_family(), QueryPlanNode::SummaryEstimate { query, .. } => match query { QueryReadout::Quantile { q } => { q.is_finite() && (0.0..=1.0).contains(q) - && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) + } + QueryReadout::Cardinality => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon)) } - QueryReadout::Cardinality => matches!(aggregation_type, HLL | UnivMon), QueryReadout::FrequencyL2 | QueryReadout::FrequencyEntropy => { - *aggregation_type == UnivMon + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) } - QueryReadout::PointCount { value: None, .. } if *aggregation_type == UnivMon => { + QueryReadout::PointCount { value: None, .. } if matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) => { true } - QueryReadout::PointCount { .. } => matches!( - aggregation_type, - CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap - ), - QueryReadout::TopK { .. } => matches!( - aggregation_type, - CountMinSketchWithHeap | CountSketchWithHeap - ), + QueryReadout::PointCount { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap)) + } + QueryReadout::TopK { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + } }, _ => false, } diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 2a451d6a..1fb3c3c5 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -902,9 +902,9 @@ mod tests { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 41b41ab7..20ff521d 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1366,9 +1366,9 @@ mod tests { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index f48ba405..880a98c8 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -65,8 +65,8 @@ use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryNode, + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, }; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; @@ -203,11 +203,13 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match agg_type { - AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease => asap_types::Statistic::Sum, + let stat = match agg_type.planner_exact_family()? { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => asap_types::Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => asap_types::Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { + asap_types::Statistic::Increase + } + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -224,9 +226,7 @@ impl GroupState { .ok() } - /// Finalize a compiler-declared exact readout. Rate and increase share - /// reset-aware Increase state physically, but remain distinct operations - /// in QueryPlan so serving never infers semantics from PromQL text. + /// Finalize the Planner-declared exact family with its matching readout. pub fn exact_value_for( &self, readout: asap_types::query_plan::ExactReadout, @@ -237,40 +237,32 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { - asap_types::Statistic::Count - } - ( - asap_types::query_plan::ExactReadout::Sum, - AggregationType::Sum | AggregationType::MultipleSum, - ) => asap_types::Statistic::Sum, - ( - asap_types::query_plan::ExactReadout::Increase, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Increase, - ( - asap_types::query_plan::ExactReadout::Rate, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Rate, - ( - asap_types::query_plan::ExactReadout::Min, - AggregationType::Min | AggregationType::MultipleMin, - ) => asap_types::Statistic::Min, - ( - asap_types::query_plan::ExactReadout::Max, - AggregationType::Max | AggregationType::MultipleMax, - ) => asap_types::Statistic::Max, - _ => return None, + if agg_type.planner_exact_family().as_ref() != Some(&readout.planner_family()) { + return None; + } + let stat = match readout { + asap_types::query_plan::ExactReadout::Count => asap_types::Statistic::Count, + asap_types::query_plan::ExactReadout::Sum => asap_types::Statistic::Sum, + asap_types::query_plan::ExactReadout::Increase => asap_types::Statistic::Increase, + asap_types::query_plan::ExactReadout::Rate => asap_types::Statistic::Rate, + asap_types::query_plan::ExactReadout::Min => asap_types::Statistic::Min, + asap_types::query_plan::ExactReadout::Max => asap_types::Statistic::Max, }; + let planner_state = entries.iter().flat_map(|w| w.values()).any(|a| { + a.as_any() + .is::() + }); // Temporal exact summaries are the hot path for long-window // dashboards. Merge their concrete, fixed-size states in one batch // instead of allocating a boxed trait object for every pane. - if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease - ) { + if !planner_state + && matches!( + readout, + asap_types::query_plan::ExactReadout::Increase + | asap_types::query_plan::ExactReadout::Rate + ) + { let accumulators = entries .iter() .flat_map(|windows| windows.values()) @@ -286,11 +278,7 @@ impl GroupState { ]); return merged.query_statistic(stat, key, &query_kwargs).ok(); } - if matches!( - agg_type, - AggregationType::Min | AggregationType::MultipleMin - ) && readout == asap_types::query_plan::ExactReadout::Min - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Min { return entries .iter() .flat_map(|windows| windows.values()) @@ -303,11 +291,7 @@ impl GroupState { .into_iter() .reduce(f64::min); } - if matches!( - agg_type, - AggregationType::Max | AggregationType::MultipleMax - ) && readout == asap_types::query_plan::ExactReadout::Max - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() .flat_map(|windows| windows.values()) @@ -334,9 +318,6 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == asap_types::query_plan::ExactReadout::Count { - return merged.aux_stats().count.map(|count| count as f64); - } merged.query_statistic(stat, key, &query_kwargs).ok() } @@ -590,9 +571,13 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + let exact_family = agg_type.planner_exact_family(); if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Increase | ExactKind::Rate, + _ + )) ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -608,12 +593,12 @@ impl QueryExecutionContext<'_> { )); } } - if let Some((reduction, is_min)) = match agg_type { - AggregationType::Min | AggregationType::MultipleMin => Some(( + if let Some((reduction, is_min)) = match exact_family.as_ref() { + Some(SummaryFamilyType::ExactAggregate(ExactKind::Min, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Min, true, )), - AggregationType::Max | AggregationType::MultipleMax => Some(( + Some(SummaryFamilyType::ExactAggregate(ExactKind::Max, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Max, false, )), @@ -665,8 +650,11 @@ impl QueryExecutionContext<'_> { // counter and extrema state. Additive pane summaries must // remain contiguous because a missing pane is not zero. if matches!( - agg_type, - AggregationType::Sum | AggregationType::MultipleSum + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count, + _ + )) ) { check_panes(windows.keys().copied().collect())?; } @@ -911,9 +899,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } ( - GroupState::ExactAgg { entries, .. }, - GroupState::ExactAgg { entries: more, .. }, + GroupState::ExactAgg { entries, agg_type }, + GroupState::ExactAgg { + entries: more, + agg_type: incoming, + }, ) => { + if agg_type.planner_exact_family() != incoming.planner_exact_family() { + return Err(SummaryExecutorError::UnsupportedFamily); + } entries.extend(more); } // `find_candidates`'s exact-match contract never produces a @@ -1255,29 +1249,19 @@ fn summary_family_matches_sketch( /// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> /// ExactKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease -/// -> ExactKind::Increase` — confirmed against that module's own -/// dispatch table rather than invented here). +/// side. Count and Rate remain distinct families even though their runtime +/// accumulators share implementations with Sum and Increase. /// -/// `ExactKind::Count`/`Rate`/`Min`/`Max` are not matched by this legacy -/// family-discovery path. For `Count`/`Rate` the final operation is ambiguous -/// from the stored accumulator alone. `Min`/`Max` were excluded for a reason -/// that no longer holds -- direction used to be unrecoverable once a summary -/// reached `AggKind::ExactAgg`, and is now the family itself -- but admitting -/// them here widens candidate discovery beyond the family split and is left -/// as follow-up. Installed QueryPlans carry an explicit `ExactReadout`, and +/// Installed QueryPlans carry an explicit `ExactReadout`, and /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (family, agg_type), - ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - AggregationType::Sum | AggregationType::MultipleSum, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - AggregationType::Increase | AggregationType::MultipleIncrease, + family, + SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count | ExactKind::Increase | ExactKind::Rate, + _ ) - ) + ) && agg_type.planner_exact_family().as_ref() == Some(family) } /// Project a full label-values map down to the requested `by` columns -- @@ -1447,6 +1431,58 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + #[test] + fn keyed_count_state_follows_planner_family_and_query_readout() { + use crate::precompute_engine::operators::KeyedSumCountAccumulator; + use asap_types::query_plan::ExactReadout; + + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut payload = KeyedSumCountAccumulator::for_family(ExactKind::Count); + payload.update(key.clone(), 10.0); + payload.update(key.clone(), 20.0); + let state = GroupState::ExactAgg { + entries: vec![Rc::new(BTreeMap::from([( + 60_000, + Arc::new(payload) as Arc, + )]))], + agg_type: AggregationType::Count, + }; + assert_eq!( + state.exact_value_for(ExactReadout::Count, &Some(key.clone()), 0, 60_000), + Some(2.0) + ); + assert_eq!( + state.exact_value_for(ExactReadout::Sum, &Some(key), 0, 60_000), + None + ); + } + + #[test] + fn state_merge_rejects_different_planner_families() { + let index = SketchStore::new(); + let context = QueryExecutionContext { + index: &index, + t0_ms: 0, + t1_ms: 60_000, + is_cumulative: true, + allowed_materializations: None, + }; + let states = vec![ + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Rate, + }, + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Increase, + }, + ]; + assert!(matches!( + context.merge_states(states), + Err(SummaryExecutorError::UnsupportedFamily) + )); + } + #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { let binding = asap_types::query_plan::MaterializationBinding { diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index 08eb75e0..0d46b252 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -102,7 +102,7 @@ impl QueryResult { /// Attach an accuracy envelope. Chainable so engine paths /// can build the bare result first and decorate once the - /// `agg_id → AggregationConfig → AccuracyProfile` lookup + /// `agg_id → PrecomputeMaterialization → AccuracyProfile` lookup /// has resolved. pub fn with_accuracy(mut self, envelope: AccuracyEnvelope) -> Self { match &mut self { diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index f3780b04..084c87cc 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -1,5 +1,5 @@ //! `AccuracyProfile` — derived error / confidence bound for each -//! `AggregationConfig`. +//! `PrecomputeMaterialization`. //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. @@ -12,7 +12,7 @@ //! //! ## Scope of this module //! -//! Pure derivation: `derive(&AggregationConfig)` +//! Pure derivation: `derive(&PrecomputeMaterialization)` //! looks at `aggregation_type` and the relevant entries in //! `config.parameters` and returns an `AccuracyProfile`. No //! runtime measurement, no sampling — just the textbook bound. @@ -31,14 +31,14 @@ use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::AggregationType; pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; use planner_types::post_asap::SketchParams as PlannerParams; /// Derive an [`AccuracyProfile`] from a pinned -/// [`AggregationConfig`]. Reads `aggregation_type` and any +/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any /// necessary entries in `parameters`; falls back to exact for /// unknown / legacy variants (harmless — the caller just gets /// "0 error" rather than a panic). @@ -52,7 +52,7 @@ use planner_types::post_asap::SketchParams as PlannerParams; /// ε_st`; the random parts compose in quadrature but the staleness part is /// adversarial, so linear addition is the honest envelope). δ is /// unchanged (staleness is not probabilistic). -pub fn derive(config: &AggregationConfig) -> AccuracyProfile { +pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { let mut profile = derive_sketch_only(config); let eps_st = config .parameters @@ -67,20 +67,20 @@ pub fn derive(config: &AggregationConfig) -> AccuracyProfile { /// Source adapter for installed aggregation configs. pub trait BackendAccuracyProfile { - fn derive(config: &AggregationConfig) -> Self; - fn derive_sketch_only(config: &AggregationConfig) -> Self; + fn derive(config: &PrecomputeMaterialization) -> Self; + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; } impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &AggregationConfig) -> Self { + fn derive(config: &PrecomputeMaterialization) -> Self { derive(config) } - fn derive_sketch_only(config: &AggregationConfig) -> Self { + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { derive_sketch_only(config) } } /// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { +fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { match config.aggregation_type { AggregationType::UnivMon => AccuracyProfile { epsilon: f64::MAX, @@ -91,13 +91,11 @@ fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { // `DeltaSetAggregator` exact-set-membership family lived // here too before its retirement.) AggregationType::Sum + | AggregationType::Count | AggregationType::Increase + | AggregationType::Rate | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax => AccuracyProfile::exact(), + | AggregationType::Max => AccuracyProfile::exact(), AggregationType::CountMinSketch => { let (rows, cols) = cms_params(config); @@ -220,7 +218,7 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { // authority on *accuracy*, not on *construction*. /// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &AggregationConfig) -> (u64, u64) { +fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { let rows = config .parameters .get("d") @@ -234,7 +232,7 @@ fn cms_params(config: &AggregationConfig) -> (u64, u64) { (rows, cols) } -fn hll_precision(config: &AggregationConfig) -> u32 { +fn hll_precision(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("precision") @@ -244,7 +242,7 @@ fn hll_precision(config: &AggregationConfig) -> u32 { .unwrap_or(14) } -fn kll_k(config: &AggregationConfig) -> u32 { +fn kll_k(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("K") @@ -254,7 +252,7 @@ fn kll_k(config: &AggregationConfig) -> u32 { .unwrap_or(200) } -fn ddsketch_alpha(config: &AggregationConfig) -> f64 { +fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { config .parameters .get("alpha") @@ -266,7 +264,7 @@ fn ddsketch_alpha(config: &AggregationConfig) -> f64 { /// from `parameters["heap_size"]` with a default of 100 — /// matches the default the control plane's planner uses when the /// caller didn't override. -fn cms_heap_size(config: &AggregationConfig) -> u64 { +fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { config .parameters .get("heap_size") @@ -373,8 +371,11 @@ mod tests { use serde_json::{json, Value}; use std::collections::HashMap; - fn base_config(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( + fn base_config( + agg_type: AggregationType, + params: HashMap, + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index 0e970b82..ce8e476d 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use std::time::{SystemTime, UNIX_EPOCH}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; @@ -507,12 +507,12 @@ impl BackfillRegistry { /// control-plane-facing HTTP endpoint can return specific 404 / /// 409 / 400 statuses. `CreateError::UnknownAgg` is no longer /// returned from this method — the caller proves the agg - /// exists by holding the `AggregationConfig` — but the variant + /// exists by holding the `PrecomputeMaterialization` — but the variant /// is kept on the enum for HTTP error-mapping compatibility /// (the handler still produces it when its own lookup misses). pub fn create_checked( &self, - config: &AggregationConfig, + config: &PrecomputeMaterialization, created_at_ms: u64, time_range: (u64, u64), source: BackfillSource, @@ -883,7 +883,9 @@ pub use service::{ default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, BackfillServiceHandle, ReaderFactory, }; +#[cfg(test)] pub use window_builder::build_backfilled_accumulator; +pub use window_builder::build_dag_accumulator; pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; #[cfg(test)] diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 430aca74..8bfcf0cf 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -22,10 +22,11 @@ use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use super::raw_sample_reader::RawSample; +#[cfg(test)] use super::window_builder::build_backfilled_accumulator; use super::worker::WindowProcessor; use super::BackfillRegistry; @@ -39,7 +40,7 @@ use super::BackfillRegistry; /// Kept local to the backfill module (not shared with live) per /// the §5e separation ask; the implementations must stay identical /// by convention. -fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { +fn extract_group_key(series_key: &str, config: &PrecomputeMaterialization) -> String { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); for label_name in &config.grouping_labels.names() { @@ -71,7 +72,7 @@ fn build_group_key_label_values(group_key: &str) -> KeyByLabelValues { /// live windows occupy the same storage row. fn resolve_backfill_bucket_sid( resolver: &SeriesIdResolver, - config: &AggregationConfig, + config: &PrecomputeMaterialization, series_key: &str, store: Option<&crate::storage_engines::sketch_db::index::SketchStore>, captured_generation: Option<&asap_types::sds::CatalogGeneration>, @@ -124,7 +125,7 @@ fn fallback_bucket_id(group_key: &str) -> u64 { pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest /// `StreamingConfig` at each window to find the - /// `AggregationConfig` for `agg_id`. The snapshot is cheap + /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. config: StreamingConfigHandle, /// Destination for rebuilt windows. Tests may omit it to record registry @@ -185,22 +186,6 @@ impl BackfillWindowProcessor { self.series_resolver = Some(series_resolver); self } - - /// Look up the `AggregationConfig` for `agg_id` in the current - /// `StreamingConfig` snapshot. Returns an error string if the - /// agg has been removed from the config since the job was - /// created — rare but worth handling (e.g. operator retired - /// the agg mid-backfill; the `BackfillWorker` will - /// `mark_failed` the job with this message). - fn config_for_agg( - &self, - agg_id: u64, - ) -> Result> { - let snap = self.config.snapshot(); - snap.get_aggregation_config(agg_id).cloned().ok_or_else(|| { - format!("agg_id {agg_id} not in current StreamingConfig — retired mid-backfill?").into() - }) - } } /// One per-sid bucket assembled by [`BackfillWindowProcessor::process_window`]. @@ -219,7 +204,16 @@ impl WindowProcessor for BackfillWindowProcessor { window_range: (u64, u64), samples: Vec, ) -> Result<(), Box> { - let config = self.config_for_agg(agg_id)?; + let snapshot = self.config.snapshot(); + let config = snapshot + .get_aggregation_config(agg_id) + .cloned() + .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + let program = snapshot.raw_programs.get(&agg_id).cloned(); + #[cfg(not(test))] + if program.is_none() { + return Err("backfill requires a post-ASAP DAG installation".into()); + } // B7.7 — sid-keyed bucketing. Per the schema-retirement #5 // step 6 plan, the backfill processor's per-window grouping is @@ -289,7 +283,18 @@ impl WindowProcessor for BackfillWindowProcessor { for (sid, bucket) in by_bucket { let SidBucket { group_key, samples } = bucket; - let accumulator = build_backfilled_accumulator(&config, &samples); + let accumulator = if let Some(program) = &program { + super::window_builder::build_dag_accumulator(program, &samples)? + } else { + #[cfg(test)] + { + build_backfilled_accumulator(&config, &samples) + } + #[cfg(not(test))] + { + return Err("missing backfill DAG producer".into()); + } + }; // Keyed accumulators (MultipleSubpopulation) carry their // subpopulation keys internally; the PrecomputedOutput's // `key` represents the *group* key (grouping_labels @@ -374,7 +379,7 @@ mod tests { use asap_types::KeyByLabelNames; use std::sync::Arc; - fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. let grouping_labels = if grouping.is_empty() { @@ -382,7 +387,7 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -401,7 +406,7 @@ mod tests { ) } - fn streaming_config_with(config: AggregationConfig) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); Arc::new(StreamingConfig::new(map)) @@ -567,7 +572,7 @@ mod tests { /// when given the same ordered samples. #[test] fn backfill_builds_bit_identical_sum_accumulator_to_live() { - use crate::precompute_engine::accumulator_factory::create_accumulator_updater; + use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; let cfg = sum_config(1, "m", vec![]); @@ -592,7 +597,7 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_accumulator_updater(&cfg); + let mut updater = create_fixture_accumulator(&cfg); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -612,7 +617,7 @@ mod tests { serialisations for SumAccumulator. \ If this test fails, something diverged — check:\n\ (1) Is `build_backfilled_accumulator` still calling \ - `create_accumulator_updater`?\n\ + `create_fixture_accumulator`?\n\ (2) Did a recent change to `SumAccumulator` introduce \ non-deterministic state (e.g. a seed)?\n\ (3) Does `serialize_to_bytes` include any timestamp \ diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index 7f9208a1..79d9ccdb 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -43,7 +43,7 @@ pub struct RawSample { /// honour. Exactly one metric name plus zero or more equality /// matchers on grouping labels — no regex, no negation, no /// lexicographic ranges. The control plane picks the subset of -/// `AggregationConfig.grouping_labels` that should gate the read. +/// `PrecomputeMaterialization.grouping_labels` that should gate the read. /// /// Rationale: every supported exact-DB backend (Prometheus, /// ClickHouse, S3+Gorilla) can evaluate this filter efficiently, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index c0b35971..86d64fac 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -342,15 +342,15 @@ mod tests { MockRawSampleReader, RawSample, }; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Mutex; - fn sum_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -369,7 +369,7 @@ mod tests { ) } - fn streaming_with(cfg: AggregationConfig) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); Arc::new(StreamingConfig::new(m)) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index 04a4eed3..6bf028c9 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -4,13 +4,16 @@ //! It shares the pure accumulator factory and update primitives with live ingest //! so both paths use the same sketch semantics. +#[cfg(test)] use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, + create_fixture_accumulator, AccumulatorUpdater, }; +#[cfg(test)] use crate::precompute_engine::worker::apply_sample; use crate::storage_engines::sketch_db::backfill::raw_sample_reader::RawSample; use crate::storage_engines::types::AggregateCore; -use asap_types::aggregation_config::AggregationConfig; +#[cfg(test)] +use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -25,11 +28,12 @@ use asap_types::aggregation_config::AggregationConfig; /// The function is synchronous + pure (no I/O, no async, no global /// state). Suitable to call from inside a `WindowProcessor` /// implementation without worrying about the async runtime. +#[cfg(test)] pub fn build_backfilled_accumulator( - config: &AggregationConfig, + config: &PrecomputeMaterialization, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_accumulator_updater(config); + let mut updater: Box = create_fixture_accumulator(config); for sample in samples { apply_sample( &mut *updater, @@ -45,14 +49,14 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_config() -> AggregationConfig { - AggregationConfig::new( + fn sum_config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), @@ -156,3 +160,28 @@ mod tests { assert!(aux.sum == Some(0.0) || aux.sum.is_none()); } } + +/// Backfill uses the same selected DAG producer and update expressions as live input. +pub fn build_dag_accumulator( + program: &crate::precompute_engine::raw_dag::RawDagProgram, + samples: &[RawSample], +) -> Result, String> { + let mut updater = program.updater()?; + let mut previous = std::collections::HashMap::new(); + for sample in samples { + let value = if program.uses_counter_delta() { + crate::precompute_engine::worker::reset_aware_counter_delta( + &mut previous, + &sample.labels, + sample.value, + sample.timestamp_ms, + ) + } else { + Some(sample.value) + }; + if let Some(value) = value { + program.apply(&mut *updater, &sample.labels, value, sample.timestamp_ms)?; + } + } + Ok(updater.take_accumulator()) +} diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index e8f3d873..e696fe52 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -137,7 +137,7 @@ pub enum AggKind { /// Complete resolver identity for a configured materialization. All live and /// replay paths must include policy semantics, not just the sketch family. pub(crate) fn materialization_kind_for_config( - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, ) -> String { format!( "{}|{}", @@ -149,7 +149,9 @@ pub(crate) fn materialization_kind_for_config( /// Resolve the physical state family produced by a precompute policy. This is /// shared by SID minting and store registration so a sketch policy can never /// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config(config: &asap_types::aggregation_config::AggregationConfig) -> AggKind { +pub fn agg_kind_for_config( + config: &asap_types::aggregation_config::PrecomputeMaterialization, +) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; // HLL is intentionally absent from raw-value accumulator dispatch because @@ -592,7 +594,7 @@ mod tests { #[test] fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::AggregationConfig::new( + let config = asap_types::aggregation_config::PrecomputeMaterialization::new( AggregationType::HLL, String::new(), HashMap::from([("precision".to_string(), serde_json::json!(12))]), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index e0c9ba55..3d6b5799 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -92,11 +92,12 @@ fn reconstruct_exact_agg( bytes: &[u8], ) -> Option> { use crate::precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleIncreaseAccumulator, - MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedCounterState, KeyedSumCountAccumulator, MaxAccumulator, + MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::AggregateCore; match type_name { + "PlannerExactAccumulatorV1" => crate::precompute_engine::operators::exact_accumulator::ExactAccumulator::deserialize_from_bytes(bytes).ok().map(|a|Box::new(a) as Box), "SumAccumulator" => SumAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), @@ -109,10 +110,12 @@ fn reconstruct_exact_agg( "MaxAccumulator" => MaxAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), - "MultipleSumAccumulator" => MultipleSumAccumulator::deserialize_from_bytes(bytes) - .ok() - .map(|a| Box::new(a) as Box), - "MultipleIncreaseAccumulator" => MultipleIncreaseAccumulator::deserialize_from_bytes(bytes) + "KeyedSumCountAccumulator" => { + KeyedSumCountAccumulator::deserialize_from_bytes(bytes) + .ok() + .map(|a| Box::new(a) as Box) + } + "KeyedCounterState" => KeyedCounterState::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), // The keyed `MultipleMin`/`MultipleMax` forms and the @@ -138,7 +141,7 @@ fn reconstruct_exact_agg( /// so the mint-driven path (B7.6) and the sid-direct path (B7.7) stay /// byte-identical on the values they hand to the index. fn build_attrs_fp_and_label_map( - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Result<(String, BTreeMap), String> { if let Some(labels) = &output.population_labels { @@ -232,7 +235,7 @@ pub struct SummarySeriesMetadata { /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source - /// `AggregationConfig` (test fixtures + the early-Phase-5 sketch + /// `PrecomputeMaterialization` (test fixtures + the early-Phase-5 sketch /// ingest path that didn't thread the config through); the index /// skips those entries — they're reachable through the legacy /// `instances_matching(metric, gbk)` walk if a query needs them. @@ -2242,7 +2245,7 @@ impl SketchStore { } /// Phase 5 M2.3.5 — query the precompute payloads across every sid - /// belonging to one `AggregationConfig` (identified by `metric` + + /// belonging to one `PrecomputeMaterialization` (identified by `metric` + /// `agg_cfg.aggregation_type`), shaped as the legacy `Store` /// trait's `TimestampedBucketsMap`. Lets the query engine swap /// `Store::query_precomputed_output` for `SketchStore` without @@ -2943,7 +2946,7 @@ impl SketchStore { } /// Phase 5 M2.3.6e — write-side helper. Given an - /// `AggregationConfig` and one `(PrecomputedOutput, AggregateCore)` + /// `PrecomputeMaterialization` and one `(PrecomputedOutput, AggregateCore)` /// pair (the shape both the live worker AND the backfill processor /// emit), compute the precompute sid, register a metadata entry on /// first sight, and append the payload window. Used by @@ -2963,7 +2966,7 @@ impl SketchStore { pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3095,7 +3098,7 @@ impl SketchStore { pub fn ingest_precompute_with_series_id( &self, sid: u64, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3113,6 +3116,18 @@ impl SketchStore { output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + let expected = agg_cfg.accumulator_spec().ok()?.family; + if matches!( + expected, + planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) + ) && accumulator + .get_accumulator_type() + .planner_exact_family() + .as_ref() + != Some(&expected) + { + return None; + } let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; // Keep the physical lifetime alive through publication. Removal takes @@ -6084,4 +6099,91 @@ mod tests { ); assert_eq!(idx.series.len(), 2); } + // Flush and reopen must preserve Planner family rather than reconstructing Rate as Increase. + #[test] + fn planner_exact_families_survive_disk_eviction_and_restart() { + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::storage_engines::types::{AggregateCore, AggregationType}; + let kinds = [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Min, + AggregationType::Max, + AggregationType::Rate, + AggregationType::Increase, + ]; + let stats = [ + asap_types::Statistic::Sum, + asap_types::Statistic::Count, + asap_types::Statistic::Min, + asap_types::Statistic::Max, + asap_types::Statistic::Rate, + asap_types::Statistic::Increase, + ]; + let expected = [16.0, 3.0, 2.0, 8.0, 3.0, 6.0]; + let temp = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + for (i, kind) in kinds.iter().enumerate() { + let mut metadata = meta(9000 + i as u64); + metadata.agg_kind = AggKind::ExactAgg { + agg_type: *kind, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }; + metadata.capability = Some(Capability::ExactAgg(*kind)); + metadata.accuracy = None; + store.register(metadata); + } + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + for window in 0..10u64 { + let mut state = + ExactAccumulator::new(kind.planner_exact_family().unwrap(), false).unwrap(); + for (time, value) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(None, value, time); + } + store.append_precompute( + 9000 + i as u64, + BTreeMap::new(), + (window * 30000, (window + 1) * 30000), + Box::new(state), + ); + } + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.approx_memory_bytes() == 0 + && store.list_sealed_epochs_len() == 0, + std::time::Duration::from_secs(5) + )); + persistence.shutdown(); + } + let store = Arc::new(SketchStore::new()); + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + let series = store.query_exact_agg_range(9000 + i as u64, 0, 30001); + assert_eq!(series.len(), 1, "{kind:?}"); + let state = &series[0].1[&30000]; + assert_eq!(state.get_accumulator_type(), *kind); + assert_eq!( + state + .query_statistic(stats[i], &None, &HashMap::new()) + .unwrap(), + expected[i] + ); + for (j, stat) in stats.iter().enumerate() { + if i != j { + assert!(state + .query_statistic(*stat, &None, &HashMap::new()) + .is_err()); + } + } + } + persistence.shutdown(); + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index d7de8c2b..768f0946 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -195,13 +195,13 @@ mod tests { use super::*; use crate::precompute_engine::operators::SumAccumulator; use crate::storage_engines::types::{AggregationType, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> AggregationConfig { - AggregationConfig { + fn sum_agg_config(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index f9a83493..0b712f45 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::Duration; use crate::storage_engines::types::StreamingConfig; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; use crate::storage_engines::sketch_db::index::SketchStore; @@ -87,7 +87,7 @@ pub fn reconcile_from_streaming_config( } // `live_signatures` is built exclusively from // `signature_from_agg_config`, which canonicalizes every - // streaming-config `AggregationConfig` to an `AggKind::ExactAgg` + // streaming-config `PrecomputeMaterialization` to an `AggKind::ExactAgg` // signature (`P`-prefixed). An `AggKind::Sketch` sid (OTLP // modified-sketch ingest path: KLL / HLL / DDSketch / CMS / // CountSketch) always produces an `S`-prefixed signature, so it @@ -166,7 +166,7 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { +fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { let agg_kind = AggKind::ExactAgg { agg_type: cfg.aggregation_type, parameters_canonical: canonical_parameters(&cfg.parameters), @@ -264,7 +264,7 @@ mod tests { use super::*; use std::collections::HashMap; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -276,8 +276,8 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), HashMap::new(), @@ -321,7 +321,7 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> StreamingConfig { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); diff --git a/data_plane/src/storage_engines/sketch_db/mod.rs b/data_plane/src/storage_engines/sketch_db/mod.rs index e2aff195..0ef39870 100644 --- a/data_plane/src/storage_engines/sketch_db/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/mod.rs @@ -16,12 +16,12 @@ pub mod sds; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; pub use backfill::{ - build_backfilled_accumulator, clickhouse_reader_factory, default_reader_factory, - noop_reader_factory, BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, - BackfillWorkerError, ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, - MockRawSampleReader, PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, - ReaderFactory, WindowProcessor, + build_dag_accumulator, clickhouse_reader_factory, default_reader_factory, noop_reader_factory, + BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, BackfillServiceHandle, + BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, BackfillWorkerError, + ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, MockRawSampleReader, + PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, ReaderFactory, + WindowProcessor, }; pub use lifecycle::{ warn_if_retention_inverted, AggStatus, SchemaEvictionConfig, SchemaEvictionHandle, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index c679d0bc..247fa9f0 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -680,7 +680,7 @@ impl ActivePhysicalPlanHandle { #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::AggregationConfig; + use crate::storage_engines::types::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -754,8 +754,8 @@ mod tests { } } - fn dummy_agg(id: u64) -> AggregationConfig { - AggregationConfig::new( + fn dummy_agg(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 57d47470..a91e8314 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -25,7 +25,7 @@ pub use streaming_config::*; pub use traits::*; // Cross-module re-export of asap_types data types so callers can -// write `crate::storage_engines::types::AggregationConfig` instead of +// write `crate::storage_engines::types::PrecomputeMaterialization` instead of // reaching across crates. pub use asap_types::aggregation_config::*; diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 4d268dc4..28e71b4c 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -62,7 +62,7 @@ pub struct PrecomputedOutput { #[serde(default)] pub origin: Origin, /// Content-addressed policy identity. The data plane's only handle - /// on which source `AggregationConfig` produced this output. + /// on which source `PrecomputeMaterialization` produced this output. /// `#[serde(default)]` on read preserves forward-compat with /// PR-3 / PR-4-era records that may not have carried the field; /// sinks treat `PolicyFingerprint::UNSET` as "skip this output" @@ -75,7 +75,7 @@ impl PrecomputedOutput { /// Construct a `Native` precompute. /// /// `policy_fp` is the content-addressed handle on the source - /// [`asap_types::AggregationConfig`]; sinks use it to look up the + /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the /// config via `PolicyRegistry::get(policy_fp)`. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 430dd10b..95055c2e 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -6,31 +6,22 @@ use std::fs::File; use std::io::BufReader; use std::ops::Index; -use asap_types::enums::QueryLanguage; -use asap_types::{AggregationConfig, MonitorSpec, PolicyRegistry}; +use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; use super::storage_backend::StorageBackend; -/// The backend's active streaming policy config: every `AggregationConfig` -/// currently pushed by the controller, plus the storage-backend pin and CDM -/// monitor specs. -/// -/// Formerly `asap_types::streaming_config::StreamingConfig` — moved here -/// (see `scratchpad/artifacts/enum-unification-plan.md`) because -/// `control_plane` never actually depended on this type: its own -/// `StreamingConfigEmitter` hand-builds wire-compatible JSON independently, -/// and `PolicyRegistry::from_streaming_config` (the only thing that made -/// `asap_types::PolicyRegistry` -- genuinely shared -- look coupled to this -/// type) had exactly one real caller, this struct's own `policy_registry()` -/// method below. `asap_types` keeps the lower-level `PolicyRegistry:: -/// from_configs` primitive this method now calls directly. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// DAG installation plus a derived in-memory routing index. The flat index is +/// never serialized as executable configuration. Raw programs are validated +/// and shared once per installed producer across all of its population states. +#[derive(Debug, Clone, Serialize)] pub struct StreamingConfig { - #[serde( - rename = "aggregation_configs", - alias = "materializations_by_policy_fingerprint" - )] - pub materializations_by_policy_fingerprint: HashMap, + #[serde(skip)] + pub(crate) raw_programs: + HashMap>, + /// Authoritative execution configuration: Planner DAGs and physical bindings. + pub precompute_plan: Option, + #[serde(skip)] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -45,15 +36,60 @@ pub struct StreamingConfig { pub monitors: Vec, } +// Flat aggregation lists are deliberately not an accepted execution document. +impl<'de> Deserialize<'de> for StreamingConfig { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + precompute_plan: asap_types::precompute_plan::PrecomputePlan, + #[serde(default)] + storage_backend: StorageBackend, + #[serde(default)] + monitors: Vec, + } + let doc = Document::deserialize(deserializer)?; + let mut config = + Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; + config.storage_backend = doc.storage_backend; + config.monitors = doc.monitors; + Ok(config) + } +} + impl StreamingConfig { - pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { + pub fn new( + materializations_by_policy_fingerprint: HashMap, + ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } } + /// Build the routing projection only after validating the DAG installation. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|c| { + c.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::new(materializations); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + /// CDM monitor specs the data-plane coordinator should serve (may be empty). pub fn monitors(&self) -> &[MonitorSpec] { &self.monitors @@ -63,10 +99,12 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), @@ -80,12 +118,15 @@ impl StreamingConfig { self.storage_backend } - pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { + pub fn get_aggregation_config( + &self, + aggregation_id: u64, + ) -> Option<&PrecomputeMaterialization> { self.materializations_by_policy_fingerprint .get(&aggregation_id) } - pub fn materializations(&self) -> &HashMap { + pub fn materializations(&self) -> &HashMap { &self.materializations_by_policy_fingerprint } @@ -126,56 +167,12 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut materializations_by_policy_fingerprint: HashMap = - HashMap::new(); - - if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { - for aggregation_data in aggregations { - // Retention comes from each aggregation entry; identity is derived from - // its configuration content. - let num_aggregates_to_retain = aggregation_data - .get("numAggregatesToRetain") - .and_then(|v| v.as_u64()); - let config = AggregationConfig::from_yaml_data( - aggregation_data, - num_aggregates_to_retain, - QueryLanguage::PromQl, - )?; - if !config.population_key_encoding.is_legacy() { - anyhow::bail!( - "legacy streaming input does not support this population key encoding" - ); - } - if config.derived_input.is_some() { - anyhow::bail!( - "legacy streaming input cannot execute a derived summary program" - ); - } - // PR 5: the map key IS the policy-fingerprint u64. - // `AggregationConfig::policy_fp_u64()` is the canonical - // accessor for this value. - materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); - } - } - - let mut config = Self::new(materializations_by_policy_fingerprint); - // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each - // entry deserializing into a MonitorSpec. Absent → empty (the common - // case). The data-plane monitor coordinator reads these. - if let Some(monitors) = data.get("monitors").and_then(|v| v.as_sequence()) { - for m in monitors { - let spec: MonitorSpec = serde_yaml::from_value(m.clone()).map_err(|e| { - anyhow::anyhow!("invalid monitor spec in streaming-config: {e}") - })?; - config.monitors.push(spec); - } - } - Ok(config) + serde_yaml::from_value(data.clone()).map_err(Into::into) } } impl Index for StreamingConfig { - type Output = AggregationConfig; + type Output = PrecomputeMaterialization; fn index(&self, aggregation_id: u64) -> &Self::Output { &self.materializations_by_policy_fingerprint[&aggregation_id] @@ -190,7 +187,7 @@ impl Default for StreamingConfig { impl StreamingConfig { #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { + pub fn get_all_aggregation_configs(&self) -> &HashMap { self.materializations() } } @@ -199,153 +196,16 @@ impl StreamingConfig { mod tests { use super::*; - /// Pre-Phase-5 deploys serialize `StreamingConfig` without the - /// `storage_backend` field; deserialize must default to `SketchStore` - /// so the router keeps dispatching to `ASAPQueryEngine` unchanged. - #[test] - fn deserialize_legacy_yaml_defaults_to_asap_tier() { - let yaml = "{\"aggregation_configs\":{}}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("legacy decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::SketchStore); - } - - #[test] - fn deserialize_with_explicit_double_write_pin() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"double_write\"}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("Phase-5 decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::DoubleWrite); - } - - /// #746 deleted the archive tier; its storage-axis spelling is no longer - /// a known variant, so a stale config naming it fails to decode rather - /// than silently pinning some other tier. + // Old flat lists cannot become execution authority through JSON or YAML. #[test] - fn deserialize_rejects_the_removed_archive_axis() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_object_store\"}"; - assert!(serde_json::from_str::(yaml).is_err()); - } - - #[test] - fn legacy_yaml_rejects_derived_summary_input() { - let data = serde_yaml::from_str::(&format!( - "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) - )).unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!(error - .to_string() - .contains("legacy streaming input cannot execute")); - } - - #[test] - fn legacy_yaml_rejects_canonical_population_key_encoding() { - let data: Value = serde_yaml::from_str( - r#" -aggregations: -- aggregationType: Sum - aggregationSubType: '' - metric: m - population_key_encoding: canonical_labels_v1 - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#, - ) - .unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!( - error.to_string().contains("population key encoding"), - "{error}" - ); - } - - /// PR 5: a streaming-config YAML that omits `aggregationId` - /// parses correctly — the backend derives identity from content - /// via `PolicyFingerprint::from_config`. The map key is the - /// fingerprint's u64 form. - #[test] - fn from_yaml_data_accepts_entry_without_aggregation_id() { - let yaml = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); - let (k, v) = cfg - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_ne!(*k, 0, "derived id is not the 0 sentinel"); - assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); - assert_eq!(v.metric, "cpu_seconds"); - } - - /// PR 5: a streaming-config YAML that still spells out - /// `aggregationId: N` parses the SAME as one without — the field - /// is silently dropped. - #[test] - fn from_yaml_data_ignores_explicit_aggregation_id() { - let with = "\ -aggregations:\n\ -- aggregationId: 42\n aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let without = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let w: Value = serde_yaml::from_str(with).expect("with yaml ok"); - let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); - let cw = StreamingConfig::from_yaml_data(&w).expect("with"); - let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - let (kwo, _) = cwo - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_eq!( - kw, kwo, - "explicit aggregationId in YAML must not change identity" - ); - assert_ne!( - *kw, 42, - "the explicit value must NOT leak through as the map key" - ); - } - - #[test] - fn from_yaml_data_parses_monitors_section() { - // CDM monitor specs: a top-level `monitors:` array must populate - // StreamingConfig.monitors (the data-plane coordinator reads these). - let yaml = "\ -aggregations: []\n\ -monitors:\n\ -- agg_id: 16346598078036168951\n key: \"\"\n tau: 5000.0\n epsilon: 0.05\n window_ms: 10000\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode monitors"); - assert_eq!(cfg.monitors().len(), 1, "monitors: section must be parsed"); - let m = &cfg.monitors()[0]; - assert_eq!(m.agg_id, 16346598078036168951); - assert_eq!(m.tau, 5000.0); - assert_eq!(m.window_ms, 10000); - assert_eq!(m.epsilon, 0.05); - } - - #[test] - fn from_yaml_data_absent_monitors_is_empty() { - let yaml = "aggregations: []\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode"); - assert!( - cfg.monitors().is_empty(), - "no monitors: → empty (byte-compat)" - ); + fn rejects_flat_aggregation_documents() { + for text in [ + r#"{"aggregation_configs":{}}"#, + "aggregations: []", + "aggregations: [{aggregationType: Sum, metric: m}]", + ] { + let yaml = serde_yaml::from_str(text).unwrap(); + assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); + } } } diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index ed88c51d..21cd2b11 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,7 +27,7 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -35,8 +35,8 @@ use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( +fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 5158398b..895df1b4 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -2,14 +2,14 @@ //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore //! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build AggregationConfig with +//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with //! the correct aggregation_type string. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationConfig, AggregationType, KeyByLabelValues, PrecomputedOutput, QueryLanguage, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, StreamingConfig, WindowKind, }; use crate::AggregateCore; @@ -23,7 +23,7 @@ use std::collections::HashMap; fn ingest_with_fresh_resolver( summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, - agg_cfg: &AggregationConfig, + agg_cfg: &PrecomputeMaterialization, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { @@ -89,7 +89,7 @@ pub fn create_engine_single_pop_with_aggregated( .collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -119,6 +119,8 @@ pub fn create_engine_single_pop_with_aggregated( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -175,7 +177,7 @@ pub fn create_engine_dual_input( let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation - let value_agg_config = AggregationConfig { + let value_agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: value_agg_type, aggregation_sub_type: String::new(), @@ -205,7 +207,7 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation - let keys_agg_config = AggregationConfig { + let keys_agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: key_agg_type, aggregation_sub_type: String::new(), @@ -235,6 +237,8 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -300,7 +304,7 @@ pub fn create_engine_two_metrics( let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config_a = AggregationConfig { + let agg_config_a = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: aggregation_type_a, aggregation_sub_type: String::new(), @@ -329,7 +333,7 @@ pub fn create_engine_two_metrics( let id_a = agg_config_a.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_a, agg_config_a); - let agg_config_b = AggregationConfig { + let agg_config_b = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: aggregation_type_b, aggregation_sub_type: String::new(), @@ -359,6 +363,8 @@ pub fn create_engine_two_metrics( materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -434,7 +440,7 @@ pub fn create_engine_three_metrics( (aggregation_type_b, &labels_b, metric_b), (aggregation_type_c, &labels_c, metric_c), ] { - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: agg_type, aggregation_sub_type: String::new(), @@ -466,6 +472,8 @@ pub fn create_engine_three_metrics( } let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -516,7 +524,7 @@ pub fn create_engine_multi_timestamp( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -546,6 +554,8 @@ pub fn create_engine_multi_timestamp( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -574,7 +584,7 @@ pub fn create_engine_multi_timestamp( /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the AggregationConfig (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -590,7 +600,7 @@ pub fn create_engine_multi_timestamp_with_window( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -620,6 +630,8 @@ pub fn create_engine_multi_timestamp_with_window( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b56408a2..a10cd3d1 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,4 +1,4 @@ -use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{KeyedSumCountAccumulator, SumAccumulator}; #[cfg(test)] use crate::storage_engines::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, @@ -18,7 +18,7 @@ fn test_single_subpopulation_interface() { #[test] fn test_multiple_subpopulation_interface() { // Multiple accumulator - matches Python behavior exactly - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let mut key = KeyByLabelValues::new(); key.insert("web".to_string()); @@ -43,7 +43,7 @@ fn test_interface_prevents_misuse() { let single_acc: Box = Box::new(SumAccumulator::with_sum(42.0)); let multi_acc: Box = - Box::new(MultipleSumAccumulator::new()); + Box::new(KeyedSumCountAccumulator::new()); // ✅ These work - correct usage let _result1 = single_acc.query(Statistic::Sum, None); @@ -68,7 +68,7 @@ fn test_python_alignment() { // Python: multiple_accumulator.query(Statistic.SUM, key) // Rust: multiple_accumulator.query(Statistic::Sum, &key) - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new(); multi_acc.add_sum(key.clone(), 100.0); let multi_trait: Box = Box::new(multi_acc); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 5150ddf0..6f33ca87 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -20,7 +20,7 @@ mod tests { use tempfile::NamedTempFile; #[test] - fn test_read_streaming_config() { + fn flat_streaming_file_is_rejected() { // PR 5: `aggregationId: 1` is silently dropped on read — the // streaming-config map key is the policy fingerprint derived // from content. The legacy field stays in this fixture to @@ -47,9 +47,6 @@ aggregations: let mut streaming_temp_file = NamedTempFile::new().unwrap(); write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.materializations_by_policy_fingerprint.is_empty()); - let agg = config.materializations().values().next().expect("one agg"); - assert_eq!(agg.num_aggregates_to_retain, Some(6)); + assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 51df1d98..db008fd8 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -851,7 +851,14 @@ async fn run_shared_dashboard(multi_pane: bool) { snapshot = serde_json::to_value(&typed).unwrap(); let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); - assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!(plan.precompute_plan.materializations.len(), 2); + let families = plan + .precompute_plan + .materializations + .iter() + .map(|m| m.aggregation_type.as_str()) + .collect::>(); + assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); if multi_pane { assert!(plan.lifecycle_estimates[0] diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 8463c9c4..f434bea2 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -37,7 +37,7 @@ //! the GET endpoint reflects the registered aggregation. //! * Test 2 — same shape with `group_by_labels: ["zone"]`; verifies //! #245's grouping plumb survives the round-trip into the backend's -//! `AggregationConfig.grouping_labels`. +//! `PrecomputeMaterialization.grouping_labels`. //! * Test 3 — full controller-to-query roundtrip: harness simulates //! the agent (builds DDSketch state with `asap_sketchlib`, encodes //! as a modified-OTLP `DdSketchDataPoint`), POSTs sketches to the @@ -45,7 +45,7 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use asap_types::AggregationConfig; +use asap_types::PrecomputeMaterialization; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -76,7 +76,7 @@ fn phase_aligned_now_ns() -> u64 { async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[AggregationConfig], + materializations: &[PrecomputeMaterialization], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the @@ -171,7 +171,7 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { use control_plane::physical::compiler::{BackendLocalPlanningInput, PhysicalPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( @@ -632,7 +632,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { // Verifies #245's grouping plumb survives the controller → backend // round-trip. The workload carries `group_by_labels: ["zone"]`; the // emitted JSON must surface `["zone"]` in `labels.grouping`, the -// backend's parser must materialise it into `AggregationConfig. +// backend's parser must materialise it into `PrecomputeMaterialization. // grouping_labels`, and the active-config snapshot must reflect that. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1252,7 +1252,7 @@ async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &AggregationConfig) -> (u32, u32) { +fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { let w = agg.parameters["w"] .as_u64() .expect("materialization must carry parameters.w") as u32; diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md new file mode 100644 index 00000000..7504c948 --- /dev/null +++ b/docs/design_docs/precompute-dag-execution.md @@ -0,0 +1,24 @@ +# Precompute execution from post-ASAP IR + +Audience: backend developers and reviewers of issue #762. + +The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. + +```mermaid +flowchart LR + P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] + I --> R[Raw source → SummaryAgg streaming kernel] + I --> M[Maintenance dependency scheduler] + R --> S[Stored summary frontier] + S --> M + S --> Q[Query projection and readout] + M --> S +``` + +For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. + +`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. + +Catalog schema version 3 carries Planner family in SDS. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. + +The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index 2522bb02..d7b3b9ce 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -197,9 +197,13 @@ The registry holds weak references, so retiring the final SID also releases its descriptors. `SketchInstanceMetadata` remains the registration and persistence compatibility DTO while older sidecars are read. -The implemented `SummaryDescriptor` currently contains one `SummaryOperator`, -one derived `FidelityGuarantee`, and a numeric state-schema version. The -implemented `DataDescriptor` contains typed source and value projections, a +The implemented `SummaryDescriptor` contains one `SummaryOperator`, +one derived `FidelityGuarantee`, and a numeric state-schema version. A configured +operator carries Planner's `SummaryFamilyType` as its semantic identity. Its +backend aggregation type and parameters describe the state codec and update +implementation; keyed grouping remains in the Data Descriptor. Descriptor +validation rejects a configured exact family that disagrees with its storage +type. The implemented `DataDescriptor` contains typed source and value projections, a canonical population filter, typed grouping columns and versioned observation semantics. The shared contract now also defines `SummaryInstance`, `ObservedSummaryInventory`, placement, completeness, From 59a4e0a84d70d121805193a0f454103a438cc056 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 22 Sep 2026 20:38:41 +0000 Subject: [PATCH 15/16] test: follow summary definition catalog in stacked plan assertions --- control_plane/tests/issue754_level1.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/control_plane/tests/issue754_level1.rs b/control_plane/tests/issue754_level1.rs index 92c649ee..da5c9f8d 100644 --- a/control_plane/tests/issue754_level1.rs +++ b/control_plane/tests/issue754_level1.rs @@ -123,7 +123,7 @@ fn assert_selected_plan(name: &str, plan: &CompiledPhysicalPlan) -> Option Date: Wed, 23 Sep 2026 03:38:27 +0000 Subject: [PATCH 16/16] fix: remove declaration for deleted edge runtime adapter --- data_plane/src/precompute_engine/operators/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index 51bdb47a..073db6e8 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -4,7 +4,6 @@ pub mod count_sketch_accumulator; pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; -pub mod edge_runtime_adapter; pub mod exact_accumulator; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator;