diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index f22bd867..5b804fd6 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -389,7 +389,7 @@ fn materialize_selected_sql( let aggregation = BackendAggregation { aggregation_id: String::new(), metric_name: format!("{table}.{}", value.column().unwrap_or("constant")), - family: crate::physical::compiler::physical_materialization_family(family), + family: family.clone(), window_secs, spatial_filter: String::new(), grouping: grouping.names(), @@ -610,7 +610,7 @@ fn bind_selected_node( .. } = clickhouse_materialization_leaf_contract(node, query.start_ms, query.end_ms) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let expected = crate::physical::compiler::physical_materialization_family(family); + let expected = family.clone(); let selected = select_materialization( &request.precompute_plan.materializations, &table_ref, diff --git a/control_plane/src/emit/backend_wire.rs b/control_plane/src/emit/backend_wire.rs index 829d1be4..ff294e8c 100644 --- a/control_plane/src/emit/backend_wire.rs +++ b/control_plane/src/emit/backend_wire.rs @@ -5,7 +5,7 @@ //! * the storage-routing table, which maps each metric's materialized summary //! families to the query shapes the ASAP tier serves natively versus the //! ones that belong to the archive; -//! * the aggregation and readout JSON the backend's `AggregationConfig` +//! * the aggregation and readout JSON the backend's `PrecomputeMaterialization` //! parser consumes. //! //! `backend_plan::from_stage_config` reuses [`build_backend_aggregation_json`] diff --git a/control_plane/src/emit/mod.rs b/control_plane/src/emit/mod.rs index c2ede7ee..d9d13f75 100644 --- a/control_plane/src/emit/mod.rs +++ b/control_plane/src/emit/mod.rs @@ -1,7 +1,7 @@ //! Backend-facing emission for a compiled physical plan. //! //! * [`backend_wire`] builds the storage-routing table and the aggregation / -//! readout JSON the backend's `AggregationConfig` parser consumes. +//! readout JSON the backend's `PrecomputeMaterialization` parser consumes. //! * [`monitor`] carries the CDM monitor declarations. pub mod backend_wire; diff --git a/control_plane/src/physical/backend_stage.rs b/control_plane/src/physical/backend_stage.rs index 165c6d11..bc60cf45 100644 --- a/control_plane/src/physical/backend_stage.rs +++ b/control_plane/src/physical/backend_stage.rs @@ -35,7 +35,7 @@ pub struct BackendAggregation { /// Internal-only id (see struct doc). Not on the wire. pub aggregation_id: String, /// Source metric the aggregation runs over. Required by the backend's - /// `AggregationConfig` parser. + /// `PrecomputeMaterialization` parser. pub metric_name: String, /// Planner-owned committed summary identity. Sketch entries carry a /// validated `SketchKind` (category + algorithm + params); exact entries diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 29ea07b0..1c54ba1f 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1256,10 +1256,7 @@ impl DeploymentPlanCompiler { .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); - // Rate is a readout over the same reset-aware counter state - // as Increase. Keep that semantic distinction in QueryPlan, - // while the physical store binds both to Increase state. - let physical_family = physical_materialization_family(&selected.family); + let physical_family = selected.family.clone(); let physical_algorithm = match &physical_family { SummaryFamilyType::ExactAggregate(kind, _) => { format!("{kind:?}").to_ascii_lowercase() @@ -1698,7 +1695,7 @@ impl DeploymentPlanCompiler { .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? .family; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != physical_materialization_family(node_family) + if materialization_family != *node_family || window_ms == 0 || source_window.unwrap_or(query.query_lookback_seconds).saturating_mul(1_000) % window_ms != 0 @@ -2843,13 +2840,11 @@ pub(super) fn estimated_state_bytes( A::HLL => 1u128 << parameter(&["precision", "p"], 14).min(24), A::DDSketch => 64 * 1024, A::Sum + | A::Count | A::Increase + | A::Rate | A::Min | A::Max - | A::MultipleSum - | A::MultipleIncrease - | A::MultipleMin - | A::MultipleMax | A::SingleSubpopulation | A::MultipleSubpopulation => 256, } @@ -2867,7 +2862,7 @@ fn retained_partition_count( if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( materialization.aggregation_type, - A::Increase | A::MultipleIncrease | A::Min | A::Max | A::MultipleMin | A::MultipleMax + A::Increase | A::Rate | A::Min | A::Max ) || !materialization.grouping_labels.names().is_empty() { @@ -3114,7 +3109,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -fn raw_time_series_input_contract( +pub fn raw_time_series_input_contract( expr: &QueryExpr, exact: bool, ) -> Result<(String, Option, String), String> { @@ -3310,7 +3305,7 @@ fn physical_aggregation( BackendAggregation { aggregation_id, metric_name: selected.metric.clone(), - family: physical_materialization_family(&selected.family), + family: selected.family.clone(), window_secs: selected.window_secs.unwrap_or(query.query_lookback_seconds), spatial_filter: selected.spatial_filter.clone(), grouping: selected @@ -3720,26 +3715,6 @@ fn collect_selected_materializations( Ok(selected) } -pub(crate) fn physical_materialization_family(family: &SummaryFamilyType) -> SummaryFamilyType { - match family { - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Count, _) => { - // The SummaryStore Sum accumulator retains the observation count - // alongside its sum. Both logical states can share this producer. - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Sum, - planner_types::post_asap::ExactParams::Sum, - ) - } - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Rate, _) => { - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, - ) - } - _ => family.clone(), - } -} - pub(super) fn sketch_params_json(params: &planner_types::post_asap::SketchParams) -> Value { use planner_types::post_asap::SketchParams as P; match params { @@ -4529,8 +4504,7 @@ pub(crate) mod tests { .find(|materialization| { matches!( materialization.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Rate ) }) .expect("reset-aware exact counter"); @@ -5101,8 +5075,7 @@ pub(crate) mod tests { .iter() .all(|m| !matches!( m.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + asap_types::AggregationType::Increase | asap_types::AggregationType::Rate ))); let entry = plan.query_plan.entries.values().next().unwrap(); assert!(!entry.materialization_bindings().is_empty()); @@ -5660,7 +5633,7 @@ pub(crate) mod tests { } #[test] - fn rate_and_increase_share_physical_counter_state() { + fn rate_and_increase_keep_planner_families_distinct() { let mut workload = request("rate", "rate(m[1m])"); workload .queries @@ -5669,10 +5642,14 @@ pub(crate) mod tests { .compile_promql(workload, environment(10_000)) .unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); for collector in &bundle.collector_plans { - assert_eq!(collector.materializations.len(), 1); - assert_eq!(collector.materializations[0].algorithm, "increase"); + let algorithms: std::collections::BTreeSet<_> = collector + .materializations + .iter() + .map(|materialization| materialization.algorithm.as_str()) + .collect(); + assert_eq!(algorithms, ["increase", "rate"].into()); } } @@ -5692,8 +5669,7 @@ pub(crate) mod tests { } #[test] - fn exact_dashboard_binds_sum_and_count_to_one_local_producer() { - // Both dashboard roots use one packed raw accumulator, with explicit readouts. + fn exact_dashboard_preserves_distinct_sum_and_count_producers() { let mut snapshot: BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -5709,7 +5685,7 @@ pub(crate) mod tests { entries.push(mean); let (request, env) = snapshot.into_physical_compilation_request().unwrap(); let bundle = DeploymentPlanCompiler.compile_promql(request, env).unwrap(); - assert_eq!(bundle.precompute_plan.materializations.len(), 1); + assert_eq!(bundle.precompute_plan.materializations.len(), 2); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { assert!( @@ -5719,7 +5695,7 @@ pub(crate) mod tests { )), "{entry:?}" ); - assert_eq!(entry.materialization_bindings().len(), 1); + assert!(!entry.materialization_bindings().is_empty()); } assert!(bundle .query_plan @@ -7484,8 +7460,8 @@ pub(crate) mod tests { assert_eq!( materialization.accumulator_spec().unwrap().family, SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Increase, - planner_types::post_asap::ExactParams::Increase, + planner_types::post_asap::ExactKind::Rate, + planner_types::post_asap::ExactParams::Rate, ) ); } diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index b76cdbf0..e1e29a4d 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -34,10 +34,7 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!( - m.aggregation_type, - AggregationType::Sum | AggregationType::MultipleSum - ) + || !matches!(m.aggregation_type, AggregationType::Sum) { continue; } diff --git a/control_plane/src/physical/post_asap/lower.rs b/control_plane/src/physical/post_asap/lower.rs index cdff8f35..aa572cdf 100644 --- a/control_plane/src/physical/post_asap/lower.rs +++ b/control_plane/src/physical/post_asap/lower.rs @@ -1,5 +1,4 @@ //! Query binding delegates selection to Planner's costed workload search. -//! Backend-specific rate normalization remains part of the physical binding. #![allow(dead_code)] @@ -118,41 +117,8 @@ fn bind_recursive( )) } - _ => { - let rewritten = rewrite_rate_to_increase(expr); - let node = crate::planner_selection::select_query(&rewritten, cost_model)?; - Ok(PostAsapPlan::Summary(node)) - } - } -} - -/// Rewrite Rate to Increase along the aggregate spine traversed by Planner. -/// This deployment computes rate by dividing the Increase readout by window -/// seconds, rather than storing a separate Rate accumulator. -fn rewrite_rate_to_increase(expr: &QueryExpr) -> QueryExpr { - match expr { - QueryExpr::Aggregate { - reduction, - measures: aggs, - output_names, - having, - child, - } => QueryExpr::Aggregate { - reduction: reduction.clone(), - measures: aggs - .iter() - .map(|intent| { - if matches!(intent, AggIntent::Rate) { - AggIntent::Increase - } else { - intent.clone() - } - }) - .collect(), - output_names: output_names.clone(), - having: having.clone(), - child: Rc::new(rewrite_rate_to_increase(child)), - }, - other => other.clone(), + _ => Ok(PostAsapPlan::Summary( + crate::planner_selection::select_query(expr, cost_model)?, + )), } } diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index e2eec8b9..070b6ea6 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -474,13 +474,9 @@ fn phase_b_pattern_only_temporal_sum_binds_to_exact_agg() { /// `ONLY_SPATIAL` — `sum by (host) (m)`. /// Control plane path: `Aggregate{Sum, by=[host]}` over a bare `Scan`. /// -/// The old locally-defined `AggregationType::MultipleSum` (keyed vs -/// unkeyed sum) identity no longer exists at the L4 IR level — -/// `SummaryKind::Sum` covers both; the keyed/unkeyed distinction now -/// lives on `SummaryAgg::by` (non-empty ⇒ the old "MultipleSum" shape), -/// per `emit::mod.rs`'s exact-accumulator classification notes. +/// Family remains Sum; the reduction carries the grouping columns. #[test] -fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { +fn phase_b_pattern_only_spatial_aggregate_binds_to_grouped_sum() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), // service column measures: vec![AggIntent::Sum { col: None }], @@ -501,7 +497,7 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { assert_eq!( reduction.group_keys().map(|k| k.keys()), Some(&[1][..]), - "keyed sum must carry the group-by column (the MultipleSum-equivalent signal)" + "Sum reduction must retain the group-by column" ); } other => panic!("expected SummaryAgg(Sum, by=[1]), got {other:?}"), @@ -511,14 +507,9 @@ fn phase_b_pattern_only_spatial_aggregate_binds_to_multiple_sum() { } /// `ONE_TEMPORAL_ONE_SPATIAL` — `sum by (host) (rate(m[5m]))`. -/// `bind_query_expr` (not `implement_tree` directly) rewrites -/// `AggIntent::Rate` to `AggIntent::Increase` before binding (see -/// `lower.rs`'s `rewrite_rate_to_increase` — this deployment's data -/// plane has no Rate accumulator). The old -/// `AggregationType::MultipleIncrease` identity is now -/// `SummaryKind::Increase` with a non-empty `by`. +/// Planner preserves the Rate family and the `by` reduction independently. #[test] -fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { +fn phase_b_pattern_temporal_and_spatial_combined_preserves_rate() { let expr = QueryExpr::Aggregate { reduction: Reduction::by(vec![1]), measures: vec![AggIntent::Rate], @@ -534,11 +525,11 @@ fn phase_b_pattern_temporal_and_spatial_combined_binds_to_multiple_increase() { } => { assert_eq!( family, - &SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase) + &SummaryFamilyType::ExactAggregate(ExactKind::Rate, ExactParams::Rate) ); assert_eq!(reduction.group_keys().map(|k| k.keys()), Some(&[1][..])); } - other => panic!("expected SummaryAgg(Increase, by=[1]), got {other:?}"), + other => panic!("expected SummaryAgg(Rate, by=[1]), got {other:?}"), }, other => panic!("expected Committed(Summary(_)), got {other:?}"), } @@ -671,12 +662,9 @@ fn phase_b_e2e_sum_by_preserves_grouping_label() { ); } -/// `rate_increase.yaml` — the legacy planner emits a MultipleIncrease -/// (counter-reset adjusted) row. Control plane path: `Aggregate{Rate}` over -/// `Window` → `bind_query_expr` rewrites `Rate` to `Increase` and binds an -/// exact accumulator (`SummaryAgg{Increase}`) — no approximate summary -/// family. Both paths produce a single non-summary streaming row; the L5 -/// emitter is the one that picks the actual MultipleIncrease processor. +/// A Rate query keeps Planner's exact Rate family through binding. The +/// physical emitter chooses the runtime processor without changing that +/// family identity. #[test] fn phase_b_e2e_rate_falls_through_to_logical() { let bound = pipeline_l1_to_l4( diff --git a/control_plane/src/physical/runtime_capability.rs b/control_plane/src/physical/runtime_capability.rs index 5b9f407c..7ebe2f03 100644 --- a/control_plane/src/physical/runtime_capability.rs +++ b/control_plane/src/physical/runtime_capability.rs @@ -101,7 +101,7 @@ pub enum Capability { /// * Sum-over-time requires archive execution because cumulative samples /// cannot be reconstructed from delta state alone. /// -/// Rate and Increase require the Increase capability; plain sum requires Sum. +/// Rate and Increase have distinct exact-family capabilities. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)] pub enum OuterFn { /// No range-style counter function in the expression — bare selector, @@ -275,8 +275,7 @@ impl Capability { /// §5/§8 Step 4) — via [`resolve_handle`], which picks a concrete /// per-family stand-in for the `Any` wildcard since `SketchAlgorithm` /// has no wildcard concept of its own; family-matching subsumes it. - /// `ExactAgg` is intentionally NOT routed through this path — see - /// [`multi_pop_satisfies_single`]'s doc for why. + /// Exact families require identity; keyed layout is checked separately. pub fn is_satisfied_by(&self, indexed: &Capability) -> bool { match (self, indexed) { (Capability::QuantileApprox(req), Capability::QuantileApprox(have)) => { @@ -313,22 +312,9 @@ impl Capability { SketchAlgorithm::CmsWithHeap, ) } - // Exact-aggregation family: the agg_type must match - // exactly OR be the single-pop ⇆ multi-pop equivalent. A - // `MultipleSum` policy can serve a `Sum` query by - // re-aggregating across keys; the `find_matching_policies` - // group_by ⊆ policy_grouping_labels check is what - // ultimately decides whether the re-aggregation is - // semantically valid. The reverse direction (single-pop - // serving multi-pop) is NOT allowed — the single-pop - // policy has lost the key dimension and can't recover it. - // - // Exact counter summaries are a distinct state contract. A sum of - // cumulative sample values cannot reconstruct reset correction or - // Prometheus boundary extrapolation. - (Capability::ExactAgg(req), Capability::ExactAgg(have)) => { - req == have || multi_pop_satisfies_single(*req, *have) - } + // Exact family identity must match. Grouping compatibility is + // checked separately by population routing. + (Capability::ExactAgg(req), Capability::ExactAgg(have)) => req == have, _ => false, } } @@ -348,30 +334,12 @@ fn sketch_algorithms_compatible( sketch_family_satisfied(required, available) } -/// True when `available` is the multi-population equivalent of -/// `required`'s single-population variant — i.e. a `MultipleSum` -/// policy can serve a `Sum` query (via re-aggregation across keys), -/// `MultipleIncrease` can serve `Increase`, `MultipleMax` can -/// serve `Max`. Asymmetric: this returns `false` for the reverse -/// direction (single-pop can't recover keys that have been collapsed -/// away). -fn multi_pop_satisfies_single(required: AggregationType, available: AggregationType) -> bool { - matches!( - (required, available), - (AggregationType::Sum, AggregationType::MultipleSum) - | (AggregationType::Increase, AggregationType::MultipleIncrease) - | (AggregationType::Min, AggregationType::MultipleMin) - | (AggregationType::Max, AggregationType::MultipleMax) - ) -} - // ── AggIntent → Capability bridge ──────────────────────────────────────────── #[cfg(test)] /// Map a semantic [`AggIntent`] to the ASAP-tier [`Capability`] that can -/// answer it. Returns `None` for intents that have no ASAP-tier sketch -/// (Sum / Min / Max / Avg / Rate / Increase / every archive-only intent -/// — see [`AggIntent::archive_only`]). +/// answer it. Returns `None` when no deployed ASAP-tier capability can +/// satisfy the intent. /// /// This is a runtime routing requirement, not a summary-selection rule. /// ASAPPlanner owns legal implementations and candidate enumeration; this @@ -395,15 +363,17 @@ pub fn capability_for(intent: &AggIntent) -> Option { } match intent { AggIntent::Sum { .. } => Some(Capability::ExactAgg(AggregationType::Sum)), + AggIntent::Count { accuracy } if is_exact(accuracy) => { + Some(Capability::ExactAgg(AggregationType::Count)) + } // Direction is part of the capability: a stored minimum cannot // answer `max_over_time` and vice versa, so these must not // collapse onto one `ExactAgg` the way they did while Planner // had a single `MinMax` accumulator. AggIntent::Min { .. } => Some(Capability::ExactAgg(AggregationType::Min)), AggIntent::Max { .. } => Some(Capability::ExactAgg(AggregationType::Max)), - AggIntent::Increase | AggIntent::Rate => { - Some(Capability::ExactAgg(AggregationType::Increase)) - } + AggIntent::Increase => Some(Capability::ExactAgg(AggregationType::Increase)), + AggIntent::Rate => Some(Capability::ExactAgg(AggregationType::Rate)), AggIntent::Quantile { accuracy, .. } if !is_exact(accuracy) => { Some(Capability::QuantileApprox(None)) } @@ -532,18 +502,14 @@ mod tests { } #[test] - fn capability_for_count_exact_routes_to_archive() { - // `count_over_time` lowers to `Count{accuracy:Exact}`. The - // PR #200/#201 follow-up briefly routed this to - // `ExactAgg(Sum)`, but the data plane has no count - // accumulator — `SumAccumulator` returns its `sum` for both - // `Statistic::Sum` and `Statistic::Count`, so the result was - // sum-of-values, not sample-count. Reverted to `None` (archive - // routing) until a real `SumCountAccumulator` lands. + fn capability_for_count_exact_preserves_count_family() { let intent = AggIntent::Count { accuracy: AccuracyTarget::Exact, }; - assert_eq!(capability_for(&intent), None); + assert_eq!( + capability_for(&intent), + Some(Capability::ExactAgg(AggregationType::Count)) + ); } #[test] @@ -593,16 +559,16 @@ mod tests { assert!(!Capability::ExactAgg(AggregationType::Max) .is_satisfied_by(&Capability::ExactAgg(AggregationType::Min))); assert!(!Capability::ExactAgg(AggregationType::Min) - .is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleMax))); + .is_satisfied_by(&Capability::ExactAgg(AggregationType::Max))); } #[test] - fn capability_for_rate_increase_route_to_exact_agg_increase() { - // PR-6 follow-up: Rate and Increase route to ASAP-tier - // ExactAgg(Increase) — the counter-reset-aware exact precompute. - // Pre-follow-up this returned `None`. + fn capability_for_rate_and_increase_preserves_family() { let exact_inc = Some(Capability::ExactAgg(AggregationType::Increase)); - assert_eq!(capability_for(&AggIntent::Rate), exact_inc); + assert_eq!( + capability_for(&AggIntent::Rate), + Some(Capability::ExactAgg(AggregationType::Rate)) + ); assert_eq!(capability_for(&AggIntent::Increase), exact_inc); } @@ -849,10 +815,6 @@ mod tests { AggregationType::Min, AggregationType::Max, AggregationType::DatasketchesKLL, - AggregationType::MultipleSum, - AggregationType::MultipleIncrease, - AggregationType::MultipleMin, - AggregationType::MultipleMax, AggregationType::HydraKLL, AggregationType::CountMinSketch, AggregationType::CountMinSketchWithHeap, @@ -873,21 +835,16 @@ mod tests { fn sum_family_cannot_impersonate_exact_counter_state() { let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum))); + assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); - let required_multi = Capability::ExactAgg(AggregationType::MultipleIncrease); - assert!( - !required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::MultipleSum)) - ); + let required_multi = Capability::ExactAgg(AggregationType::Increase); + assert!(!required_multi.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } #[test] fn is_satisfied_by_sum_family_does_not_answer_required_multi_increase_from_single_sum() { - // Same single/multi-population direction as multi_pop_satisfies_single: - // a single-pop available (Sum) can't serve a multi-pop required - // capability (MultipleIncrease) -- it already lost the per-key - // breakdown a multi-pop caller needs. - let required = Capability::ExactAgg(AggregationType::MultipleIncrease); + // A different exact family cannot supply counter state. + let required = Capability::ExactAgg(AggregationType::Increase); assert!(!required.is_satisfied_by(&Capability::ExactAgg(AggregationType::Sum))); } @@ -905,30 +862,26 @@ mod tests { // ── capability_for: ExactAgg dormancy ──────────────────────────────── #[test] - fn exact_agg_routing_covers_sum_rate_increase_only() { - // `Capability::ExactAgg` routing covers the three intents the - // data plane has a real accumulator for: `Sum` (SumAccumulator) - // and `Rate` / `Increase` (IncreaseAccumulator). + fn exact_agg_routing_keeps_sum_rate_and_increase_distinct() { assert_eq!( capability_for(&AggIntent::Sum { col: None }), Some(Capability::ExactAgg(AggregationType::Sum)) ); assert_eq!( capability_for(&AggIntent::Rate), - Some(Capability::ExactAgg(AggregationType::Increase)) + Some(Capability::ExactAgg(AggregationType::Rate)) ); assert_eq!( capability_for(&AggIntent::Increase), Some(Capability::ExactAgg(AggregationType::Increase)) ); - // `Count{Exact}` (count_over_time) and `Avg` both need a real - // count accumulator that doesn't exist yet — they route to - // archive until `SumCountAccumulator` lands. + // Exact count follows the Planner Count family; Avg still needs + // its own composition contract. assert_eq!( capability_for(&AggIntent::Count { accuracy: AccuracyTarget::Exact, }), - None + Some(Capability::ExactAgg(AggregationType::Count)) ); assert_eq!(capability_for(&AggIntent::Avg { col: None }), None); } diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index 6bb7fc32..d6b74cd4 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -18,8 +18,7 @@ use planner_types::pre_asap::AggIntent; /// `http_requests_total`, which the MVP demo's `mvp-workload.yaml` /// registers three times (entries 2/3/4 of [`deploy/configs/mvp-workload.yaml`]): /// * `sum by (zone) (http_requests_total)` → [`AggRole::Sum`] -/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Sum`] -/// (rate binds to ExactAgg(Sum)-shaped capability) +/// * `sum by (zone) (rate(http_requests_total[5m]))` → [`AggRole::Rate`] /// * `count(http_requests_total{zone="z0"})` → [`AggRole::Count`] /// /// Before this enum: the `WorkloadStore` was keyed by metric name alone @@ -30,7 +29,7 @@ use planner_types::pre_asap::AggIntent; /// shape). /// /// After: the store is keyed by `(metric, role)` so each shape gets its -/// own plan, its own `AggregationConfig` on the backend's streaming +/// own plan, its own `PrecomputeMaterialization` on the backend's streaming /// config, and its own routing-connector pipeline. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -39,15 +38,15 @@ pub enum AggRole { /// or workload entries with `sketch_family_override: DDSketch | KLL`. /// Routes to a quantile-shaped sketch (DDSketch / KLL). Quantile, - /// Bare counter selector, `sum(...)`, `sum_over_time(...)`, - /// `rate(...)`, `increase(...)`. All bind to ExactAgg(Sum)-shaped - /// capability on the data plane; the streaming-config emits an - /// `aggregation_type: Sum` rather than a sketch. + /// Bare selector or Sum-shaped exact aggregation. Sum, + /// Reset-aware per-second counter rate. + Rate, + /// Reset-aware counter increase over the selected window. + Increase, /// `count(...)`, `count_over_time(...)`, `count_distinct_over_time(...)`, /// or workload entries with `sketch_family_override: HLL`. Routes - /// to HLL when a sketch is appropriate, otherwise to a Sum-as-count - /// exact-aggregation. + /// to HLL when a sketch is appropriate, otherwise to exact Count. Count, /// `topk(...)`, or workload entries with /// `sketch_family_override: CountSketch | CountMinSketch`. Routes @@ -70,6 +69,8 @@ impl AggRole { match self { AggRole::Quantile => "quantile", AggRole::Sum => "sum", + AggRole::Rate => "rate", + AggRole::Increase => "increase", AggRole::Count => "count", AggRole::Topk => "topk", AggRole::Other => "other", @@ -97,15 +98,16 @@ impl std::fmt::Display for AggRole { /// through the same canonical pipeline the live serving path uses /// (`query_parser::parse_query_expr_canonical` → /// `asap_tier_analysis::collect_agg_intents`), and the OUTERMOST -/// intent (the one bound to the data-plane capability) is matched: +/// intent is matched, except that a Sum wrapping a counter function +/// keeps the inner Rate or Increase role: /// * [`AggIntent::Quantile`] → [`AggRole::Quantile`] /// * [`AggIntent::TopK`] → [`AggRole::Topk`] /// * [`AggIntent::Cardinality`], [`AggIntent::Count`], or the /// windowed-Count-as-Frequency extension /// (`intent_algebra::as_frequency`) → [`AggRole::Count`] /// * [`AggIntent::Sum`], [`AggIntent::Rate`], [`AggIntent::Increase`] -/// → [`AggRole::Sum`] -/// * Anything else recognised but not one of the four shapes above +/// → their respective roles +/// * Anything else recognised but not one of the listed shapes above /// (`Min`/`Max`/`Avg`/`StdDev`/histogram accessors/…) → /// [`AggRole::Other`]. /// * Bare metric selector (no `Aggregate` node at all) → @@ -125,11 +127,7 @@ impl std::fmt::Display for AggRole { /// semantics, whichever the lowerer picks). The Sum-shaped /// alternative is rare in practice; users who want it write /// `sum_over_time(count(...))` which classifies as Sum. -/// * `rate` / `irate` / `increase` — Sum. `irate` folds onto -/// `AggIntent::Rate` at L3 same as `rate`; both bind to -/// ExactAgg(Increase) on the data plane (see -/// `data_plane/src/precompute_engine/ingest_handler.rs`'s handling -/// of `AggKind::ExactAgg { Increase }`). +/// * `irate` currently folds onto `AggIntent::Rate` in the frontend. pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { // 1. `sketch_family_override` wins. if let Some(family) = entry.sketch_family_override.as_ref() { @@ -167,11 +165,30 @@ pub fn derive_agg_role(entry: &WorkloadEntry) -> AggRole { if crate::planner_selection::as_frequency(outer).is_some() { return AggRole::Count; } + // A spatial `sum by (...)` around a counter function still requires the + // counter family's state; using Sum as the registration key would let it + // overwrite a bare Sum workload for the same metric. + if matches!(outer, AggIntent::Sum { .. }) { + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Rate)) + { + return AggRole::Rate; + } + if intents + .iter() + .any(|intent| matches!(intent, AggIntent::Increase)) + { + return AggRole::Increase; + } + } match outer { AggIntent::Quantile { .. } => AggRole::Quantile, AggIntent::TopK { .. } => AggRole::Topk, AggIntent::Cardinality { .. } | AggIntent::Count { .. } => AggRole::Count, - AggIntent::Sum { .. } | AggIntent::Rate | AggIntent::Increase => AggRole::Sum, + AggIntent::Sum { .. } => AggRole::Sum, + AggIntent::Rate => AggRole::Rate, + AggIntent::Increase => AggRole::Increase, _ => AggRole::Other, } } @@ -970,13 +987,7 @@ mod tests { #[test] fn agg_role_sum_query_strings() { - for q in [ - "sum by (zone) (m)", - "sum_over_time(m[5m])", - "rate(m[5m])", - "increase(m[5m])", - "sum by (zone) (rate(m[5m]))", - ] { + for q in ["sum by (zone) (m)", "sum_over_time(m[5m])"] { assert_eq!( derive_agg_role(&entry("m", Some(q), None)), AggRole::Sum, @@ -985,6 +996,18 @@ mod tests { } } + #[test] + fn counter_functions_have_distinct_workload_roles() { + for (query, expected) in [ + ("rate(m[5m])", AggRole::Rate), + ("sum by (zone) (rate(m[5m]))", AggRole::Rate), + ("increase(m[5m])", AggRole::Increase), + ("sum by (zone) (increase(m[5m]))", AggRole::Increase), + ] { + assert_eq!(derive_agg_role(&entry("m", Some(query), None)), expected); + } + } + #[test] fn agg_role_count_query_strings() { for q in [ @@ -1095,7 +1118,7 @@ mod tests { } #[test] - fn three_synthetic_http_requests_total_entries_classify_to_two_distinct_roles() { + fn three_synthetic_http_requests_total_entries_keep_distinct_roles() { // Synthetic mirror of `deploy/configs/mvp-workload.yaml` // entries 2/3/4 — proves `derive_agg_role` produces distinct // roles for the three http_requests_total shapes. Pre-B2 the @@ -1120,15 +1143,8 @@ mod tests { ), ]; let roles: Vec = entries.iter().map(derive_agg_role).collect(); - assert_eq!(roles, vec![AggRole::Sum, AggRole::Sum, AggRole::Count]); - // The store distinguishes Sum vs Count keys, so two of the - // three entries (the two Sum-shaped ones) still collide - // under (metric, role). That's the documented behaviour — - // two YAML entries with the SAME (metric, role) overwrite, - // which is the legitimate "operator updated their workload" - // path. The fix scope is collisions across DIFFERENT shapes, - // not idempotent re-registers. + assert_eq!(roles, vec![AggRole::Sum, AggRole::Rate, AggRole::Count]); let distinct: std::collections::HashSet<_> = roles.iter().copied().collect(); - assert_eq!(distinct.len(), 2, "Sum + Count = 2 distinct roles"); + assert_eq!(distinct.len(), 3); } } diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index 482ef6d6..bdf5949b 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,68 +1,12 @@ -//! Typed accumulator dispatch derived from legacy streaming config. +//! Validate stored materialization descriptors against Planner summary families. //! -//! The semantic identity is ASAPPlanner's [`SummaryFamilyType`]. This module -//! only adds the backend execution concern of keyed versus unkeyed state and -//! adapts the stable legacy wire fields into that canonical representation. -//! -//! ## This is an additive representation, not a replacement (yet) -//! -//! `AggregationConfig` keeps its `aggregation_type` / `aggregation_sub_type` -//! / `parameters` fields untouched. Two hard constraints ruled out full -//! removal in this pass: -//! -//! 1. **`PolicyFingerprint` hash stability.** [`crate::policy_fingerprint`] -//! hashes `aggregation_type` / `aggregation_sub_type` / `parameters` -//! directly, and its own module doc is explicit that the byte layout -//! it produces is a stability *contract* ("Don't reorder fields... -//! any such change invalidates every deployed fingerprint and forces -//! a cold-start rebuild"). Changing what feeds that hash — even by -//! routing it through an equivalent typed shape — risks producing a -//! different byte sequence for the same logical policy, which strands -//! on-disk sids after a deploy. `policy_fingerprint.rs` is -//! deliberately **not touched** by this module; it keeps reading the -//! original three fields, unchanged. -//! 2. **Consumer fan-out.** `AggregationType` is read by ~40 files across -//! `data_plane` and `asap_types` — persistence (`sid_metadata.json` -//! round-trip), query-time capability matching -//! (`capability_matching.rs`, unrelated to accumulator dispatch), -//! the query engine, reconciliation, index maintenance — not just -//! `accumulator_factory.rs` (the single highest-risk consumer, and -//! the one this module targets). Migrating all of them in one PR was -//! judged too large to land and review safely; that's tracked as -//! follow-up, not done here. -//! -//! So: `AccumulatorSpec` is *computed from* `AggregationConfig`'s -//! existing fields via [`AggregationConfig::accumulator_spec`], and -//! consumed by `data_plane::precompute_engine::accumulator_factory` -//! instead of the raw fields. The wire format (`aggregationType` / -//! `aggregationSubType` / `parameters` JSON/YAML keys) is completely -//! unaffected — nothing here changes how `AggregationConfig::from_yaml` -//! / `from_json` parse or how `serialize_to_json` emits. -//! -//! Backend-specific execution details remain deliberately separate: -//! -//! - **Min/max direction.** Direction is part of the family now, not a -//! string riding alongside it: `AggregationType::{Min, Max}` (and the -//! keyed `{MultipleMin, MultipleMax}`) map to `ExactKind::Min` and -//! `ExactKind::Max` respectively — upstream still spells its -//! maximum accumulator `MinMax`, but it is a maximum. Nothing reads -//! `AggregationConfig::aggregation_sub_type` for the direction any -//! more, so a min state can no longer content-address onto a max one. -//! - **HydraKLL's `(row, col)` tiling.** `SketchParams::Kll` carries -//! only `k` — upstream has no concept of the CMS-like grid-of-KLL-cells -//! layout `HydraKllSketchAccumulator` uses to parallelize a keyed KLL -//! across many populations. `accumulator_factory.rs` calls -//! [`cms_params`] directly for keyed KLL execution -//! arm, same extraction the plain CMS arms use, because `w`/`d` are -//! genuinely the same wire keys for both. -//! - **Top-k ranking mode (`weight_mode`).** Not a sketch structural -//! parameter — a data_plane-only "what to accumulate" axis -//! (`accumulator_factory::TopkWeight`) with no upstream equivalent. -//! Stays a raw-`parameters`-reading helper in `accumulator_factory.rs`. +//! This projection supports catalog identity and imported state metadata. It is +//! not an execution program. Raw and maintenance execution dispatch directly +//! on the selected post-ASAP DAG payload; the descriptor must agree with it. use serde_json::Value; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::key_by_label_names::KeyByLabelNames; use crate::AggregationType; use planner_types::post_asap::{ @@ -75,8 +19,8 @@ use planner_types::post_asap::{ /// accumulator to run (`kind`), with what tuning (`params`), and /// whether it's keyed by a group-by label set (`grouping`). /// -/// Computed on demand from an [`AggregationConfig`] via -/// [`AggregationConfig::accumulator_spec`] — not stored on the config +/// Computed on demand from an [`PrecomputeMaterialization`] via +/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config /// itself, so there is exactly one source of truth for the fields that /// feed [`crate::policy_fingerprint::PolicyFingerprint`]. #[derive(Debug, Clone, PartialEq)] @@ -85,10 +29,7 @@ pub struct AccumulatorSpec { /// validated `SketchKind` (category + algorithm + params), following the /// ASAP-aware-mapping vocabulary. pub family: SummaryFamilyType, - /// `Some(labels)` for a keyed (multi-population) accumulator, - /// `None` for a single-population one. This is the axis - /// `AggregationType` wrongly folded into identity (`Sum` vs - /// `MultipleSum`) — here it's a sibling field instead. + /// Physical keyed-state layout, independent of semantic family. pub grouping: Option, } @@ -96,8 +37,8 @@ pub struct AccumulatorSpec { /// /// This is execution semantics, separate from the summary family: the same /// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Legacy streaming artifacts still encode the rule in -/// `parameters`; callers use [`AggregationConfig::sample_update_rule`] so the +/// counter deltas. Stored descriptors encode the rule in +/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the /// runtime does not branch on ad-hoc strings. #[derive(Debug, Clone, Copy, PartialEq)] pub enum SampleUpdateRule { @@ -134,7 +75,7 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) } -impl AggregationConfig { +impl PrecomputeMaterialization { pub fn sample_update_rule(&self) -> SampleUpdateRule { let scale = self .parameters @@ -157,23 +98,14 @@ impl AggregationConfig { } } -/// Why [`AggregationConfig::accumulator_spec`] couldn't resolve a config -/// into an [`AccumulatorSpec`]. Each variant matches one of the three -/// distinct fallback paths `accumulator_factory::create_accumulator_updater` -/// took pre-Step-5 — preserved verbatim (including which default -/// updater and which warning text each one produced) so this refactor -/// changes *how* the dispatch is expressed, not what it does for any -/// input. +/// A storage descriptor cannot be resolved to a supported Planner family. #[derive(Debug, Clone, PartialEq, Eq)] pub enum AccumulatorSpecError { /// `aggregation_type` was `SingleSubpopulation` with an /// `aggregation_sub_type` string not in the recognized alias list. - /// Pre-Step-5 this defaulted to `SumAccumulatorUpdater`. UnknownSingleSubpopulationSubType(String), /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. Pre-Step-5 this defaulted - /// to `MultipleSumAccumulatorUpdater` (note: a *different* default - /// than the `SingleSubpopulation` case). + /// unrecognized `aggregation_sub_type`. UnknownMultipleSubpopulationSubType(String), /// `aggregation_type` itself has no accumulator-dispatch mapping. /// Also returned for an invalid HLL precision. A resolved family identifies @@ -185,36 +117,24 @@ impl std::fmt::Display for AccumulatorSpecError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { Self::UnknownSingleSubpopulationSubType(s) => { - write!( - f, - "Unknown SingleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown SingleSubpopulation sub_type '{s}'") } Self::UnknownMultipleSubpopulationSubType(s) => { - write!( - f, - "Unknown MultipleSubpopulation sub_type '{s}', defaulting to Sum" - ) + write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") } - Self::UnmappedAggregationType(t) => write!( - f, - "Unknown aggregation_type '{t:?}', defaulting to SingleSubpopulation Sum" - ), + Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), } } } impl std::error::Error for AccumulatorSpecError {} -impl AggregationConfig { +impl PrecomputeMaterialization { /// Resolve this config's `(aggregation_type, aggregation_sub_type, /// parameters)` triple into a typed [`AccumulatorSpec`]. /// - /// Mirrors `accumulator_factory::create_accumulator_updater`'s - /// pre-Step-5 dispatch exactly — same sub_type alias lists, same - /// numeric defaults, same three fallback paths (see - /// [`AccumulatorSpecError`]) — just re-expressed as data instead of - /// as a 14-arm match baked into the accumulator constructor. + /// Unsupported descriptors return an error; this projection never chooses + /// a fallback family and cannot authorize DAG execution. pub fn accumulator_spec(&self) -> Result { use AggregationType::*; @@ -227,21 +147,9 @@ impl AggregationConfig { ) }; let (family, keyed) = match self.aggregation_type { - Sum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - Increase => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - Min => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - Max => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, + Sum | Count | Increase | Rate | Min | Max => ( + self.aggregation_type.planner_exact_family().unwrap(), + !self.aggregated_labels.is_empty(), ), DatasketchesKLL => ( independent_sketch( @@ -252,22 +160,6 @@ impl AggregationConfig { ), false, ), - MultipleSum => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - MultipleIncrease => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - MultipleMin => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - MultipleMax => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), HydraKLL => { let k = kll_k_param(self) as u32; ( @@ -497,7 +389,7 @@ impl AggregationConfig { /// Extract the KLL `k` parameter. Capital `"K"` takes precedence over /// lowercase `"k"` to match the convention used by the top-level /// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &AggregationConfig) -> u16 { +pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { config .parameters .get("K") @@ -513,7 +405,7 @@ pub fn kll_k_param(config: &AggregationConfig) -> u16 { /// matches what the control plane's `sketch_params_to_json` emits and /// what `sketch_config_to_params` uses for OTLP policy_fp content /// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { +pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { let row_num = config .parameters .get("d") @@ -530,7 +422,7 @@ pub fn cms_params(config: &AggregationConfig) -> (usize, usize) { /// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` /// from `parameters`; defaults to 20 (the heap holds the top-k /// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &AggregationConfig) -> usize { +pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { config .parameters .get("heap_size") @@ -545,7 +437,7 @@ pub fn heap_size_param(config: &AggregationConfig) -> usize { /// Pull `relativeAccuracy` (or canonical aliases) out of a /// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, /// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { +pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { let parsed = param_f64(config, "relativeAccuracy") .or_else(|| param_f64(config, "relative_accuracy")) .or_else(|| param_f64(config, "alpha")) @@ -561,7 +453,7 @@ pub fn ddsketch_alpha_param(config: &AggregationConfig) -> f64 { } } -fn param_f64(config: &AggregationConfig, key: &str) -> Option { +fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { config.parameters.get(key).and_then(Value::as_f64) } @@ -599,8 +491,8 @@ mod tests { sub_type: &str, params: HashMap, grouping_labels: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), params, @@ -630,13 +522,9 @@ mod tests { } #[test] - fn multiple_sum_is_keyed_sum() { - let cfg = make_config( - AggregationType::MultipleSum, - "", - HashMap::new(), - vec!["zone"], - ); + fn keyed_layout_preserves_sum_family() { + let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); + cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); let spec = cfg.accumulator_spec().expect("resolves"); assert_exact(&spec, ExactKind::Sum); assert_eq!( @@ -862,16 +750,16 @@ mod tests { assert_eq!( AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown SingleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus', defaulting to Sum" + "Unknown MultipleSubpopulation sub_type 'Bogus'" ); assert_eq!( AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL', defaulting to SingleSubpopulation Sum" + "Unknown aggregation_type 'HLL'" ); } @@ -905,7 +793,7 @@ mod tests { // ---- PolicyFingerprint stability guard --------------------------- /// `accumulator_spec()` must be a pure, additional *read* of - /// `AggregationConfig` — it must not change what + /// `PrecomputeMaterialization` — it must not change what /// `PolicyFingerprint::from_config` hashes. This locks in a fixed /// fingerprint for a fixed config as a tripwire: if this test ever /// needs its expected constant updated, `policy_fingerprint.rs` diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index 92dfd41d..f4fe8699 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -87,8 +87,9 @@ impl WindowMaterializationLayout { } } -/// Per-aggregation policy with content-derived [`PolicyFingerprint`] identity. -/// An `aggregationId` field in input YAML is ignored for compatibility. +/// Physical materialization metadata with content-derived identity. +/// This descriptor cannot authorize execution: the enclosing PrecomputePlan +/// must bind it to a compatible Planner DAG producer. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputeMaterialization { pub aggregation_type: AggregationType, @@ -196,10 +197,6 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} -/// Compatibility name for legacy streaming-config and precompute call sites. -/// New CompiledPhysicalPlan code should use [`PrecomputeMaterialization`]. -pub type AggregationConfig = PrecomputeMaterialization; - impl PrecomputeMaterialization { pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection @@ -338,7 +335,7 @@ impl PrecomputeMaterialization { /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s - /// `HashMap` keys). **Always** equal to + /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. pub fn policy_fp_u64(&self) -> u64 { @@ -818,12 +815,18 @@ mod tests { /// SAME config as a fixture without it. #[test] fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = - AggregationConfig::from_yaml_data(&sample_yaml(true), None, QueryLanguage::PromQl) - .expect("parse ok"); - let without = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse ok"); + let with = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(true), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); + let without = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse ok"); assert_eq!( with.policy_fingerprint(), without.policy_fingerprint(), @@ -834,10 +837,18 @@ mod tests { /// Round-tripping the same content yields the same fingerprint. #[test] fn fingerprint_is_deterministic_per_content() { - let a = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse a"); - let b = AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse b"); + let a = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse a"); + let b = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse b"); assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); assert_ne!( a.policy_fingerprint().as_u64(), @@ -862,37 +873,44 @@ mod tests { ] { yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); let config = - AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) + .unwrap(); assert_eq!(config.window_layout, layout); let mut wire = config.serialize_to_json(); wire["groupingLabels"] = serde_json::to_value(&config.grouping_labels).unwrap(); wire["aggregatedLabels"] = serde_json::to_value(&config.aggregated_labels.labels).unwrap(); wire["rollupLabels"] = serde_json::to_value(&config.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&wire).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); assert_eq!(decoded.window_layout, layout); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); wire["window_layout"] = wire["windowLayout"].clone(); - assert!(AggregationConfig::deserialize_from_json(&wire).is_err()); + assert!(PrecomputeMaterialization::deserialize_from_json(&wire).is_err()); } yaml.as_mapping_mut() .unwrap() .remove(serde_yaml::Value::from("windowLayout")); - let legacy = AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); + let legacy = + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); assert_eq!( legacy.window_layout, WindowMaterializationLayout::Pane { pane_secs: 10 } ); yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!(AggregationConfig::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err()); + assert!( + PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() + ); } #[test] fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let mut epoch = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let unknown = epoch.policy_fingerprint(); epoch.pane_origin_ms = Some(7_000); let planned = epoch.policy_fingerprint(); @@ -910,13 +928,13 @@ mod tests { .as_object_mut() .unwrap() .insert("paneOriginMs".into(), origin); - let decoded: AggregationConfig = serde_json::from_value(derived.clone()).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(derived.clone()).unwrap(); assert_eq!(decoded.pane_origin_ms, Some(7_000)); let mut legacy = derived; legacy.as_object_mut().unwrap().remove("paneOriginMs"); assert_eq!( - serde_json::from_value::(legacy) + serde_json::from_value::(legacy) .expect("decode legacy wire") .pane_origin_ms, None @@ -926,18 +944,24 @@ mod tests { /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. #[test] fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); } /// PR 5: `serialize_to_json` no longer emits `aggregationId`. #[test] fn serialize_to_json_omits_aggregation_id() { - let cfg = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .expect("parse"); + let cfg = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .expect("parse"); let json = cfg.serialize_to_json(); assert!( json.get("aggregationId").is_none(), @@ -949,9 +973,12 @@ mod tests { fn typed_projection_roundtrips_and_legacy_column_keeps_identity() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = - AggregationConfig::from_yaml_data(&sample_yaml(false), None, QueryLanguage::PromQl) - .unwrap(); + let mut config = PrecomputeMaterialization::from_yaml_data( + &sample_yaml(false), + None, + QueryLanguage::PromQl, + ) + .unwrap(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -960,7 +987,7 @@ mod tests { let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); - let decoded: AggregationConfig = serde_json::from_value(legacy).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(legacy).unwrap(); assert_eq!(decoded.policy_fingerprint(), column_identity); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), @@ -977,8 +1004,8 @@ mod tests { "rollup": config.rollup_labels.serialize_to_json(), }); assert!(wire.get("valueColumn").is_none()); - let json = AggregationConfig::deserialize_from_json(&wire).unwrap(); - let yaml = AggregationConfig::from_yaml_data( + let json = PrecomputeMaterialization::deserialize_from_json(&wire).unwrap(); + let yaml = PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(&wire).unwrap(), None, QueryLanguage::ClickHouseSql, @@ -994,8 +1021,8 @@ mod tests { ); let mut conflicting = wire; conflicting["valueColumn"] = serde_json::json!("other_column"); - assert!(AggregationConfig::deserialize_from_json(&conflicting).is_err()); - assert!(AggregationConfig::from_yaml_data( + assert!(PrecomputeMaterialization::deserialize_from_json(&conflicting).is_err()); + assert!(PrecomputeMaterialization::from_yaml_data( &serde_yaml::to_value(conflicting).unwrap(), None, QueryLanguage::ClickHouseSql diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ccdcbec0..647f7604 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -14,15 +14,13 @@ use std::str::FromStr; pub enum AggregationType { // ---------- single-population (non-keyed) ---------- Sum, + Count, Increase, + Rate, Min, Max, DatasketchesKLL, // ---------- multi-population (keyed) ---------- - MultipleSum, - MultipleIncrease, - MultipleMin, - MultipleMax, HydraKLL, CountMinSketch, CountMinSketchWithHeap, @@ -38,17 +36,31 @@ pub enum AggregationType { } impl AggregationType { + /// Adapt a storage/processor tag to Planner's exact family. Keyed storage + /// changes the payload layout, not the semantic family. + pub fn planner_exact_family(self) -> Option { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + _ => return None, + }; + Some(SummaryFamilyType::ExactAggregate(kind, params)) + } + pub fn as_str(self) -> &'static str { match self { AggregationType::Sum => "Sum", + AggregationType::Count => "Count", AggregationType::Increase => "Increase", + AggregationType::Rate => "Rate", AggregationType::Min => "Min", AggregationType::Max => "Max", AggregationType::DatasketchesKLL => "DatasketchesKLL", - AggregationType::MultipleSum => "MultipleSum", - AggregationType::MultipleIncrease => "MultipleIncrease", - AggregationType::MultipleMin => "MultipleMin", - AggregationType::MultipleMax => "MultipleMax", AggregationType::HydraKLL => "HydraKLL", AggregationType::CountMinSketch => "CountMinSketch", AggregationType::CountMinSketchWithHeap => "CountMinSketchWithHeap", @@ -67,10 +79,6 @@ impl AggregationType { matches!( self, AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax | AggregationType::CountMinSketch | AggregationType::CountMinSketchWithHeap | AggregationType::CountSketch @@ -93,14 +101,12 @@ impl FromStr for AggregationType { match s { // Canonical names "Sum" => Ok(AggregationType::Sum), + "Count" => Ok(AggregationType::Count), "Increase" => Ok(AggregationType::Increase), + "Rate" => Ok(AggregationType::Rate), "Min" => Ok(AggregationType::Min), "Max" => Ok(AggregationType::Max), "DatasketchesKLL" => Ok(AggregationType::DatasketchesKLL), - "MultipleSum" => Ok(AggregationType::MultipleSum), - "MultipleIncrease" => Ok(AggregationType::MultipleIncrease), - "MultipleMin" => Ok(AggregationType::MultipleMin), - "MultipleMax" => Ok(AggregationType::MultipleMax), "HydraKLL" => Ok(AggregationType::HydraKLL), "CountMinSketch" => Ok(AggregationType::CountMinSketch), "CountMinSketchWithHeap" => Ok(AggregationType::CountMinSketchWithHeap), @@ -121,12 +127,6 @@ impl FromStr for AggregationType { "DatasketchesKLLAccumulator" | "KLL" | "kll" | "datasketches_kll" => { Ok(AggregationType::DatasketchesKLL) } - "MultipleSumAccumulator" | "multiple_sum" => Ok(AggregationType::MultipleSum), - "MultipleIncreaseAccumulator" | "multiple_increase" => { - Ok(AggregationType::MultipleIncrease) - } - "MultipleMinAccumulator" | "multiple_min" => Ok(AggregationType::MultipleMin), - "MultipleMaxAccumulator" | "multiple_max" => Ok(AggregationType::MultipleMax), "HydraKllSketchAccumulator" | "hydra_kll" => Ok(AggregationType::HydraKLL), "CountMinSketchAccumulator" | "CMS" | "cms" | "count_min_sketch" => { Ok(AggregationType::CountMinSketch) @@ -149,7 +149,7 @@ impl FromStr for AggregationType { | "MultipleMinMaxAccumulator" | "multiple_min_max" => Err(format!( "Retired aggregation type: '{s}' -- min and max are separate types now, \ - use 'Min'/'Max' (or 'MultipleMin'/'MultipleMax')" + use 'Min'/'Max'" )), _ => Err(format!("Unknown aggregation type: '{s}'")), } @@ -168,3 +168,48 @@ impl<'de> Deserialize<'de> for AggregationType { s.parse().map_err(serde::de::Error::custom) } } + +#[cfg(test)] +mod tests { + use super::*; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + /// Removed layout tags cannot be installed as semantic families. + #[test] + fn rejects_keyed_family_aliases() { + for name in [ + "MultipleSum", + "MultipleIncrease", + "MultipleMin", + "MultipleMax", + ] { + assert!(name.parse::().is_err(), "{name}"); + } + } + + #[test] + fn storage_layout_tags_do_not_create_planner_families() { + for (storage, expected) in [ + (AggregationType::Sum, ExactKind::Sum), + (AggregationType::Count, ExactKind::Count), + (AggregationType::Increase, ExactKind::Increase), + (AggregationType::Rate, ExactKind::Rate), + ] { + let family = storage.planner_exact_family().unwrap(); + assert!( + matches!(family, SummaryFamilyType::ExactAggregate(kind, _) if kind == expected) + ); + } + assert_eq!( + AggregationType::Rate.planner_exact_family(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Rate, + ExactParams::Rate + )) + ); + assert_ne!( + AggregationType::Rate.planner_exact_family(), + AggregationType::Increase.planner_exact_family() + ); + } +} diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index deb7fe3f..5cd902b7 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -2,7 +2,7 @@ //! //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` -//! (`AggregationConfig::grouping_labels`, `PolicyFingerprint`, +//! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, //! `PolicyRegistry`, `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index 535ec3ef..a22352be 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -44,7 +44,7 @@ impl MonitorFunctional { /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both -/// sides can reach — same reasoning as `AggregationConfig`/`PolicyFingerprint`. +/// sides can reach — same reasoning as `PrecomputeMaterialization`/`PolicyFingerprint`. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct MonitorSpec { pub agg_id: u64, diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index ca7353aa..b68e385c 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -4,7 +4,7 @@ //! the controller-allocated `aggregation_id: u64`. Where `aggregation_id` //! is a counter the control plane mints and ships in the streaming-config //! YAML, `PolicyFingerprint` is derived deterministically from the -//! `AggregationConfig`'s content — so two control planes producing the +//! `PrecomputeMaterialization`'s content — so two control planes producing the //! same policy independently produce the same fingerprint, and the data //! plane can index without a separate id allocation. //! @@ -14,7 +14,7 @@ //! grouping_labels, aggregated_labels, rollup_labels, window_size, //! slide_interval, window_type, pane_origin_ms, spatial_filter_normalized)` //! -//! The hash includes **every** field of `AggregationConfig` that +//! The hash includes **every** field of `PrecomputeMaterialization` that //! determines what the policy does — sketch / exact-agg shape, //! group-by + rollup layout, window cadence, spatial filter. Two //! configs that compare equal on these dimensions produce the same @@ -50,9 +50,9 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use xxhash_rust::xxh64::xxh64; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; -/// Stable, content-addressed handle for an `AggregationConfig`. +/// Stable, content-addressed handle for an `PrecomputeMaterialization`. /// /// Wrap a `u64` so callers can't accidentally swap a `PolicyFingerprint` /// with an `aggregation_id` — they're both u64-shaped but they index @@ -75,14 +75,14 @@ impl PolicyFingerprint { } impl PolicyFingerprint { - /// Compute the fingerprint of an [`AggregationConfig`]. + /// Compute the fingerprint of an [`PrecomputeMaterialization`]. /// /// Hash inputs are concatenated with `\0` byte separators and /// canonicalized so that map/iteration order can't affect the /// outcome. Parameter values are rendered via `serde_json::to_string` /// for nested-shape determinism (matches the existing /// `parameters_canonical` form used in `AggKind::ExactAgg`). - pub fn from_config(cfg: &AggregationConfig) -> Self { + pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { let mut buf: Vec = Vec::with_capacity(512); if !cfg.population_key_encoding.is_legacy() { @@ -265,8 +265,8 @@ mod tests { group_by: Vec<&str>, window_size: u64, spatial_filter: &str, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -298,7 +298,7 @@ mod tests { ); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); let mut canonical = legacy.clone(); @@ -306,7 +306,7 @@ mod tests { assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); - let decoded: AggregationConfig = serde_json::from_value(wire).unwrap(); + let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); use crate::traits::SerializableToSink; let mut sink = canonical.serialize_to_json(); @@ -315,7 +315,7 @@ mod tests { sink["aggregatedLabels"] = serde_json::to_value(&canonical.aggregated_labels.labels).unwrap(); sink["rollupLabels"] = serde_json::to_value(&canonical.rollup_labels.labels).unwrap(); - let decoded = AggregationConfig::deserialize_from_json(&sink).unwrap(); + let decoded = PrecomputeMaterialization::deserialize_from_json(&sink).unwrap(); assert_eq!( decoded.population_key_encoding, canonical.population_key_encoding @@ -462,7 +462,7 @@ mod tests { ); } - /// Pre-PR-5 the `aggregation_id` field on `AggregationConfig` was + /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was /// excluded from the fingerprint hash. PR 5 deletes the field /// entirely — identity *is* the fingerprint — so this is now /// vacuously true. Kept as a doc-comment anchor; no runtime test @@ -525,7 +525,7 @@ mod tests { fn spatial_filter_canonicalization_drives_fingerprint() { // Two filters that differ only in matcher ordering produce the // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `AggregationConfig::new` (via + // canonicalization step in `PrecomputeMaterialization::new` (via // `normalize_spatial_filter`) sorts matchers by key. let a = cfg( "http_lat", diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index 4b4f9e95..dde5e5cc 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -1,7 +1,7 @@ //! Content-addressed policy registry. //! -//! Derived view over a collection of `AggregationConfig`s that maps -//! [`PolicyFingerprint`] → [`AggregationConfig`]. This is the +//! Derived view over a collection of `PrecomputeMaterialization`s that maps +//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated //! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` //! carries (see `data_plane::storage_engines::types::streaming_config`'s @@ -20,7 +20,7 @@ //! //! ## Identity invariants //! -//! Two `AggregationConfig`s that produce the same `PolicyFingerprint` +//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` //! ARE the same policy. The registry treats this as a *deduplication* //! invariant — if two distinct entries in the source `materializations_by_policy_fingerprint` //! map produce the same fingerprint, the later one wins (last-write @@ -30,13 +30,13 @@ use std::collections::HashMap; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; /// Content-addressed lookup table for active aggregation policies. #[derive(Debug, Clone, Default)] pub struct PolicyRegistry { - policies: HashMap, + policies: HashMap, } impl PolicyRegistry { @@ -46,7 +46,7 @@ impl PolicyRegistry { /// them. pub fn from_configs(configs: I) -> Self where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); for cfg in configs { @@ -63,7 +63,7 @@ impl PolicyRegistry { /// surfacing. pub fn from_configs_with_collisions(configs: I) -> (Self, usize) where - I: IntoIterator, + I: IntoIterator, { let mut policies = HashMap::new(); let mut collisions = 0usize; @@ -77,12 +77,12 @@ impl PolicyRegistry { } /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.policies.get(&fp) } /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { + pub fn iter(&self) -> impl Iterator { self.policies.iter() } @@ -110,11 +110,11 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(_id: u64, metric: &str) -> AggregationConfig { + fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is derived from // content. Kept as a parameter so existing call sites in the // tests below don't churn. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index a6375da3..47f30192 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -101,11 +101,10 @@ pub struct PlanEnvelope { pub capability_snapshot_id: String, } -/// Backend-side materialization projection consumed by the streaming -/// precompute engine. This is deliberately config-driven: it contains no -/// PromQL string or ad-hoc scheduler job. The aggregation definitions are -/// emitted to `/api/v1/streaming-config`, where the runtime matches incoming -/// series, maintains windows, and writes content-addressed materializations. +/// DAG-format precompute installation. Planner node payloads and dependency +/// edges define execution; materializations attach storage/window placement. +/// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths +/// execute through the maintenance DAG scheduler at stored-state frontiers. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -154,6 +153,8 @@ pub enum StateEncoding { SketchlibProtobufV1, SketchCoreMsgpackV1, ExactAccumulatorV1, + /// Persisted backend state with explicit Planner family and population layout. + PlannerExactAccumulatorV1, ExactCounterAccumulatorV2, } @@ -912,8 +913,14 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate, _, - ) => vec![StateEncoding::ExactCounterAccumulatorV2], - SummaryFamilyType::ExactAggregate(..) => vec![StateEncoding::ExactAccumulatorV1], + ) => vec![ + StateEncoding::ExactCounterAccumulatorV2, + StateEncoding::PlannerExactAccumulatorV1, + ], + SummaryFamilyType::ExactAggregate(..) => vec![ + StateEncoding::ExactAccumulatorV1, + StateEncoding::PlannerExactAccumulatorV1, + ], SummaryFamilyType::Sketch(kind, _) if matches!( kind.algorithm(), diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 9e3e04bf..f7071c7b 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -665,6 +665,22 @@ pub enum ExactReadout { Max, } +impl ExactReadout { + /// Planner family required by this installed DAG readout node. + pub fn planner_family(self) -> planner_types::post_asap::SummaryFamilyType { + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + let (kind, params) = match self { + Self::Sum => (ExactKind::Sum, ExactParams::Sum), + Self::Count => (ExactKind::Count, ExactParams::Count), + Self::Increase => (ExactKind::Increase, ExactParams::Increase), + Self::Rate => (ExactKind::Rate, ExactParams::Rate), + Self::Min => (ExactKind::Min, ExactParams::Min), + Self::Max => (ExactKind::Max, ExactParams::Max), + }; + SummaryFamilyType::ExactAggregate(kind, params) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryReadout { diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 4e0dd515..40a90fe3 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `AggregationConfig`s, so it represents planned policy +//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -32,7 +32,7 @@ use std::collections::{BTreeSet, HashMap}; -use crate::aggregation_config::AggregationConfig; +use crate::aggregation_config::PrecomputeMaterialization; use crate::policy_fingerprint::PolicyFingerprint; use crate::policy_registry::PolicyRegistry; @@ -62,7 +62,7 @@ impl RoutingIndex { /// Tier 1 — exact fingerprint lookup. Delegates to the underlying /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&AggregationConfig> { + pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { self.registry.get(fp) } @@ -136,8 +136,8 @@ mod tests { use crate::KeyByLabelNames; use std::collections::HashMap as StdHashMap; - fn cfg(metric: &str) -> AggregationConfig { - AggregationConfig::new( + fn cfg(metric: &str) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -172,7 +172,7 @@ mod tests { fn multiple_policies_for_the_same_metric_all_bucket_together() { // Same metric, distinct group-by shapes -> distinct fingerprints, // same bucket. - let a = AggregationConfig::new( + let a = PrecomputeMaterialization::new( AggregationType::Sum, String::new(), StdHashMap::new(), @@ -220,8 +220,9 @@ mod tests { #[test] fn len_and_is_empty_match_registry() { - let idx = - RoutingIndex::build(PolicyRegistry::from_configs(Vec::::new())); + let idx = RoutingIndex::build(PolicyRegistry::from_configs( + Vec::::new(), + )); assert!(idx.is_empty()); assert_eq!(idx.len(), 0); @@ -234,7 +235,7 @@ mod tests { fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { let mut parameters = StdHashMap::new(); parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::DDSketch, String::new(), parameters, diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index 896c9556..c19a1819 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -479,6 +479,9 @@ pub enum SummaryOperator { /// Complete planner materialization configuration, including heap/Hydra /// dimensions and readout/update subtype. Never equal to a legacy projection. Configured { + /// Planner-selected semantic family; grouping and pane layout live in + /// the data descriptor and summary definition, respectively. + family: planner_types::post_asap::SummaryFamilyType, aggregation_type: AggregationType, aggregation_sub_type: String, parameters: BTreeMap, @@ -694,13 +697,48 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; + if let SummaryOperator::Configured { + family, + aggregation_type, + .. + } = &operator + { + if let Some(expected) = aggregation_type.planner_exact_family() { + if family != &expected { + return Err(SdsError( + "configured storage type disagrees with Planner family".into(), + )); + } + } else { + use AggregationType as A; + let expected = match aggregation_type { + A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), + A::CountMinSketch => Some(SketchAlgorithm::Cms), + A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), + A::CountSketch => Some(SketchAlgorithm::CountSketch), + A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), + A::DDSketch => Some(SketchAlgorithm::DDSketch), + A::HLL => Some(SketchAlgorithm::Hll), + A::UnivMon => Some(SketchAlgorithm::UnivMon), + _ => None, + }; + if let Some(expected) = expected { + if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) + { + return Err(SdsError( + "configured sketch storage disagrees with Planner family".into(), + )); + } + } + } + } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), )); } let content = json!({"operator":operator,"fidelity":fidelity,"state_schema_version":state_schema_version}); - let id = SummaryDescriptorId(format!("summary:v2:{}", canonical(&content))); + let id = SummaryDescriptorId(format!("summary:v3:{}", canonical(&content))); Ok(Self { id, operator, @@ -736,6 +774,10 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { + family: config + .accumulator_spec() + .map_err(|error| SdsError(error.to_string()))? + .family, aggregation_type: config.aggregation_type, aggregation_sub_type: config.aggregation_sub_type.clone(), parameters: config @@ -763,11 +805,8 @@ impl FidelityGuarantee { matches!( (aggregation_type, self), (A::UnivMon, UnivMonFrequency { .. }) - | ( - A::Sum | A::MultipleSum | A::Min | A::Max | A::MultipleMin | A::MultipleMax, - Exact - ) - | (A::Increase | A::MultipleIncrease, ExactCounter { .. }) + | (A::Sum | A::Count | A::Min | A::Max, Exact) + | (A::Increase | A::Rate, ExactCounter { .. }) | (A::DatasketchesKLL | A::HydraKLL, KllRankError { .. }) | (A::DDSketch, DdSketchRelativeError { .. }) | (A::HLL, HllCardinalityError { .. }) @@ -1584,6 +1623,7 @@ mod tests { .is_err()); assert!(SummaryDescriptor::new( SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::new(), @@ -1608,6 +1648,24 @@ mod tests { ) .is_err()); } + + #[test] + fn configured_descriptor_rejects_family_storage_disagreement() { + assert!(SummaryDescriptor::new( + SummaryOperator::Configured { + family: AggregationType::Rate.planner_exact_family().unwrap(), + aggregation_type: AggregationType::Increase, + aggregation_sub_type: String::new(), + parameters: BTreeMap::new(), + }, + FidelityGuarantee::ExactCounter { + model: "prometheus.extrapolated-rate.v1".into(), + full_pane_coverage_required: true, + }, + 2, + ) + .is_err()); + } #[test] fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); @@ -1664,11 +1722,13 @@ mod tests { #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { let a = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), }; let b = SummaryOperator::Configured { + family: AggregationType::Sum.planner_exact_family().unwrap(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 01162475..11cbfa4a 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -395,13 +395,13 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; use std::collections::HashMap; - let cfg = AggregationConfig::new( + let cfg = PrecomputeMaterialization::new( AT::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 230fde04..0d9a9304 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -582,7 +582,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t } /// Resolve the bucket sid (and `policy_fp`) for a single data point -/// against a single matching `AggregationConfig`. +/// against a single matching `PrecomputeMaterialization`. /// /// B7.6 — sid is the bucket identity in the precompute engine; this /// helper folds `(config, grouping-label-values)` into a single u64 via @@ -602,7 +602,7 @@ fn flush_barrier_drops(_state: &IngestState, drops: &HashMap, driver_t /// their separate wire-level identity protocol. fn resolve_bucket_sid_for_agg_config( ingest_state: &Arc, - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, point_labels: &HashMap, captured_generation: Option<&asap_types::sds::CatalogGeneration>, ) -> Result<(u64, asap_types::PolicyFingerprint), String> { @@ -1783,15 +1783,16 @@ async fn route_modified_otlp_sketches_to_precompute( // Detection is independent of the legacy dual-write // (it only drives the routed/unconfigured accounting), // so we walk it whether or not the worker push fires. - let matching_configs: Vec<&asap_types::aggregation_config::AggregationConfig> = - agg_configs - .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) - .collect(); + let matching_configs: Vec< + &asap_types::aggregation_config::PrecomputeMaterialization, + > = agg_configs + .values() + .filter(|config| { + config.metric == canonical_name + || config.spatial_filter_normalized == canonical_name + || config.spatial_filter == canonical_name + }) + .collect(); let matched_any = !matching_configs.is_empty(); // CQ-2 — only pay the worker push (and the per-config @@ -1869,7 +1870,7 @@ async fn route_modified_otlp_sketches_to_precompute( routed += 1; } else { // CQ-6 — a decoded sketch that matched no - // AggregationConfig in the running streaming config. + // PrecomputeMaterialization in the running streaming config. ingest_state .observability .dropped_unconfigured @@ -1914,7 +1915,7 @@ async fn route_modified_otlp_sketches_to_precompute( /// `AggregationType`. Inverse direction is in /// `sketch_algorithm_for` above. Used by /// [`derive_sketch_policy_fp`] to find the policy whose -/// `AggregationConfig.aggregation_type` matches a freshly-ingested +/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested /// sketch. /// /// `Any` is a control-plane analysis-time wildcard — it doesn't @@ -3415,7 +3416,7 @@ mod policy_fp_lookup_tests { fn sketch_config_to_params_uses_canonical_keys() { // The param-name vocabulary must match what the control plane // writes in streaming-config YAML (see - // `asap_types::aggregation_config::AggregationConfig::from_yaml_data`). + // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). // Drift surfaces as `find_policy_by_content` missing matches. let dd = sketch_config_to_params(&SketchConfig::DDSketch { relative_accuracy: 0.01, @@ -4516,7 +4517,7 @@ mod sid_bucketing_tests { metric::Data, number_data_point::Value as NumberValue, Gauge as PbGauge, Metric as PbMetric, NumberDataPoint, ResourceMetrics, ScopeMetrics, }; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -4533,8 +4534,8 @@ mod sid_bucketing_tests { } } - fn sum_agg_config(metric: &str, grouping: &[&str]) -> AggregationConfig { - AggregationConfig::new( + fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 7459c6e2..123c0b83 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -717,11 +717,9 @@ fn route_messages( && matches!( config.aggregation_type, asap_types::AggregationType::Increase - | asap_types::AggregationType::MultipleIncrease + | asap_types::AggregationType::Rate | asap_types::AggregationType::Min | asap_types::AggregationType::Max - | asap_types::AggregationType::MultipleMin - | asap_types::AggregationType::MultipleMax )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -1040,8 +1038,8 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; - let aggregation = AggregationConfig { + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; + let aggregation = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -1165,10 +1163,10 @@ mod tests { #[test] fn global_topk_cms_routes_once_while_counters_remain_per_series() { use asap_types::enums::WindowKind; - use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; + use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let config = - |aggregation_type, grouping: Vec, aggregated: Vec| AggregationConfig { + let config = |aggregation_type, grouping: Vec, aggregated: Vec| { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -1203,7 +1201,8 @@ mod tests { table_timestamp_column: None, partitioning: None, value_source_column: None, - }; + } + }; let cms = config( AggregationType::CountMinSketchWithHeap, vec![], diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 1608d3e0..776e9197 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -1004,9 +1004,9 @@ fn metric_has_exact_agg_sum_sid( cap, Capability::ExactAgg( AggregationType::Sum - | AggregationType::MultipleSum + | AggregationType::Count | AggregationType::Increase - | AggregationType::MultipleIncrease + | AggregationType::Rate ) ) { return true; @@ -2848,100 +2848,18 @@ mod tests { /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and /// the GET snapshot emission. #[tokio::test] - async fn test_streaming_config_hot_reload_round_trip() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload.clone())).await; - let client = Client::new(); - - // Initial GET: empty config, 0 entries. - let initial = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .send() - .await - .expect("GET failed"); - assert!(initial.status().is_success()); - let initial_body: serde_json::Value = initial.json().await.unwrap(); - assert_eq!(initial_body["aggregation_count"], 0); - - // POST a new config with two aggregation_ids. The YAML shape - // matches what `StreamingConfig::from_yaml_data` parses — see - // `asap-common/dependencies/rs/asap_types/src/streaming_config.rs`. - let new_config_yaml = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 102 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host, region] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 120 - windowType: tumbling - spatialFilter: '' -"#; - let post_resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(new_config_yaml.to_string()) - .send() - .await - .expect("POST failed"); - let post_status = post_resp.status(); - let post_body: serde_json::Value = post_resp.json().await.unwrap(); - assert!( - post_status.is_success(), - "POST returned {post_status}: {post_body}" - ); - assert_eq!(post_body["status"], "success"); - assert_eq!(post_body["new_aggregation_count"], 2); - // PR 5: the YAML's `aggregationId` fields are silently - // dropped — `agg_ids_added` carries fingerprint u64s. - let added = post_body["agg_ids_added"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(added.len(), 2, "exactly two distinct aggs were added"); - assert!(added.iter().all(|id| *id != 0), "fingerprints are non-zero"); - - // GET again: should reflect the two new ids. - let after = client - .get(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) + async fn flat_streaming_config_is_rejected_without_mutating_active_state() { + let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; + let before = handle.snapshot(); + let response = Client::new() + .post(format!("http://127.0.0.1:{port}/api/v1/streaming-config")) + .body("aggregations: [{aggregationType: Sum, metric: m}]") .send() .await - .expect("GET after swap failed"); - assert!(after.status().is_success()); - let after_body: serde_json::Value = after.json().await.unwrap(); - assert_eq!(after_body["aggregation_count"], 2); - - // The underlying StreamingConfigHandle handle (cloned into - // the server at setup) also reflects the swap — proving that - // downstream consumers that re-snapshot would see the new - // state. PR 5: the map is keyed on fingerprints, so just - // assert the entry count. - let direct_snap = hot_reload.snapshot(); - assert_eq!(direct_snap.materializations_by_policy_fingerprint.len(), 2); + .unwrap(); + assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert!(Arc::ptr_eq(&before, &handle.snapshot())); } #[tokio::test] @@ -2984,7 +2902,7 @@ aggregations: .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::BAD_REQUEST); + assert_eq!(resp.status(), reqwest::StatusCode::GONE); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "error"); } @@ -3049,192 +2967,6 @@ aggregations: }); } - #[tokio::test] - async fn test_streaming_config_swap_drives_sid_reconcile() { - // Schema retirement final cut: the swap handler now drives a - // single sid-level reconcile (no `SchemaRegistry`). Sids that - // already exist in the catalog and whose content signature - // does not appear in the new config get force-retired; the - // response surfaces them under `sids_retired`. There is no - // `sids_added` — sids are minted lazily by the ingest path, - // not by the swap handler. - use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::sketch_db::AggStatus; - - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let summary_store = Arc::new(SketchStore::new()); - // Pre-register two Active sids whose signatures match the - // first config below; only sid 1 will survive the second - // swap. - register_precompute_sid(&summary_store, 1, "cpu_usage", &["host"]); - register_precompute_sid(&summary_store, 2, "mem_usage", &["host"]); - let server_port = setup_test_server_with_hot_reload_and_sketch_index( - hot_reload.clone(), - summary_store.clone(), - ) - .await; - let client = Client::new(); - - // POST a config whose signatures cover both pre-registered - // sids. Nothing should retire. - let yaml_two = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' - - aggregationId: 202 - aggregationType: Sum - aggregationSubType: '' - metric: mem_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_two.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - let retired_ids = body["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert!( - retired_ids.is_empty(), - "no sid should retire when every signature still appears in the new config; got {retired_ids:?}", - ); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Active - ); - - // Swap to a config that drops `mem_usage`. Sid 2's signature - // is now orphaned; the handler must force-retire it. - let yaml_one = r#" -aggregations: - - aggregationId: 101 - aggregationType: Sum - aggregationSubType: '' - metric: cpu_usage - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp2 = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .expect("POST failed"); - let body2: serde_json::Value = resp2.json().await.unwrap(); - let retired = body2["sids_retired"] - .as_array() - .unwrap() - .iter() - .map(|v| v.as_u64().unwrap()) - .collect::>(); - assert_eq!(retired, vec![2u64]); - assert_eq!( - summary_store.instance(1).unwrap().status(), - AggStatus::Active - ); - assert_eq!( - summary_store.instance(2).unwrap().status(), - AggStatus::Retired - ); - } - - #[tokio::test] - async fn test_streaming_config_swap_response_shape_with_empty_catalog() { - // With no registered sids, the swap still works — it just - // produces an empty `sids_retired` array. The `agg_ids_added` - // / `agg_ids_removed` / `new_aggregation_count` fields are - // driven purely by the diff of the two configs and are - // independent of the sid catalog. - // - // PR 5: the YAML's `aggregationId: 42` is silently dropped at - // parse time — the backend identity is content-addressed via - // `PolicyFingerprint::from_config`. The `agg_ids_added` u64 - // in the HTTP response is the fingerprint's `as_u64()` form, - // NOT the literal `42` the YAML once spelled out. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); - let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; - let client = Client::new(); - - let yaml = r#" -aggregations: - - aggregationType: Sum - aggregationSubType: '' - metric: m - labels: - grouping: [] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml.to_string()) - .send() - .await - .expect("POST failed"); - assert!(resp.status().is_success()); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "success"); - assert_eq!(body["new_aggregation_count"], 1); - let added = body["agg_ids_added"].as_array().expect("array"); - assert_eq!(added.len(), 1, "exactly one agg was added"); - assert_ne!( - added[0].as_u64().unwrap(), - 0, - "agg id is not the 0 sentinel" - ); - assert_eq!(body["agg_ids_removed"], serde_json::json!([])); - // No pre-registered sids → nothing to retire. - assert_eq!(body["sids_retired"].as_array().unwrap().len(), 0); - } - #[tokio::test] async fn test_get_schemas_returns_active_and_retired_sids_with_status_filter() { // Schema retirement final cut: `/api/v1/db/schemas` now @@ -3254,29 +2986,12 @@ aggregations: .await; let client = Client::new(); - // Retire sid 2 by pushing a config covering only `m1`. - let yaml_one = r#" -aggregations: - - aggregationId: 1 - aggregationType: Sum - aggregationSubType: '' - metric: m1 - labels: { grouping: [], rollup: [], aggregated: [] } - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#; - let resp = client - .post(format!( - "http://127.0.0.1:{server_port}/api/v1/streaming-config" - )) - .header("content-type", "application/x-yaml") - .body(yaml_one.to_string()) - .send() - .await - .unwrap(); - assert!(resp.status().is_success()); + assert!(summary_store + .force_retire( + 2, + crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION + ) + .is_some()); // GET /api/v1/db/schemas (no filter = all). let resp = client @@ -3513,7 +3228,7 @@ aggregations: registry: Arc, active_agg_ids: &[u64], ) -> (u16, std::collections::HashMap) { - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -3535,7 +3250,7 @@ aggregations: let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), @@ -5618,96 +5333,11 @@ async fn handle_get_streaming_config(State(state): State) -> axum::res (StatusCode::OK, axum::Json(body)).into_response() } -async fn handle_post_streaming_config( - State(state): State, - body: axum::body::Bytes, -) -> axum::response::Response { - use axum::http::StatusCode; +async fn handle_post_streaming_config() -> axum::response::Response { use axum::response::IntoResponse; - use std::collections::HashSet; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - - let yaml_text = match std::str::from_utf8(&body) { - Ok(s) => s, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("request body is not valid UTF-8: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let yaml_value: serde_yaml::Value = match serde_yaml::from_str(yaml_text) { - Ok(v) => v, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("YAML parse error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - let new_config = - match crate::storage_engines::types::StreamingConfig::from_yaml_data(&yaml_value) { - Ok(c) => c, - Err(e) => { - let body = serde_json::json!({ - "status": "error", - "error": format!("StreamingConfig build error: {e}")}); - return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); - } - }; - - let new_ids: HashSet = new_config - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let old_arc = handle.swap(new_config); - let old_ids: HashSet = old_arc - .materializations_by_policy_fingerprint - .keys() - .copied() - .collect(); - let added: Vec = new_ids.difference(&old_ids).copied().collect(); - let removed: Vec = old_ids.difference(&new_ids).copied().collect(); - - if !removed.is_empty() { - warn!( - "streaming-config hot-reload removed agg_ids {:?} — any in-flight \ - precompute worker groups for these ids will continue with their \ - construction-time config until they close naturally (phase 1 \ - limitation; see StreamingConfigHandle module doc)", - removed - ); - } - - // Schema retirement final cut: the sid catalog is the only - // lifecycle registry. The legacy per-`agg_id` `SchemaRegistry` is - // gone, so the swap handler now drives a single sid-level - // reconcile (`reconcile_from_streaming_config`) which force-retires - // any sid whose content signature no longer appears in the new - // config. There is no "added" set: sids are minted lazily at the - // first ingest write under the new config (see - // `SketchStore::ingest_precompute_for_agg_config`). - let snap = handle.snapshot(); - let sid_summary = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( - state.summary_store.as_ref(), - snap.as_ref(), - crate::storage_engines::sketch_db::DEFAULT_RETIREMENT_RETENTION, - ); - - let body = serde_json::json!({ - "status": "success", - "agg_ids_added": added, - "agg_ids_removed": removed, - "new_aggregation_count": new_ids.len(), - "sids_retired": sid_summary.retired}); - (StatusCode::OK, axum::Json(body)).into_response() + (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ + "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" + }))).into_response() } pub use asap_types::plan_publication::PhysicalPlanInstallRequest; @@ -5736,11 +5366,48 @@ pub fn validate_and_build_runtime_plan( .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; } + for entry in request.query_plan.entries.values() { + for binding in entry.materialization_bindings() { + let materialization = request + .precompute_plan + .materializations + .iter() + .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) + .ok_or_else(|| "query binding has no precompute definition".to_string())?; + if binding.window_ms != materialization.stored_window_ms() { + return Err( + "query physical pane duration differs from installed precompute definition" + .into(), + ); + } + if binding.pane_origin_ms != materialization.pane_origin_ms { + return Err( + "query physical pane origin differs from installed precompute definition" + .into(), + ); + } + // `full_window_slide_ms` is `#[serde(default)]`, so a publication from an + // older controller -- or one replayed from a stored artifact -- arrives as + // `None` on a FullWindow materialization. Without this gate the readout + // silently takes the overlap-merging path and counts observations twice, + // which is exactly what the full-window binding exists to prevent. + let full_window_slide_ms = matches!( + materialization.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) + .then_some(materialization.slide_interval.saturating_mul(1_000)); + if binding.full_window_slide_ms != full_window_slide_ms { + return Err( + "query window layout differs from installed precompute definition".into(), + ); + } + } + } asap_types::plan_publication::validate_stored_output_references( &request.precompute_plan, &request.query_plan, )?; - let runtime_materializations = request + let _runtime_materializations = request .precompute_plan .runtime_materializations() .map_err(|error| format!("PrecomputePlan validation error: {error}"))?; @@ -5755,8 +5422,10 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = - crate::storage_engines::types::StreamingConfig::new(runtime_materializations); + let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; let typed_fps: BTreeSet<_> = streaming_config .materializations_by_policy_fingerprint .keys() diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index 2721bc68..8c3ac483 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -37,13 +37,13 @@ pub mod utils; // Re-export commonly used types to avoid glob import conflicts pub use storage_engines::types::{ - AggregateCore, AggregationConfig, KeyByLabelValues, Measurement, MergeableAccumulator, - MultipleSubpopulationAggregate, PrecomputedOutput, SerializableToSink, - SingleSubpopulationAggregate, + AggregateCore, KeyByLabelValues, Measurement, MergeableAccumulator, + MultipleSubpopulationAggregate, PrecomputeMaterialization, PrecomputedOutput, + SerializableToSink, SingleSubpopulationAggregate, }; pub use precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; pub use storage_engines::StoreResult; diff --git a/data_plane/src/precompute_engine/accumulator_factory.rs b/data_plane/src/precompute_engine/accumulator_factory.rs index 43a779fc..9f94c8c9 100644 --- a/data_plane/src/precompute_engine/accumulator_factory.rs +++ b/data_plane/src/precompute_engine/accumulator_factory.rs @@ -1,30 +1,19 @@ use crate::precompute_engine::operators::{ CountMinSketchAccumulator, CountMinSketchWithHeapAccumulator, CountSketchAccumulator, CountSketchWithHeapAccumulator, DDSketchAccumulator, DatasketchesKLLAccumulator, - HydraKllSketchAccumulator, IncreaseAccumulator, MaxAccumulator, MinAccumulator, - MultipleIncreaseAccumulator, MultipleMaxAccumulator, MultipleMinAccumulator, - MultipleSumAccumulator, SumAccumulator, + HydraKllSketchAccumulator, IncreaseAccumulator, KeyedCounterState, KeyedMaxState, + KeyedMinState, KeyedSumCountAccumulator, MaxAccumulator, MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::{ AggregateCore, AggregationType, KeyByLabelValues, Measurement, }; -use asap_types::aggregation_config::AggregationConfig; -// Step 5 (sketch-identity unification, see -// scratchpad/artifacts/enum-unification-plan.md): dispatch below is -// driven by `AccumulatorSpec` (SummaryFamilyType + typed family parameters + -// keyed-axis grouping) instead of raw `AggregationType` + -// `aggregation_sub_type` string matching. Numeric params come straight -// off the committed family's typed params (no HashMap lookups) except -// `cms_params`, kept as a raw-`parameters` read for the one case Planner's -// family parameters have no field for: HydraKLL's `(row, col)` tiling grid (see -// `asap_types::accumulator_spec`'s module doc for why). `cms_params` -// now lives there — the only place that still needs the other three -// former local helpers (`kll_k_param`, `heap_size_param`, -// `ddsketch_alpha_param`) is that module's own `AccumulatorSpec` -// construction, so they aren't re-imported here. +use asap_types::aggregation_config::PrecomputeMaterialization; +// Production dispatch consumes Planner SummaryAgg payloads directly. The +// config adapter below is compiled only for isolated historical kernel tests. use super::operators::hll_sketch_accumulator::HllSketchAccumulator; use super::operators::univmon_accumulator::UnivMonAccumulator; -use asap_types::accumulator_spec::{cms_params, AccumulatorSpecError}; +#[cfg(test)] +use asap_types::accumulator_spec::cms_params; use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; /// Generate the two boilerplate clone-based `AccumulatorUpdater` methods @@ -379,28 +368,32 @@ impl AccumulatorUpdater for DDSketchAccumulatorUpdater { } // --------------------------------------------------------------------------- -// MultipleSumAccumulatorUpdater +// KeyedSumCountAccumulatorUpdater // --------------------------------------------------------------------------- -pub struct MultipleSumAccumulatorUpdater { - acc: MultipleSumAccumulator, +pub struct KeyedSumCountAccumulatorUpdater { + acc: KeyedSumCountAccumulator, } -impl MultipleSumAccumulatorUpdater { +impl KeyedSumCountAccumulatorUpdater { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { Self { - acc: MultipleSumAccumulator::new(), + acc: KeyedSumCountAccumulator::for_family(family), } } } -impl Default for MultipleSumAccumulatorUpdater { +impl Default for KeyedSumCountAccumulatorUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { +impl AccumulatorUpdater for KeyedSumCountAccumulatorUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -415,7 +408,7 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleSumAccumulator::new(); + self.acc = KeyedSumCountAccumulator::for_family(self.acc.family.clone()); } fn is_keyed(&self) -> bool { @@ -423,13 +416,13 @@ impl AccumulatorUpdater for MultipleSumAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() - + self.acc.sums.len() * (std::mem::size_of::() + 8) + std::mem::size_of::() + + self.acc.sums.len() * (std::mem::size_of::() + 16) } } // --------------------------------------------------------------------------- -// MultipleMinAccumulatorUpdater / MultipleMaxAccumulatorUpdater +// KeyedMinStateUpdater / KeyedMaxStateUpdater // --------------------------------------------------------------------------- macro_rules! multiple_extremum_updater { @@ -475,32 +468,32 @@ macro_rules! multiple_extremum_updater { }; } -multiple_extremum_updater!(MultipleMinAccumulatorUpdater, MultipleMinAccumulator); -multiple_extremum_updater!(MultipleMaxAccumulatorUpdater, MultipleMaxAccumulator); +multiple_extremum_updater!(KeyedMinStateUpdater, KeyedMinState); +multiple_extremum_updater!(KeyedMaxStateUpdater, KeyedMaxState); // --------------------------------------------------------------------------- -// MultipleIncreaseAccumulatorUpdater +// KeyedCounterStateUpdater // --------------------------------------------------------------------------- -pub struct MultipleIncreaseAccumulatorUpdater { - acc: MultipleIncreaseAccumulator, +pub struct KeyedCounterStateUpdater { + acc: KeyedCounterState, } -impl MultipleIncreaseAccumulatorUpdater { +impl KeyedCounterStateUpdater { pub fn new() -> Self { Self { - acc: MultipleIncreaseAccumulator::new(), + acc: KeyedCounterState::new(), } } } -impl Default for MultipleIncreaseAccumulatorUpdater { +impl Default for KeyedCounterStateUpdater { fn default() -> Self { Self::new() } } -impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { +impl AccumulatorUpdater for KeyedCounterStateUpdater { fn update_single(&mut self, _value: f64, _timestamp_ms: i64) { debug_assert!( false, @@ -528,7 +521,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { impl_clone_accumulator_methods!(acc); fn reset(&mut self) { - self.acc = MultipleIncreaseAccumulator::new(); + self.acc = KeyedCounterState::new(); } fn is_keyed(&self) -> bool { @@ -536,7 +529,7 @@ impl AccumulatorUpdater for MultipleIncreaseAccumulatorUpdater { } fn memory_usage_bytes(&self) -> usize { - std::mem::size_of::() + std::mem::size_of::() + self.acc.increases.len() * (std::mem::size_of::() + std::mem::size_of::()) @@ -899,27 +892,20 @@ impl AccumulatorUpdater for HydraKllAccumulatorUpdater { /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &AggregationConfig) -> bool { - matches!( - config.aggregation_type, - AggregationType::MultipleSubpopulation - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax - | AggregationType::CountMinSketch - | AggregationType::CountMinSketchWithHeap - | AggregationType::CountSketch - | AggregationType::CountSketchWithHeap - | AggregationType::HydraKLL - ) +pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { + config + .accumulator_spec() + .expect("valid fixture") + .grouping + .is_some() } /// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. /// /// * `value` / `sum`: sum values per key (default). /// * `count` / `frequency` / `freq`: count occurrences per key. -fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { +#[cfg(test)] +fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { match config.sample_update_rule() { asap_types::SampleUpdateRule::Count => TopkWeight::Count, asap_types::SampleUpdateRule::Value { .. } @@ -927,7 +913,8 @@ fn topk_weight_param(config: &AggregationConfig) -> TopkWeight { } } -fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { +#[cfg(test)] +fn topk_weight_scale_param(config: &PrecomputeMaterialization) -> f64 { match config.sample_update_rule() { asap_types::SampleUpdateRule::Value { scale } => scale, asap_types::SampleUpdateRule::CounterDelta { scale } => scale, @@ -943,6 +930,7 @@ fn topk_weight_scale_param(config: &AggregationConfig) -> f64 { /// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with /// `SketchParams::Kll`, so the /// other arm is unreachable from a `spec` this module builds itself. +#[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { // Lossless: `accumulator_spec()` only ever stores a value that @@ -955,12 +943,12 @@ fn kll_k(params: &SketchParams) -> u16 { } } -/// Read `(width, depth)` out of `SketchParams::Cms` or `::CountSketch` +/// Read `(rows = depth, columns = width)` out of `SketchParams::Cms` or `::CountSketch` /// — same shape, different variant per bare-sketch identity. fn cms_dims(params: &SketchParams) -> (usize, usize) { match params { SketchParams::Cms { width, depth } | SketchParams::CountSketch { width, depth } => { - (*width as usize, *depth as usize) + (*depth as usize, *width as usize) } other => unreachable!( "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" @@ -968,7 +956,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { } } -/// Read `(width, depth, heap_size)` out of `SketchParams::CmsWithHeap` +/// Read `(rows = depth, columns = width, heap_size)` out of `SketchParams::CmsWithHeap` /// or `::CountSketchWithHeap`. fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { match params { @@ -981,7 +969,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { width, depth, heap_size, - } => (*width as usize, *depth as usize, *heap_size as usize), + } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), @@ -989,6 +977,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { } /// Read the DDSketch relative-accuracy `alpha` out of `SketchParams::DDSketch`. +#[cfg(test)] fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, @@ -998,54 +987,28 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { } } -/// Create an appropriate `AccumulatorUpdater` from an `AggregationConfig`. -/// -/// Dispatches on [`asap_types::AccumulatorSpec`] — `SummaryFamilyType` identity -/// plus the keyed/unkeyed `grouping` axis — instead of the pre-Step-5 -/// `AggregationType` + `aggregation_sub_type` string combo. See -/// `asap_types::accumulator_spec`'s module doc for why min/max direction, -/// HydraKLL's `(row, col)` tiling, and top-k `weight_mode` still read -/// `config` directly rather than going through Planner family parameters — -/// none of those three have a field in the Planner-owned types. -pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - let spec = match config.accumulator_spec() { - Ok(spec) => spec, - // Three fallback paths, preserved verbatim from the pre-Step-5 - // dispatch: same warning text, same default updater per case - // (Single- and MultipleSubpopulation default to *different* - // updaters — see `AccumulatorSpecError`'s doc). - Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown SingleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(SumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType(sub_type)) => { - tracing::warn!( - "Unknown MultipleSubpopulation sub_type '{}', defaulting to Sum", - sub_type - ); - return Box::new(MultipleSumAccumulatorUpdater::new()); - } - Err(AccumulatorSpecError::UnmappedAggregationType(other)) => { - tracing::warn!( - "Unknown aggregation_type '{:?}', defaulting to SingleSubpopulation Sum", - other - ); - return Box::new(SumAccumulatorUpdater::new()); - } - }; +/// Construct isolated payload fixtures for kernel/storage unit tests. +/// Production execution requires a validated Planner DAG program. +#[cfg(test)] +pub fn create_fixture_accumulator( + config: &PrecomputeMaterialization, +) -> Box { + let spec = config + .accumulator_spec() + .expect("invalid isolated kernel fixture"); let keyed = spec.grouping.is_some(); match (&spec.family, keyed) { - (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), false) => { Box::new(SumAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), true) => { - Box::new(MultipleSumAccumulatorUpdater::new()) + Box::new(KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Sum)) } + (SummaryFamilyType::ExactAggregate(ExactKind::Count, _), true) => Box::new( + KeyedSumCountAccumulatorUpdater::for_family(ExactKind::Count), + ), // Direction comes off the family itself now. It used to be read // back out of `aggregation_sub_type` because Planner had one @@ -1056,20 +1019,20 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - Box::new(MultipleMinAccumulatorUpdater::new()) + Box::new(KeyedMinStateUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), false) => { Box::new(MaxAccumulatorUpdater::new()) } (SummaryFamilyType::ExactAggregate(ExactKind::Max, _), true) => { - Box::new(MultipleMaxAccumulatorUpdater::new()) + Box::new(KeyedMaxStateUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), false) => { + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), false) => { Box::new(IncreaseAccumulatorUpdater::new()) } - (SummaryFamilyType::ExactAggregate(ExactKind::Increase, _), true) => { - Box::new(MultipleIncreaseAccumulatorUpdater::new()) + (SummaryFamilyType::ExactAggregate(ExactKind::Increase | ExactKind::Rate, _), true) => { + Box::new(KeyedCounterStateUpdater::new()) } (SummaryFamilyType::Sketch(kind, _), false) @@ -1187,14 +1150,8 @@ pub fn create_accumulator_updater(config: &AggregationConfig) -> Box { - tracing::warn!( - "SummaryFamilyType {:?} (keyed={}) has no accumulator_factory mapping, defaulting to Sum", - other_family, - keyed - ); - Box::new(SumAccumulatorUpdater::new()) + panic!("unsupported isolated kernel fixture {other_family:?}, keyed={keyed}") } } } @@ -1271,7 +1228,7 @@ mod tests { #[test] fn hll_and_univmon_raw_updates_share_value_identity() { for family in [AggregationType::HLL, AggregationType::UnivMon] { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( family, String::new(), Default::default(), @@ -1288,7 +1245,7 @@ mod tests { None, None, ); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); for value in [0.0, -0.0, 2.0, 2.0, f64::NAN] { updater.update_single(value, 1000); } @@ -1362,7 +1319,7 @@ mod tests { #[test] fn test_multiple_sum_updater() { - let mut updater = MultipleSumAccumulatorUpdater::new(); + let mut updater = KeyedSumCountAccumulatorUpdater::new(); assert!(updater.is_keyed()); let key_a = KeyByLabelValues::new_with_labels(vec!["a".to_string()]); @@ -1372,7 +1329,7 @@ mod tests { updater.update_keyed(&key_b, 2.0, 2000); let acc = updater.take_accumulator(); - assert_eq!(acc.type_name(), "MultipleSumAccumulator"); + assert_eq!(acc.type_name(), "KeyedSumCountAccumulator"); } #[test] @@ -1424,7 +1381,7 @@ mod tests { use std::collections::HashMap; let make_config = |agg_type: AggregationType, sub_type: &str| { - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, sub_type.to_string(), HashMap::new(), @@ -1463,18 +1420,15 @@ mod tests { AggregationType::MultipleSubpopulation, "Sum" ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleSum, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleIncrease, - "" - ))); - assert!(config_is_keyed(&make_config( - AggregationType::MultipleMax, - "" - ))); + let mut keyed = make_config(AggregationType::Sum, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Increase, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); + let mut keyed = make_config(AggregationType::Max, ""); + keyed.aggregated_labels = asap_types::KeyByLabelNames::new(vec!["host".into()]); + assert!(config_is_keyed(&keyed)); assert!(config_is_keyed(&make_config( AggregationType::CountMinSketch, "" @@ -1497,12 +1451,12 @@ mod tests { for (agg_type, sub_type) in &[ (AggregationType::SingleSubpopulation, "Sum"), (AggregationType::MultipleSubpopulation, "Sum"), - (AggregationType::MultipleSum, ""), + (AggregationType::Sum, ""), (AggregationType::DatasketchesKLL, ""), (AggregationType::CountMinSketch, ""), ] { let config = make_config(*agg_type, sub_type); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); assert_eq!( config_is_keyed(&config), updater.is_keyed(), @@ -1518,7 +1472,7 @@ mod tests { use std::collections::HashMap; let mut params = HashMap::new(); params.insert("K".to_string(), serde_json::Value::from(50_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "DatasketchesKLL".to_string(), params, @@ -1535,7 +1489,7 @@ mod tests { None, None, ); - let updater = create_accumulator_updater(&config); + let updater = create_fixture_accumulator(&config); let acc = updater.snapshot_accumulator(); let kll = acc .as_any() @@ -1555,7 +1509,7 @@ mod tests { let mut params = HashMap::new(); params.insert("d".to_string(), serde_json::Value::from(7_u64)); params.insert("w".to_string(), serde_json::Value::from(2048_u64)); - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), params, @@ -1575,7 +1529,7 @@ mod tests { assert_eq!(super::cms_params(&config), (7, 2048)); // Empty params — defaults `(4, 1000)`. - let empty_config = AggregationConfig::new( + let empty_config = PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), HashMap::new(), @@ -1601,7 +1555,10 @@ mod tests { /// Build a `*WithHeap` config keyed by group-by label `host`, with the /// given `weight_mode` param (None → default = value-weighted). - fn topk_config(agg_type: AggregationType, weight_mode: Option<&str>) -> AggregationConfig { + fn topk_config( + agg_type: AggregationType, + weight_mode: Option<&str>, + ) -> PrecomputeMaterialization { use std::collections::HashMap; let mut params = HashMap::new(); // Small, deterministic geometry; heap big enough to hold all hosts. @@ -1611,7 +1568,7 @@ mod tests { if let Some(m) = weight_mode { params.insert("weight_mode".to_string(), serde_json::Value::from(m)); } - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, String::new(), params, @@ -1699,7 +1656,7 @@ mod tests { fn value_weighted_topk_ranks_hosts_by_sum_of_value() { // DEFAULT mode (no weight_mode param) must be value-weighted. let config = topk_config(AggregationType::CountMinSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); assert!(updater.is_keyed()); feed_stream(&mut *updater); @@ -1725,14 +1682,24 @@ mod tests { #[test] fn counter_delta_scale_preserves_sub_unit_membership_weights() { - let mut config = topk_config( - AggregationType::CountMinSketchWithHeap, - Some("counter_delta"), - ); - config - .parameters - .insert("weight_scale".into(), serde_json::json!(1_000_000)); - let mut updater = create_accumulator_updater(&config); + use planner_types::post_asap::{ + EntityIdentity, NonNegativeWeightProof, SummaryInputExpr, SummaryUpdate, WeightDomain, + }; + let config = topk_config(AggregationType::CountMinSketchWithHeap, None); + let family = config.accumulator_spec().unwrap().family; + let input = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::ResetAwareCounterDelta { + value: planner_types::pre_asap::ColumnRef::SampleValue, + series: EntityIdentity::PromqlLabelSet { excluding: vec![] }, + }, + weight_domain: WeightDomain::NonNegative { + proof: NonNegativeWeightProof::ResetAwareCounterDerivative, + }, + }; + let mut updater = create_planner_accumulator(&family, &input, &Default::default()).unwrap(); updater.update_keyed(&host_key("payment"), 0.004, 1_000); updater.update_keyed(&host_key("order"), 0.002, 1_000); let ranked = ranked_topk(&*updater.take_accumulator()); @@ -1744,7 +1711,7 @@ mod tests { fn count_weighted_topk_still_ranks_by_occurrence_frequency() { // Opt-in frequency-top-k: weight_mode=count must rank by event count. let config = topk_config(AggregationType::CountMinSketchWithHeap, Some("count")); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); @@ -1764,7 +1731,7 @@ mod tests { // (real median-of-signed-rows math) — same value-weighted default // as the CMS-family heap path, but no longer conflated with it. let config = topk_config(AggregationType::CountSketchWithHeap, None); - let mut updater = create_accumulator_updater(&config); + let mut updater = create_fixture_accumulator(&config); feed_stream(&mut *updater); let acc = updater.take_accumulator(); assert_eq!(acc.type_name(), "CountSketchWithHeapAccumulator"); @@ -1800,3 +1767,278 @@ mod tests { } } } + +#[cfg(test)] +mod planner_family_regression { + use super::*; + use asap_types::{enums::WindowKind, KeyByLabelNames}; + + // Every installed exact producer must retain its family in runtime state. + #[test] + fn exact_state_identity_survives_factory_and_reset() { + for kind in [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Rate, + AggregationType::Increase, + AggregationType::Min, + AggregationType::Max, + ] { + let config = PrecomputeMaterialization::new( + kind, + String::new(), + Default::default(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + KeyByLabelNames::empty(), + String::new(), + 60, + 60, + WindowKind::Tumbling, + String::new(), + "metric".into(), + None, + None, + None, + ); + let mut updater = create_planner_accumulator( + &config.accumulator_spec().unwrap().family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &Default::default(), + ) + .unwrap(); + updater.update_single(4.0, 1000); + updater.update_single(7.0, 2000); + assert_eq!(updater.take_accumulator().get_accumulator_type(), kind); + assert_eq!(updater.snapshot_accumulator().get_accumulator_type(), kind); + } + } +} + +/// Construct the kernel declared by a Planner SummaryAgg. No backend config +/// tags participate in this dispatch and unsupported payloads are errors. +pub fn create_planner_accumulator( + family: &SummaryFamilyType, + input: &planner_types::post_asap::SummaryUpdate, + grouping: &planner_types::post_asap::GroupingStrategy, +) -> Result, String> { + use planner_types::post_asap::GroupingStrategy; + if grouping != &GroupingStrategy::PerSubpopulationInstance { + return Err("shared summary grouping requires a supported Planner Hydra kernel".into()); + } + if matches!(family, SummaryFamilyType::ExactAggregate(..)) { + return Ok(Box::new(PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator::new( + family.clone(), + input.item.is_some(), + )?, + })); + } + let SummaryFamilyType::Sketch(kind, family_grouping) = family else { + return Err(format!("unsupported Planner summary family {family:?}")); + }; + if family_grouping != grouping { + return Err("Planner family and operator grouping disagree".into()); + } + // Heap counters use fixed-point storage for fractional counter deltas. + // This encodes the selected update; it does not choose another family. + let weight_scale = if matches!( + input.weight, + planner_types::post_asap::SummaryInputExpr::ResetAwareCounterDelta { .. } + ) { + 1_000_000.0 + } else { + 1.0 + }; + let updater: Box = match (kind.algorithm(), kind.params()) { + (SketchAlgorithm::Kll, SketchParams::Kll { k }) => Box::new(KllAccumulatorUpdater::new( + u16::try_from(*k).map_err(|_| "KLL k exceeds runtime bound")?, + )), + (SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Box::new(DDSketchAccumulatorUpdater::new(*alpha)) + } + (SketchAlgorithm::Cms, params @ SketchParams::Cms { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CmsAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CountSketch, params @ SketchParams::CountSketch { .. }) => { + let (r, c) = cms_dims(params); + Box::new(CountSketchAccumulatorUpdater::new(r, c)) + } + (SketchAlgorithm::CmsWithHeap, params @ SketchParams::CmsWithHeap { .. }) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CmsHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + ( + SketchAlgorithm::CountSketchWithHeap, + params @ SketchParams::CountSketchWithHeap { .. }, + ) => { + let (r, c, h) = cms_heap_dims(params); + Box::new(CountSketchWithHeapAccumulatorUpdater::with_weight_scale( + r, + c, + h, + TopkWeight::Value, + weight_scale, + )) + } + (SketchAlgorithm::Hll, SketchParams::Hll { precision }) => Box::new(HllUpdater { + acc: HllSketchAccumulator::new( + asap_sketchlib::HllVariant::Regular, + u32::from(*precision), + ), + }), + ( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, + }, + ) => Box::new(UnivMonUpdater { + acc: UnivMonAccumulator::new( + *heap_size as usize, + *sketch_rows as usize, + *sketch_cols as usize, + *layers as usize, + ) + .map_err(|e| e.to_string())?, + }), + _ => { + return Err(format!( + "unsupported Planner algorithm/parameters: {kind:?}" + )) + } + }; + if updater.is_keyed() != input.item.is_some() + && !asap_types::accumulator_spec::is_unit_sample_frequency(input) + { + return Err("Planner item expression does not match the selected kernel layout".into()); + } + Ok(updater) +} + +struct PlannerExactUpdater { + acc: super::operators::exact_accumulator::ExactAccumulator, +} +impl AccumulatorUpdater for PlannerExactUpdater { + fn update_single(&mut self, value: f64, timestamp: i64) { + self.acc.update(None, value, timestamp); + } + fn update_keyed(&mut self, key: &KeyByLabelValues, value: f64, timestamp: i64) { + self.acc.update(Some(key), value, timestamp); + } + impl_clone_accumulator_methods!(acc); + fn reset(&mut self) { + self.acc = super::operators::exact_accumulator::ExactAccumulator::new( + self.acc.family().clone(), + self.acc.is_keyed(), + ) + .expect("installed exact family"); + } + fn is_keyed(&self) -> bool { + self.acc.is_keyed() + } + fn memory_usage_bytes(&self) -> usize { + self.acc.approx_memory_bytes() + } +} + +#[cfg(test)] +mod planner_parameter_regression { + use super::*; + use planner_types::post_asap::{SketchKind, SummaryInputExpr, SummaryUpdate}; + + // Planner width is the bucket count; depth is the independent hash-row count. + #[test] + fn planner_sketch_dimensions_are_not_transposed() { + for (algorithm, params) in [ + ( + SketchAlgorithm::Cms, + SketchParams::Cms { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { + width: 128, + depth: 3, + }, + ), + ( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width: 128, + depth: 3, + heap_size: 8, + }, + ), + ] { + let family = SummaryFamilyType::Sketch( + SketchKind::new(algorithm.clone(), params), + Default::default(), + ); + let update = SummaryUpdate { + item: Some(SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::Named("host".into()), + )), + weight: SummaryInputExpr::Constant(1.0), + weight_domain: Default::default(), + }; + let state = create_planner_accumulator(&family, &update, &Default::default()) + .unwrap() + .snapshot_accumulator(); + let dims = match algorithm { + SketchAlgorithm::Cms => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketch => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows, s.inner.cols) + } + SketchAlgorithm::CmsWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + SketchAlgorithm::CountSketchWithHeap => { + let s = state + .as_any() + .downcast_ref::() + .unwrap(); + (s.inner.rows(), s.inner.cols()) + } + _ => unreachable!(), + }; + assert_eq!(dims, (3, 128), "{algorithm:?}"); + } + } +} diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index 47593276..9e499adc 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -56,7 +56,7 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::AggregationConfig, + config: &asap_types::PrecomputeMaterialization, timestamp_ms: i64, value: f64, ) { @@ -264,8 +264,8 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::AggregationConfig) { - let config = asap_types::AggregationConfig::new( + fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { + let config = asap_types::PrecomputeMaterialization::new( asap_types::AggregationType::HLL, String::new(), Default::default(), diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index d975eb6a..67940478 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,7 +1,7 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::StreamingConfigHandle; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -28,11 +28,11 @@ pub struct IngestObservability { /// A full / delta frame failed to decode (or a delta failed to /// apply) and was dropped. pub dropped_decode_fail: AtomicU64, - /// A decoded sketch matched no `AggregationConfig` in the running + /// A decoded sketch matched no `PrecomputeMaterialization` in the running /// streaming config (legacy routing-side bucketing miss). pub dropped_unconfigured: AtomicU64, /// The output sink could not resolve a `policy_fp` to an - /// `AggregationConfig` (registry miss) and skipped the write. + /// `PrecomputeMaterialization` (registry miss) and skipped the write. pub dropped_policy_miss: AtomicU64, /// RES-1 — max number of distinct tumbling windows a per-series /// snapshot base may lag behind the newest observed `window_start` @@ -120,7 +120,7 @@ pub struct IngestState { pub samples_blocked_by_schema_barrier: std::sync::atomic::AtomicU64, /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. - /// This replaces the old frozen `Vec>`. + /// This replaces the old frozen `Vec>`. pub hot_reload_config: StreamingConfigHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, @@ -157,7 +157,7 @@ impl IngestState { /// visible immediately without restart. /// /// Returns the shared `Arc` — no cloning of - /// individual AggregationConfig objects, just an atomic refcount + /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() @@ -247,7 +247,7 @@ impl IngestState { /// ingest sources (e.g. OTLP) can reuse it. pub fn extract_group_key_for( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { extract_group_key(series_key, config) } @@ -261,7 +261,7 @@ impl IngestState { /// [`Self::extract_group_key_for`] does after the round-trip. pub fn extract_group_key_from_labels( labels: &std::collections::HashMap, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map( |name| { @@ -278,7 +278,7 @@ impl IngestState { /// for a given series key and aggregation config. fn extract_group_key( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> Arc { let labels = parse_labels_from_series_key(series_key); crate::precompute_engine::group_key::intern_pairs(config.grouping_labels.iter().map(|name| { @@ -294,18 +294,18 @@ mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Kept as a parameter to // avoid churning the call sites below. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::CountMinSketch, String::new(), std::collections::HashMap::new(), diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 44f7d67d..428a42af 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -122,7 +122,7 @@ fn frozen_population_value( struct OperatorAdapter<'a> { binding: &'a BackendExecutableBinding, inputs: MaintenanceInputs<'a>, - configs: &'a [asap_types::aggregation_config::AggregationConfig], + configs: &'a [asap_types::aggregation_config::PrecomputeMaterialization], } impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { @@ -193,7 +193,12 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } finalize_exact(node, inputs) } - ExecutableOperatorPayload::SummaryAgg { family, input, .. } => { + ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } => { let [value] = inputs else { return Err("maintenance SummaryAgg requires exactly one row input".into()); }; @@ -251,7 +256,9 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = super::accumulator_factory::create_accumulator_updater(config); + let mut updater = super::accumulator_factory::create_planner_accumulator( + family, input, grouping, + )?; if updater.is_keyed() { return Err("keyed maintenance accumulator requires an item expression".into()); } diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 068ac13b..3f744870 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -11,6 +11,7 @@ pub(crate) mod metrics; pub mod multisource_coordinator; pub mod operators; pub mod output_sink; +pub mod raw_dag; pub mod series_buffer; pub mod series_router; pub mod subdag_scheduler; diff --git a/data_plane/src/precompute_engine/operators/exact_accumulator.rs b/data_plane/src/precompute_engine/operators/exact_accumulator.rs new file mode 100644 index 00000000..b7fcead1 --- /dev/null +++ b/data_plane/src/precompute_engine/operators/exact_accumulator.rs @@ -0,0 +1,327 @@ +//! Exact summary state identified by Planner family, independent of keyed layout. +use super::increase_accumulator::IncreaseAccumulator; +use crate::storage_engines::types::{ + AggregateCore, AggregationType, AuxStats, KeyByLabelValues, Measurement, SerializableToSink, +}; +use asap_types::Statistic; +use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +type Error = Box; + +#[derive(Debug, Clone, Serialize, Deserialize)] +enum ScalarState { + Sum(f64), + Count(u64), + Min(Option), + Max(Option), + Counter(Option), +} + +/// Both the family and population layout survive persistence. Sharing counter +/// arithmetic never authorizes a Rate state to answer an Increase readout. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ExactAccumulator { + family: SummaryFamilyType, + scalar: ScalarState, + keyed: Option>, +} + +impl ExactAccumulator { + pub fn new(family: SummaryFamilyType, keyed: bool) -> Result { + use ExactKind as K; + use ExactParams as P; + let scalar = match &family { + SummaryFamilyType::ExactAggregate(K::Sum, P::Sum) => ScalarState::Sum(0.0), + SummaryFamilyType::ExactAggregate(K::Count, P::Count) => ScalarState::Count(0), + SummaryFamilyType::ExactAggregate(K::Min, P::Min) => ScalarState::Min(None), + SummaryFamilyType::ExactAggregate(K::Max, P::Max) => ScalarState::Max(None), + SummaryFamilyType::ExactAggregate(K::Rate, P::Rate) + | SummaryFamilyType::ExactAggregate(K::Increase, P::Increase) => { + ScalarState::Counter(None) + } + _ => return Err(format!("unsupported exact Planner family: {family:?}")), + }; + Ok(Self { + family, + scalar, + keyed: keyed.then(HashMap::new), + }) + } + + pub fn family(&self) -> &SummaryFamilyType { + &self.family + } + pub fn is_keyed(&self) -> bool { + self.keyed.is_some() + } + + pub fn update(&mut self, key: Option<&KeyByLabelValues>, value: f64, timestamp: i64) { + let state = match (&mut self.keyed, key) { + (Some(states), Some(key)) => states + .entry(key.clone()) + .or_insert_with(|| self.scalar.clone()), + (None, None) => &mut self.scalar, + _ => panic!("exact update population layout differs from installed DAG"), + }; + match state { + ScalarState::Sum(sum) => *sum += value, + ScalarState::Count(count) => { + *count = count.checked_add(1).expect("exact count overflow") + } + ScalarState::Min(current) => { + *current = Some(current.map_or(value, |old| old.min(value))) + } + ScalarState::Max(current) => { + *current = Some(current.map_or(value, |old| old.max(value))) + } + ScalarState::Counter(current) => match current { + Some(counter) => counter.update(Measurement::new(value), timestamp), + None => { + *current = Some(IncreaseAccumulator::new( + Measurement::new(value), + timestamp, + Measurement::new(value), + timestamp, + )) + } + }, + } + } + + pub fn deserialize_from_bytes(bytes: &[u8]) -> Result { + let state: Self = rmp_serde::from_slice(bytes)?; + let expected = Self::new(state.family.clone(), state.is_keyed())?; + let same_variant = |value: &ScalarState| { + std::mem::discriminant(value) == std::mem::discriminant(&expected.scalar) + }; + if !same_variant(&state.scalar) + || state + .keyed + .as_ref() + .is_some_and(|states| states.values().any(|s| !same_variant(s))) + { + return Err("exact payload differs from declared Planner family".into()); + } + Ok(state) + } + + fn statistic(&self) -> Statistic { + match self.family { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Min, _) => Statistic::Min, + SummaryFamilyType::ExactAggregate(ExactKind::Max, _) => Statistic::Max, + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => Statistic::Rate, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => Statistic::Increase, + _ => unreachable!("validated exact family"), + } + } +} + +fn merge_scalar(left: &ScalarState, right: &ScalarState) -> Result { + Ok(match (left, right) { + (ScalarState::Sum(a), ScalarState::Sum(b)) => ScalarState::Sum(a + b), + (ScalarState::Count(a), ScalarState::Count(b)) => { + ScalarState::Count(a.checked_add(*b).ok_or("exact count overflow")?) + } + (ScalarState::Min(a), ScalarState::Min(b)) => { + ScalarState::Min(a.iter().chain(b).copied().reduce(f64::min)) + } + (ScalarState::Max(a), ScalarState::Max(b)) => { + ScalarState::Max(a.iter().chain(b).copied().reduce(f64::max)) + } + (ScalarState::Counter(a), ScalarState::Counter(b)) => { + ScalarState::Counter(match (a, b) { + (Some(a), Some(b)) => Some( + >::merge_accumulators(vec![a.clone(), b.clone()])?, + ), + (a, b) => a.clone().or_else(|| b.clone()), + }) + } + _ => return Err("exact scalar state families differ".into()), + }) +} + +impl SerializableToSink for ExactAccumulator { + fn serialize_to_json(&self) -> serde_json::Value { + serde_json::json!({"family": self.family, "scalar": self.scalar, "keyed": self.keyed.as_ref().map(|m|m.iter().collect::>())}) + } + fn serialize_to_bytes(&self) -> Vec { + rmp_serde::to_vec_named(self).expect("exact state encoding") + } +} + +impl AggregateCore for ExactAccumulator { + fn clone_boxed_core(&self) -> Box { + Box::new(self.clone()) + } + fn type_name(&self) -> &'static str { + "PlannerExactAccumulatorV1" + } + fn as_any(&self) -> &dyn std::any::Any { + self + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn merge_with(&self, other: &dyn AggregateCore) -> Result, Error> { + let other = other + .as_any() + .downcast_ref::() + .ok_or("merge requires Planner exact state")?; + if self.family != other.family || self.is_keyed() != other.is_keyed() { + return Err("cannot merge different Planner families or layouts".into()); + } + let mut merged = self.clone(); + if let (Some(target), Some(source)) = (&mut merged.keyed, &other.keyed) { + for (key, state) in source { + let combined = match target.get(key) { + Some(old) => merge_scalar(old, state)?, + None => state.clone(), + }; + target.insert(key.clone(), combined); + } + } else { + merged.scalar = merge_scalar(&self.scalar, &other.scalar)?; + } + Ok(Box::new(merged)) + } + fn get_accumulator_type(&self) -> AggregationType { + match self.statistic() { + Statistic::Sum => AggregationType::Sum, + Statistic::Count => AggregationType::Count, + Statistic::Min => AggregationType::Min, + Statistic::Max => AggregationType::Max, + Statistic::Rate => AggregationType::Rate, + Statistic::Increase => AggregationType::Increase, + _ => unreachable!(), + } + } + fn approx_memory_bytes(&self) -> usize { + std::mem::size_of::() + + self.keyed.as_ref().map_or(0, |m| { + m.keys() + .map(|k| { + std::mem::size_of::() + + k.labels.iter().map(String::len).sum::() + }) + .sum::() + }) + } + fn aux_stats(&self) -> AuxStats { + if self.is_keyed() { + return AuxStats::empty(); + } + match self.scalar { + ScalarState::Sum(value) => AuxStats { + sum: Some(value), + ..AuxStats::empty() + }, + ScalarState::Count(value) => AuxStats { + count: Some(value), + ..AuxStats::empty() + }, + ScalarState::Min(value) => AuxStats { + min: value, + ..AuxStats::empty() + }, + ScalarState::Max(value) => AuxStats { + max: value, + ..AuxStats::empty() + }, + ScalarState::Counter(_) => AuxStats::empty(), + } + } + fn get_keys(&self) -> Option> { + self.keyed.as_ref().map(|m| m.keys().cloned().collect()) + } + fn query_statistic( + &self, + statistic: Statistic, + key: &Option, + kwargs: &HashMap, + ) -> Result { + if statistic != self.statistic() { + return Err("readout differs from Planner exact family".into()); + } + let state = match (&self.keyed, key) { + (Some(states), Some(key)) => states.get(key).ok_or("unknown exact population")?, + (None, None) => &self.scalar, + _ => return Err("readout population differs from installed layout".into()), + }; + match state { + ScalarState::Sum(sum) => Ok(*sum), + ScalarState::Count(count) => Ok(*count as f64), + ScalarState::Min(value) | ScalarState::Max(value) => { + value.ok_or_else(|| "empty exact population".into()) + } + ScalarState::Counter(Some(counter)) => { + counter.query_statistic(statistic, &None, kwargs) + } + ScalarState::Counter(None) => Err("empty counter population".into()), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Identity, population isolation, and readout survive the persisted format. + #[test] + fn exact_families_roundtrip_and_reject_cross_family_operations() { + let families = [ + (ExactKind::Sum, ExactParams::Sum, Statistic::Sum, 16.0), + (ExactKind::Count, ExactParams::Count, Statistic::Count, 3.0), + (ExactKind::Min, ExactParams::Min, Statistic::Min, 2.0), + (ExactKind::Max, ExactParams::Max, Statistic::Max, 8.0), + (ExactKind::Rate, ExactParams::Rate, Statistic::Rate, 3.0), + ( + ExactKind::Increase, + ExactParams::Increase, + Statistic::Increase, + 6.0, + ), + ]; + for keyed in [false, true] { + let key = keyed.then(|| KeyByLabelValues::new_with_labels(vec!["a".into()])); + let mut states = Vec::new(); + for (kind, params, stat, value) in &families { + let mut state = ExactAccumulator::new( + SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), + keyed, + ) + .unwrap(); + for (ts, v) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(key.as_ref(), v, ts); + } + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()).unwrap(); + assert_eq!(restored.family(), state.family()); + assert_eq!( + restored + .query_statistic(*stat, &key, &HashMap::new()) + .unwrap(), + *value + ); + for (_, _, wrong, _) in &families { + if wrong != stat { + assert!(restored + .query_statistic(*wrong, &key, &HashMap::new()) + .is_err()); + } + } + states.push(restored); + } + for (i, a) in states.iter().enumerate() { + for (j, b) in states.iter().enumerate() { + assert_eq!(a.merge_with(b).is_ok(), i == j); + } + } + } + } +} diff --git a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs similarity index 86% rename from data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_counter_state.rs index c1d59aa1..b94d2fab 100644 --- a/data_plane/src/precompute_engine/operators/multiple_increase_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_counter_state.rs @@ -12,11 +12,11 @@ use asap_types::Statistic; /// Accumulator that maintains separate increase accumulators for multiple keys /// Allows tracking rate/increase for different label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleIncreaseAccumulator { +pub struct KeyedCounterState { pub increases: HashMap, } -impl MultipleIncreaseAccumulator { +impl KeyedCounterState { pub fn new() -> Self { Self { increases: HashMap::new(), @@ -91,13 +91,13 @@ impl MultipleIncreaseAccumulator { } } -impl Default for MultipleIncreaseAccumulator { +impl Default for KeyedCounterState { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleIncreaseAccumulator { +impl SerializableToSink for KeyedCounterState { fn serialize_to_json(&self) -> Value { let entries: Vec = self .increases @@ -135,13 +135,13 @@ impl SerializableToSink for MultipleIncreaseAccumulator { } } -impl AggregateCore for MultipleIncreaseAccumulator { +impl AggregateCore for KeyedCounterState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleIncreaseAccumulator" + "KeyedCounterState" } fn as_any(&self) -> &dyn std::any::Any { @@ -156,20 +156,20 @@ impl AggregateCore for MultipleIncreaseAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleIncreaseAccumulator + // Check if other is also a KeyedCounterState if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleIncreaseAccumulator with {}", + "Cannot merge KeyedCounterState with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleIncreaseAccumulator + // Downcast to KeyedCounterState let other_multiple_increase = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleIncreaseAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedCounterState")?; // Clone self once, then merge each matching counter with the same // reset-aware, boundary-aware implementation used by the unkeyed path. @@ -189,7 +189,7 @@ impl AggregateCore for MultipleIncreaseAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleIncrease + AggregationType::Increase } fn approx_memory_bytes(&self) -> usize { @@ -210,14 +210,12 @@ impl AggregateCore for MultipleIncreaseAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleIncreaseAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedCounterState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { +impl MultipleSubpopulationAggregate for KeyedCounterState { fn query( &self, statistic: Statistic, @@ -227,7 +225,7 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { let data = self .increases .get(key) - .ok_or_else(|| format!("Key {key} not found in MultipleIncreaseAccumulator"))?; + .ok_or_else(|| format!("Key {key} not found in KeyedCounterState"))?; data.query(statistic, query_kwargs) } @@ -237,15 +235,15 @@ impl MultipleSubpopulationAggregate for MultipleIncreaseAccumulator { } } -impl MergeableAccumulator for MultipleIncreaseAccumulator { +impl MergeableAccumulator for KeyedCounterState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleIncreaseAccumulator::new(); + let mut result = KeyedCounterState::new(); for accumulator in accumulators { for (key, data) in accumulator.increases { @@ -291,14 +289,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_creation() { - let acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_creation() { + let acc = KeyedCounterState::new(); assert!(acc.increases.is_empty()); } #[test] - fn test_multiple_increase_accumulator_update() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_update() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -316,8 +314,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_query() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_query() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -344,14 +342,14 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_sum_per_key() { - // `sum by (zone) (counter)` reaches MultipleIncreaseAccumulator + fn test_keyed_counter_state_sum_per_key() { + // `sum by (zone) (counter)` reaches KeyedCounterState // only when the ASAP-tier ingest groups multiple series under // a single accumulator (the `Multiple*` variant). In that case // each per-key Sum should be the series' latest cumulative // value; the engine's outer `by` aggregation does the cross-key // grouping. (Issue ProjectASAP/ASAPCollector#46.) - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let east = KeyByLabelValues::new_with_labels(vec!["us-east-1".to_string()]); let west = KeyByLabelValues::new_with_labels(vec!["us-west-2".to_string()]); @@ -369,9 +367,9 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_merge() { - let mut acc1 = MultipleIncreaseAccumulator::new(); - let mut acc2 = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_merge() { + let mut acc1 = KeyedCounterState::new(); + let mut acc2 = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -387,7 +385,7 @@ mod tests { create_test_increase_accumulator_with_time(15.0, 2000, 30.0, 3000), ); // Later time range - let merged = MultipleIncreaseAccumulator::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = KeyedCounterState::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.increases.len(), 2); assert!(merged.increases.contains_key(&key1)); @@ -400,8 +398,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_serialization() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_serialization() { + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let second_key = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -415,7 +413,7 @@ mod tests { // Test JSON serialization let json_value = acc.serialize_to_json(); - let deserialized = MultipleIncreaseAccumulator::deserialize_from_json(&json_value).unwrap(); + let deserialized = KeyedCounterState::deserialize_from_json(&json_value).unwrap(); assert_eq!(deserialized.increases.len(), 2); let deserialized_acc = deserialized.increases.get(&key).unwrap(); @@ -425,8 +423,7 @@ mod tests { // Test binary serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = - MultipleIncreaseAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedCounterState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.increases.len(), 2); let deserialized_acc_bytes = deserialized_bytes.increases.get(&key).unwrap(); @@ -445,8 +442,8 @@ mod tests { } #[test] - fn test_multiple_increase_accumulator_get_keys() { - let mut acc = MultipleIncreaseAccumulator::new(); + fn test_keyed_counter_state_get_keys() { + let mut acc = KeyedCounterState::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); let key2 = KeyByLabelValues::new_with_labels(vec!["api".to_string()]); @@ -462,7 +459,7 @@ mod tests { #[test] fn test_trait_object() { - let mut acc = MultipleIncreaseAccumulator::new(); + let mut acc = KeyedCounterState::new(); let key = KeyByLabelValues::new(); acc.update(key.clone(), create_test_increase_accumulator(10.0, 25.0)); @@ -477,7 +474,7 @@ mod tests { } // #[test] - // fn test_multiple_increase_accumulator_arroyo_deserialization() { + // fn test_keyed_counter_state_arroyo_deserialization() { // // Create test data in Arroyo MessagePack format // // Format: {key: [starting_value, starting_timestamp, last_seen_value, last_seen_timestamp]} // let mut test_data = std::collections::HashMap::new(); @@ -489,7 +486,7 @@ mod tests { // // Test Arroyo deserialization // let deserialized_acc = - // MultipleIncreaseAccumulator::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); + // KeyedCounterState::deserialize_from_bytes_arroyo(&arroyo_buffer).unwrap(); // // Verify the deserialized accumulator has the correct data // assert_eq!(deserialized_acc.increases.len(), 2); diff --git a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_max_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_max_state.rs index 1865d268..4309c04a 100644 --- a/data_plane/src/precompute_engine/operators/multiple_max_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_max_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key maximum over many populations, mergeable by comparison. /// /// The minimum direction is -/// [`MultipleMinAccumulator`](super::multiple_min_accumulator::MultipleMinAccumulator), +/// [`KeyedMinState`](super::keyed_min_state::KeyedMinState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMaxAccumulator { +pub struct KeyedMaxState { pub values: HashMap, } -impl MultipleMaxAccumulator { +impl KeyedMaxState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMaxAccumulator { } } -impl SerializableToSink for MultipleMaxAccumulator { +impl SerializableToSink for KeyedMaxState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMaxAccumulator { } } -impl AggregateCore for MultipleMaxAccumulator { +impl AggregateCore for KeyedMaxState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMaxAccumulator" + "KeyedMaxState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMaxAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMaxAccumulator with {}", + "Cannot merge KeyedMaxState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMaxAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMaxAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMaxState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMaxAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMax + AggregationType::Max } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMaxAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMaxAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMaxState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { +impl MultipleSubpopulationAggregate for KeyedMaxState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMaxAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMaxAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMaxState").into()), + other => Err(format!("Unsupported statistic in KeyedMaxState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMaxAccumulator { } } -impl MergeableAccumulator for MultipleMaxAccumulator { +impl MergeableAccumulator for KeyedMaxState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMaxAccumulator::new(); + let mut result = KeyedMaxState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_largest_per_key() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Min, &key("a"), None).is_err()); assert!(acc.query(Statistic::Max, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMaxAccumulator::new(); + let mut left = KeyedMaxState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMaxAccumulator::new(); + let mut right = KeyedMaxState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Max, &key("a"), None).unwrap(), 10.0); assert_eq!(merged.query(Statistic::Max, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_min_accumulator::MultipleMinAccumulator; - let mine = MultipleMaxAccumulator::new(); - let theirs = MultipleMinAccumulator::new(); + use super::super::keyed_min_state::KeyedMinState; + let mine = KeyedMaxState::new(); + let theirs = KeyedMinState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMaxAccumulator::new(); + let mut acc = KeyedMaxState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMaxAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMaxState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMaxAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMaxState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Max, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_min_state.rs similarity index 81% rename from data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_min_state.rs index 00c25040..5be698f5 100644 --- a/data_plane/src/precompute_engine/operators/multiple_min_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_min_state.rs @@ -11,16 +11,16 @@ use asap_types::Statistic; /// Exact per-key minimum over many populations, mergeable by comparison. /// /// The maximum direction is -/// [`MultipleMaxAccumulator`](super::multiple_max_accumulator::MultipleMaxAccumulator), +/// [`KeyedMaxState`](super::keyed_max_state::KeyedMaxState), /// a separate type: these used to be one `MultipleMinMaxAccumulator` whose /// direction lived in a `sub_type` string that every layer above had to carry /// alongside the family. #[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct MultipleMinAccumulator { +pub struct KeyedMinState { pub values: HashMap, } -impl MultipleMinAccumulator { +impl KeyedMinState { pub fn new() -> Self { Self::default() } @@ -116,7 +116,7 @@ impl MultipleMinAccumulator { } } -impl SerializableToSink for MultipleMinAccumulator { +impl SerializableToSink for KeyedMinState { fn serialize_to_json(&self) -> Value { let mut values_obj = serde_json::Map::new(); for (key, value) in &self.values { @@ -153,13 +153,13 @@ impl SerializableToSink for MultipleMinAccumulator { } } -impl AggregateCore for MultipleMinAccumulator { +impl AggregateCore for KeyedMinState { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleMinAccumulator" + "KeyedMinState" } fn as_any(&self) -> &dyn std::any::Any { @@ -176,7 +176,7 @@ impl AggregateCore for MultipleMinAccumulator { ) -> Result, Box> { if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleMinAccumulator with {}", + "Cannot merge KeyedMinState with {}", other.get_accumulator_type() ) .into()); @@ -184,8 +184,8 @@ impl AggregateCore for MultipleMinAccumulator { let other_multiple = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleMinAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedMinState")?; let merged = Self::merge_accumulators(vec![self.clone(), other_multiple.clone()])?; @@ -193,7 +193,7 @@ impl AggregateCore for MultipleMinAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleMin + AggregationType::Min } fn approx_memory_bytes(&self) -> usize { @@ -212,14 +212,12 @@ impl AggregateCore for MultipleMinAccumulator { query_kwargs: &std::collections::HashMap, ) -> Result> { use crate::storage_engines::types::MultipleSubpopulationAggregate; - let key_val = key - .as_ref() - .ok_or("Key required for MultipleMinAccumulator")?; + let key_val = key.as_ref().ok_or("Key required for KeyedMinState")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleMinAccumulator { +impl MultipleSubpopulationAggregate for KeyedMinState { fn query( &self, statistic: Statistic, @@ -231,10 +229,8 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { .values .get(key) .copied() - .ok_or_else(|| format!("Key {key} not found in MultipleMinAccumulator").into()), - other => { - Err(format!("Unsupported statistic in MultipleMinAccumulator: {other:?}").into()) - } + .ok_or_else(|| format!("Key {key} not found in KeyedMinState").into()), + other => Err(format!("Unsupported statistic in KeyedMinState: {other:?}").into()), } } @@ -243,15 +239,15 @@ impl MultipleSubpopulationAggregate for MultipleMinAccumulator { } } -impl MergeableAccumulator for MultipleMinAccumulator { +impl MergeableAccumulator for KeyedMinState { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleMinAccumulator::new(); + let mut result = KeyedMinState::new(); for acc in accumulators { for (key, value) in acc.values { @@ -273,7 +269,7 @@ mod tests { #[test] fn keeps_the_smallest_per_key() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 10.0); acc.update(key("a"), 5.0); acc.update(key("a"), 15.0); @@ -285,7 +281,7 @@ mod tests { #[test] fn refuses_the_opposite_statistic_and_unknown_keys() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 1.0); assert!(acc.query(Statistic::Max, &key("a"), None).is_err()); assert!(acc.query(Statistic::Min, &key("missing"), None).is_err()); @@ -293,16 +289,17 @@ mod tests { #[test] fn merges_per_key() { - let mut left = MultipleMinAccumulator::new(); + let mut left = KeyedMinState::new(); left.update(key("a"), 10.0); - let mut right = MultipleMinAccumulator::new(); + let mut right = KeyedMinState::new(); right.update(key("a"), 5.0); right.update(key("b"), 3.0); - let merged = >::merge_accumulators(vec![left, right]) - .unwrap(); + let merged = + >::merge_accumulators(vec![ + left, right, + ]) + .unwrap(); assert_eq!(merged.query(Statistic::Min, &key("a"), None).unwrap(), 5.0); assert_eq!(merged.query(Statistic::Min, &key("b"), None).unwrap(), 3.0); @@ -310,26 +307,26 @@ mod tests { #[test] fn refuses_to_merge_with_the_opposite_direction() { - use super::super::multiple_max_accumulator::MultipleMaxAccumulator; - let mine = MultipleMinAccumulator::new(); - let theirs = MultipleMaxAccumulator::new(); + use super::super::keyed_max_state::KeyedMaxState; + let mine = KeyedMinState::new(); + let theirs = KeyedMaxState::new(); assert!(mine.merge_with(&theirs).is_err()); } #[test] fn round_trips_through_both_serializations() { - let mut acc = MultipleMinAccumulator::new(); + let mut acc = KeyedMinState::new(); acc.update(key("a"), 4.0); let json = acc.serialize_to_json(); - let from_json = MultipleMinAccumulator::deserialize_from_json(&json).unwrap(); + let from_json = KeyedMinState::deserialize_from_json(&json).unwrap(); assert_eq!( from_json.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 ); let bytes = acc.serialize_to_bytes(); - let from_bytes = MultipleMinAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let from_bytes = KeyedMinState::deserialize_from_bytes(&bytes).unwrap(); assert_eq!( from_bytes.query(Statistic::Min, &key("a"), None).unwrap(), 4.0 diff --git a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs similarity index 50% rename from data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs rename to data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs index 85a9982d..c39d5583 100644 --- a/data_plane/src/precompute_engine/operators/multiple_sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/keyed_sum_count_accumulator.rs @@ -7,26 +7,53 @@ use serde_json::Value; use std::collections::HashMap; use asap_types::Statistic; +use planner_types::post_asap::ExactKind; + +fn sum_family() -> ExactKind { + ExactKind::Sum +} /// Accumulator that maintains separate sum values for multiple keys /// Allows querying sums for specific label combinations #[derive(Debug, Clone, Serialize, Deserialize)] -pub struct MultipleSumAccumulator { +pub struct KeyedSumCountAccumulator { + #[serde(default = "sum_family")] + pub family: ExactKind, pub sums: HashMap, + #[serde(default)] + pub counts: HashMap, } -impl MultipleSumAccumulator { +impl KeyedSumCountAccumulator { pub fn new() -> Self { + Self::for_family(ExactKind::Sum) + } + + pub fn for_family(family: ExactKind) -> Self { + assert!(matches!(family, ExactKind::Sum | ExactKind::Count)); Self { + family, sums: HashMap::new(), + counts: HashMap::new(), } } pub fn update(&mut self, key: KeyByLabelValues, value: f64) { - *self.sums.entry(key).or_insert(0.0) += value; + let is_new = !self.sums.contains_key(&key); + *self.sums.entry(key.clone()).or_insert(0.0) += value; + if let Some(count) = self.counts.get(&key).copied() { + if let Some(next) = count.checked_add(1).filter(|next| *next != u64::MAX) { + self.counts.insert(key, next); + } else { + self.counts.remove(&key); + } + } else if is_new { + self.counts.insert(key, 1); + } } pub fn add_sum(&mut self, key: KeyByLabelValues, sum: f64) { + self.counts.remove(&key); self.sums.insert(key, sum); } @@ -43,7 +70,28 @@ impl MultipleSumAccumulator { sums.insert(key, sum); } - Ok(Self { sums }) + let mut counts = HashMap::new(); + if let Some(counts_data) = data.get("counts").and_then(Value::as_object) { + for (key_str, value) in counts_data { + let key_json: Value = serde_json::from_str(key_str)?; + let key = KeyByLabelValues::deserialize_from_json(&key_json)?; + let count = value.as_u64().ok_or("Invalid count value")?; + if !sums.contains_key(&key) { + return Err("Count key missing from sums".into()); + } + counts.insert(key, count); + } + } + let family = match data.get("family").and_then(Value::as_str) { + None | Some("Sum") => ExactKind::Sum, + Some("Count") => ExactKind::Count, + _ => return Err("Invalid keyed additive family".into()), + }; + Ok(Self { + family, + sums, + counts, + }) } pub fn deserialize_from_bytes(buffer: &[u8]) -> Result> { @@ -62,6 +110,7 @@ impl MultipleSumAccumulator { offset += 4; let mut sums = HashMap::new(); + let mut keys = Vec::new(); for _ in 0..num_entries { // Read key length and data @@ -99,20 +148,50 @@ impl MultipleSumAccumulator { ]); offset += 8; + keys.push(key.clone()); sums.insert(key, sum); } - - Ok(Self { sums }) + let remaining = buffer.len() - offset; + let count_bytes = num_entries + .checked_mul(8) + .ok_or("Count section too large")?; + if remaining != 0 && remaining != count_bytes && remaining != count_bytes + 1 { + return Err("Invalid count section length".into()); + } + let mut counts = HashMap::new(); + if count_bytes != 0 && remaining >= count_bytes { + for key in keys { + let count = u64::from_le_bytes(buffer[offset..offset + 8].try_into()?); + offset += 8; + if count != u64::MAX { + counts.insert(key, count); + } + } + } + let family = if remaining == count_bytes + 1 { + match buffer[offset] { + 0 => ExactKind::Sum, + 1 => ExactKind::Count, + _ => return Err("Invalid keyed additive family tag".into()), + } + } else { + ExactKind::Sum + }; + Ok(Self { + family, + sums, + counts, + }) } } -impl Default for MultipleSumAccumulator { +impl Default for KeyedSumCountAccumulator { fn default() -> Self { Self::new() } } -impl SerializableToSink for MultipleSumAccumulator { +impl SerializableToSink for KeyedSumCountAccumulator { fn serialize_to_json(&self) -> Value { let mut sums_obj = serde_json::Map::new(); for (key, sum) in &self.sums { @@ -124,8 +203,16 @@ impl SerializableToSink for MultipleSumAccumulator { ); } + let mut counts_obj = serde_json::Map::new(); + for (key, count) in &self.counts { + let key_str = serde_json::to_string(&key.serialize_to_json()).unwrap(); + counts_obj.insert(key_str, Value::from(*count)); + } + serde_json::json!({ - "sums": sums_obj + "family": if self.family == ExactKind::Count { "Count" } else { "Sum" }, + "sums": sums_obj, + "counts": counts_obj }) } @@ -136,7 +223,9 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&(self.sums.len() as u32).to_le_bytes()); // Write each key-value pair + let mut ordered_keys = Vec::with_capacity(self.sums.len()); for (key, sum) in &self.sums { + ordered_keys.push(key); let key_bytes = key.serialize_to_bytes(); // Write key length and data @@ -147,17 +236,34 @@ impl SerializableToSink for MultipleSumAccumulator { buffer.extend_from_slice(&sum.to_le_bytes()); } + for key in ordered_keys { + buffer.extend_from_slice( + &self + .counts + .get(key) + .copied() + .unwrap_or(u64::MAX) + .to_le_bytes(), + ); + } + + buffer.push(if self.family == ExactKind::Count { + 1 + } else { + 0 + }); + buffer } } -impl AggregateCore for MultipleSumAccumulator { +impl AggregateCore for KeyedSumCountAccumulator { fn clone_boxed_core(&self) -> Box { Box::new(self.clone()) } fn type_name(&self) -> &'static str { - "MultipleSumAccumulator" + "KeyedSumCountAccumulator" } fn as_any(&self) -> &dyn std::any::Any { @@ -172,20 +278,20 @@ impl AggregateCore for MultipleSumAccumulator { &self, other: &dyn AggregateCore, ) -> Result, Box> { - // Check if other is also a MultipleSumAccumulator + // Check if other is also a KeyedSumCountAccumulator if other.get_accumulator_type() != self.get_accumulator_type() { return Err(format!( - "Cannot merge MultipleSumAccumulator with {}", + "Cannot merge KeyedSumCountAccumulator with {}", other.get_accumulator_type() ) .into()); } - // Downcast to MultipleSumAccumulator + // Downcast to KeyedSumCountAccumulator let other_multiple_sum = other .as_any() - .downcast_ref::() - .ok_or("Failed to downcast to MultipleSumAccumulator")?; + .downcast_ref::() + .ok_or("Failed to downcast to KeyedSumCountAccumulator")?; // Use the existing merge_accumulators method let merged = Self::merge_accumulators(vec![self.clone(), other_multiple_sum.clone()])?; @@ -194,13 +300,17 @@ impl AggregateCore for MultipleSumAccumulator { } fn get_accumulator_type(&self) -> AggregationType { - AggregationType::MultipleSum + if self.family == ExactKind::Count { + AggregationType::Count + } else { + AggregationType::Sum + } } fn approx_memory_bytes(&self) -> usize { // HashMap. Label strings dominate; use a // conservative per-entry estimate plus HashMap overhead. - const BYTES_PER_ENTRY: usize = 96; + const BYTES_PER_ENTRY: usize = 112; std::mem::size_of::() + self.sums.len() * BYTES_PER_ENTRY } @@ -217,26 +327,35 @@ impl AggregateCore for MultipleSumAccumulator { use crate::storage_engines::types::MultipleSubpopulationAggregate; let key_val = key .as_ref() - .ok_or("Key required for MultipleSumAccumulator")?; + .ok_or("Key required for KeyedSumCountAccumulator")?; self.query(statistic, key_val, Some(query_kwargs)) } } -impl MultipleSubpopulationAggregate for MultipleSumAccumulator { +impl MultipleSubpopulationAggregate for KeyedSumCountAccumulator { fn query( &self, statistic: Statistic, key: &KeyByLabelValues, _query_kwargs: Option<&HashMap>, ) -> Result> { - match statistic { - Statistic::Sum | Statistic::Count => self - .sums + match (&self.family, statistic) { + (ExactKind::Sum, Statistic::Sum) => self.sums.get(key).copied().ok_or_else(|| { + "Key not found in KeyedSumCountAccumulator" + .to_string() + .into() + }), + (ExactKind::Count, Statistic::Count) => self + .counts .get(key) - .copied() - .ok_or_else(|| "Key not found in MultipleSumAccumulator".to_string().into()), + .map(|count| *count as f64) + .ok_or_else(|| { + "Sample count unavailable in KeyedSumCountAccumulator" + .to_string() + .into() + }), _ => Err( - format!("Unsupported statistic in MultipleSumAccumulator: {statistic:?}").into(), + format!("Unsupported statistic in KeyedSumCountAccumulator: {statistic:?}").into(), ), } } @@ -246,17 +365,41 @@ impl MultipleSubpopulationAggregate for MultipleSumAccumulator { } } -impl MergeableAccumulator for MultipleSumAccumulator { +impl MergeableAccumulator for KeyedSumCountAccumulator { fn merge_accumulators( - accumulators: Vec, - ) -> Result> { + accumulators: Vec, + ) -> Result> { if accumulators.is_empty() { return Err("No accumulators to merge".into()); } - let mut result = MultipleSumAccumulator::new(); + let family = accumulators[0].family.clone(); + if accumulators.iter().any(|acc| acc.family != family) { + return Err("Cannot merge different keyed additive families".into()); + } + let mut result = KeyedSumCountAccumulator::for_family(family); for acc in accumulators { + for key in acc.sums.keys() { + match ( + result.counts.get(key).copied(), + acc.counts.get(key).copied(), + ) { + (None, Some(count)) if !result.sums.contains_key(key) => { + result.counts.insert(key.clone(), count); + } + (Some(existing), Some(count)) => { + if let Some(total) = existing.checked_add(count) { + result.counts.insert(key.clone(), total); + } else { + result.counts.remove(key); + } + } + _ => { + result.counts.remove(key); + } + } + } for (key, sum) in acc.sums { *result.sums.entry(key).or_insert(0.0) += sum; } @@ -273,14 +416,14 @@ mod tests { use super::*; #[test] - fn test_multiple_sum_accumulator_creation() { - let acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_creation() { + let acc = KeyedSumCountAccumulator::new(); assert!(acc.sums.is_empty()); } #[test] - fn test_multiple_sum_accumulator_update() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_update() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -295,8 +438,37 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_query() { - let mut acc = MultipleSumAccumulator::new(); + fn grouped_count_reads_sample_count_and_survives_merge_and_round_trip() { + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut first = KeyedSumCountAccumulator::for_family(ExactKind::Count); + first.update(key.clone(), 10.0); + first.update(key.clone(), 20.0); + let mut second = KeyedSumCountAccumulator::for_family(ExactKind::Count); + second.update(key.clone(), 7.0); + let merged = KeyedSumCountAccumulator::merge_accumulators(vec![first, second]).unwrap(); + for acc in [ + merged.clone(), + KeyedSumCountAccumulator::deserialize_from_json(&merged.serialize_to_json()).unwrap(), + KeyedSumCountAccumulator::deserialize_from_bytes(&merged.serialize_to_bytes()).unwrap(), + ] { + assert_eq!(acc.family, ExactKind::Count); + assert!(acc.query(Statistic::Sum, &key, None).is_err()); + assert_eq!(acc.query(Statistic::Count, &key, None).unwrap(), 3.0); + } + } + + #[test] + fn keyed_additive_merge_rejects_different_planner_families() { + assert!(KeyedSumCountAccumulator::merge_accumulators(vec![ + KeyedSumCountAccumulator::for_family(ExactKind::Sum), + KeyedSumCountAccumulator::for_family(ExactKind::Count), + ]) + .is_err()); + } + + #[test] + fn test_keyed_sum_count_accumulator_query() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -315,8 +487,8 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_get_keys() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_get_keys() { + let mut acc = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -332,9 +504,9 @@ mod tests { } #[test] - fn test_multiple_sum_accumulator_merge() { - let mut acc1 = MultipleSumAccumulator::new(); - let mut acc2 = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_merge() { + let mut acc1 = KeyedSumCountAccumulator::new(); + let mut acc2 = KeyedSumCountAccumulator::new(); let key1 = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -345,15 +517,15 @@ mod tests { acc2.add_sum(key1.clone(), 5.0); // Same key, different accumulator - let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); + let merged = >::merge_accumulators(vec![acc1, acc2]).unwrap(); assert_eq!(merged.sums.get(&key1), Some(&15.0)); // Should be merged assert_eq!(merged.sums.get(&key2), Some(&20.0)); // Should be preserved } #[test] - fn test_multiple_sum_accumulator_serialization() { - let mut acc = MultipleSumAccumulator::new(); + fn test_keyed_sum_count_accumulator_serialization() { + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["service".to_string()]); @@ -361,18 +533,18 @@ mod tests { // Test JSON serialization let json = acc.serialize_to_json(); - let deserialized = MultipleSumAccumulator::deserialize_from_json(&json).unwrap(); + let deserialized = KeyedSumCountAccumulator::deserialize_from_json(&json).unwrap(); assert_eq!(deserialized.sums.get(&key), Some(&42.5)); // Test byte serialization let bytes = acc.serialize_to_bytes(); - let deserialized_bytes = MultipleSumAccumulator::deserialize_from_bytes(&bytes).unwrap(); + let deserialized_bytes = KeyedSumCountAccumulator::deserialize_from_bytes(&bytes).unwrap(); assert_eq!(deserialized_bytes.sums.get(&key), Some(&42.5)); } #[test] fn test_trait_object() { - let mut acc = MultipleSumAccumulator::new(); + let mut acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); @@ -381,6 +553,6 @@ mod tests { let trait_obj: Box = Box::new(acc); // Test type name through trait object - assert_eq!(trait_obj.type_name(), "MultipleSumAccumulator"); + assert_eq!(trait_obj.type_name(), "KeyedSumCountAccumulator"); } } diff --git a/data_plane/src/precompute_engine/operators/mod.rs b/data_plane/src/precompute_engine/operators/mod.rs index 9df95ba1..073db6e8 100644 --- a/data_plane/src/precompute_engine/operators/mod.rs +++ b/data_plane/src/precompute_engine/operators/mod.rs @@ -4,15 +4,16 @@ pub mod count_sketch_accumulator; pub mod count_sketch_with_heap_accumulator; pub mod datasketches_kll_accumulator; pub mod dd_sketch_accumulator; +pub mod exact_accumulator; pub mod hll_sketch_accumulator; pub mod hydra_kll_accumulator; pub mod increase_accumulator; +pub mod keyed_counter_state; +pub mod keyed_max_state; +pub mod keyed_min_state; +pub mod keyed_sum_count_accumulator; pub mod max_accumulator; pub mod min_accumulator; -pub mod multiple_increase_accumulator; -pub mod multiple_max_accumulator; -pub mod multiple_min_accumulator; -pub mod multiple_sum_accumulator; pub mod sketch_envelope_accumulator; pub mod sum_accumulator; pub mod univmon_accumulator; @@ -26,11 +27,11 @@ pub use dd_sketch_accumulator::*; pub use hll_sketch_accumulator::*; pub use hydra_kll_accumulator::*; pub use increase_accumulator::*; +pub use keyed_counter_state::*; +pub use keyed_max_state::*; +pub use keyed_min_state::*; +pub use keyed_sum_count_accumulator::*; pub use max_accumulator::*; pub use min_accumulator::*; -pub use multiple_increase_accumulator::*; -pub use multiple_max_accumulator::*; -pub use multiple_min_accumulator::*; -pub use multiple_sum_accumulator::*; pub use sketch_envelope_accumulator::*; pub use sum_accumulator::*; diff --git a/data_plane/src/precompute_engine/operators/sum_accumulator.rs b/data_plane/src/precompute_engine/operators/sum_accumulator.rs index 2cbe66fe..d5ff3b02 100644 --- a/data_plane/src/precompute_engine/operators/sum_accumulator.rs +++ b/data_plane/src/precompute_engine/operators/sum_accumulator.rs @@ -197,7 +197,11 @@ impl SingleSubpopulationAggregate for SumAccumulator { } match statistic { - Statistic::Sum | Statistic::Count => Ok(self.sum), + Statistic::Sum => Ok(self.sum), + Statistic::Count => self + .observation_count + .map(|count| count as f64) + .ok_or_else(|| "sample count is unavailable for this Sum payload".into()), _ => Err(format!("Unsupported statistic in SumAccumulator: {statistic:?}").into()), } } @@ -308,10 +312,7 @@ mod tests { crate::SingleSubpopulationAggregate::query(&acc, Statistic::Sum, None).unwrap(), 42.0 ); - assert_eq!( - crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), - 42.0 - ); + assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).is_err()); assert!(crate::SingleSubpopulationAggregate::query(&acc, Statistic::Min, None).is_err()); // SumAccumulator is a single subpopulation accumulator, doesn't need key-based queries @@ -321,6 +322,17 @@ mod tests { ); } + #[test] + fn count_readout_uses_observation_count_not_sum() { + let mut acc = SumAccumulator::new(); + acc.update(10.0); + acc.update(20.0); + assert_eq!( + crate::SingleSubpopulationAggregate::query(&acc, Statistic::Count, None).unwrap(), + 2.0 + ); + } + #[test] fn test_sum_accumulator_merge() { let acc1 = SumAccumulator::with_sum(10.0); diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index a710e57a..f9583d60 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -112,7 +112,7 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `AggregationConfig` via + /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via /// `PolicyRegistry::get(output.policy_fp)`. The legacy /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` /// is the only identity handle on `PrecomputedOutput`. Outputs @@ -338,7 +338,7 @@ mod tests { use crate::precompute_engine::operators::{DDSketchAccumulator, SumAccumulator}; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -384,11 +384,11 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> AggregationConfig { + fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers obtain the id // via `config.policy_fp_u64()`. - AggregationConfig { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs new file mode 100644 index 00000000..ee76c8b4 --- /dev/null +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -0,0 +1,295 @@ +//! Bind raw ingestion to a selected Planner producer and its raw dependency edge. +use super::accumulator_factory::{create_planner_accumulator, AccumulatorUpdater}; +use crate::storage_engines::types::KeyByLabelValues; +use asap_types::{executable_plan::BackendNodeBinding, PrecomputeMaterialization}; +use planner_types::post_asap::{ + EdgeRole, ExecutableOperatorPayload, GroupingStrategy, PostAsapNodeId, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; +use planner_types::pre_asap::{ColumnRef, QueryExpr, Source}; +use std::collections::HashMap; + +/// A validated executable projection; semantics come from the installed node. +/// The retained node ID makes failures attributable to the selected DAG. +#[derive(Debug, Clone)] +pub struct RawDagProgram { + pub node: PostAsapNodeId, + pub family: SummaryFamilyType, + pub input: SummaryUpdate, + pub grouping: GroupingStrategy, + pub reduction: planner_types::pre_asap::Reduction, + projected_column: Option, +} + +impl RawDagProgram { + pub fn from_plan( + plan: &asap_types::precompute_plan::PrecomputePlan, + config: &PrecomputeMaterialization, + ) -> Result { + let mut selected: Option = None; + for installed in plan.executable_dags.values() { + installed.validate()?; + let dag = installed.document.decode()?; + for node in &dag.nodes { + if !matches!(installed.binding.node(node.id), Some(BackendNodeBinding::Materialization { summary_definition }) if summary_definition.fingerprint() == config.policy_fingerprint()) + { + continue; + } + let ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + reduction, + } = &node.payload + else { + return Err( + "raw materialization binding must identify a Planner SummaryAgg".into(), + ); + }; + if config.derived_input.is_some() { + return Err("derived producer must execute through maintenance DAG".into()); + } + let incoming: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node.id).collect(); + let [edge] = incoming.as_slice() else { + return Err("raw SummaryAgg must have exactly one DAG input".into()); + }; + if edge.role != EdgeRole::Input { + return Err("raw SummaryAgg input edge has wrong role".into()); + } + let source = dag + .nodes + .iter() + .find(|n| n.id == edge.producer) + .ok_or("missing raw DAG input")?; + let ExecutableOperatorPayload::Fallback { expression } = &source.payload else { + return Err("raw producer requires an executable source input; maintenance edges cannot be bypassed".into()); + }; + let scan = match expression { + QueryExpr::TimeRange { child, .. } => child.as_ref(), + source => source, + }; + match scan { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + .. + } if metric == &config.metric => { + let (metric, window, filter) = + control_plane::physical::compiler::raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + )?; + if metric != config.metric + || window.is_some_and(|seconds| seconds != config.window_size) + || asap_types::utils::normalize_spatial_filter(&filter) + != config.spatial_filter_normalized + { + return Err( + "raw DAG source filter/window differs from physical binding".into(), + ); + } + } + QueryExpr::Scan { + source: Source::Table { table_ref }, + .. + } if config.table_name.as_ref() == Some(table_ref) => { + return Err( + "raw table execution requires a validated table scan executor".into(), + ); + } + _ => return Err("raw DAG input does not match installed source routing".into()), + } + if let planner_types::pre_asap::Reduction::Reduce(keys) = reduction { + if keys.is_without() { + return Err( + "raw without reduction requires explicit dynamic population routing" + .into(), + ); + } + let names = keys + .keys() + .iter() + .map(|id| { + source + .output_schema + .fields + .get(*id) + .map(|f| f.name.clone()) + .ok_or("missing reduction column") + }) + .collect::, _>>()?; + if names != config.grouping_labels.names() { + return Err("DAG reduction differs from physical population binding".into()); + } + } + if let SummaryFamilyType::ExactAggregate(kind, _) = family { + if input.item.is_some() { + return Err( + "raw exact populations must follow Planner reduction, not an item map" + .into(), + ); + } + if !matches!(input.weight, SummaryInputExpr::Column(_)) + && !(matches!(kind, planner_types::post_asap::ExactKind::Count) + && input.weight == SummaryInputExpr::Constant(1.0)) + { + return Err("raw exact update differs from stored source projection".into()); + } + } + if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { + return Err( + "materialization storage family differs from selected Planner node".into(), + ); + } + // The stored descriptor must name the same update semantics; its + // content identity cannot be reused for an unrelated DAG program. + let update_matches = match (&input.weight, config.sample_update_rule()) { + ( + SummaryInputExpr::Column(_), + asap_types::SampleUpdateRule::Value { scale }, + ) => scale == 1.0, + (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { + *value == 1.0 + } + ( + SummaryInputExpr::ResetAwareCounterDelta { .. }, + asap_types::SampleUpdateRule::CounterDelta { scale }, + ) => scale == 1_000_000.0, + _ => { + asap_types::accumulator_spec::is_unit_sample_frequency(input) + || (matches!( + family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ) + ) && input.weight == SummaryInputExpr::Constant(1.0)) + } + }; + if !update_matches { + return Err("DAG update differs from stored summary identity".into()); + } + let program = Self { + node: node.id, + family: family.clone(), + input: input.clone(), + grouping: grouping.clone(), + reduction: reduction.clone(), + projected_column: config + .effective_value_projection() + .column() + .map(str::to_owned), + }; + program.validate()?; + if let Some(old) = &selected { + if old.family != program.family + || old.input != program.input + || old.grouping != program.grouping + || old.reduction != program.reduction + { + return Err( + "one stored definition is bound to incompatible Planner producers" + .into(), + ); + } + } else { + selected = Some(program); + } + } + } + selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) + } + + pub fn updater(&self) -> Result, String> { + create_planner_accumulator(&self.family, &self.input, &self.grouping) + } + + fn validate(&self) -> Result<(), String> { + match &self.input.weight { + SummaryInputExpr::Column(ColumnRef::SampleValue) | SummaryInputExpr::Constant(_) => {} + SummaryInputExpr::Column( + ColumnRef::Named(name) | ColumnRef::Qualified { name, .. }, + ) if self.projected_column.as_ref() == Some(name) => {} + SummaryInputExpr::ResetAwareCounterDelta { + value: ColumnRef::SampleValue, + series: planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + } if excluding.is_empty() => {} + _ => return Err("raw DAG weight expression is unsupported".into()), + } + fn item(expr: &SummaryInputExpr) -> bool { + match expr { + SummaryInputExpr::Column(ColumnRef::Named(_) | ColumnRef::SampleValue) => true, + SummaryInputExpr::Tuple(items) => items.iter().all(item), + SummaryInputExpr::EntityIdentity( + planner_types::post_asap::EntityIdentity::PromqlLabelSet { excluding }, + ) => excluding.is_empty(), + _ => false, + } + } + if self.input.item.as_ref().is_some_and(|e| !item(e)) { + return Err("raw DAG item expression is unsupported".into()); + } + self.updater().map(|_| ()) + } + + pub fn uses_counter_delta(&self) -> bool { + matches!( + self.input.weight, + SummaryInputExpr::ResetAwareCounterDelta { .. } + ) + } + + pub fn apply( + &self, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + ) -> Result<(), String> { + let weight = match &self.input.weight { + SummaryInputExpr::Constant(c) => *c, + // The worker retains one previous value per series across pane rotation. + SummaryInputExpr::Column(_) | SummaryInputExpr::ResetAwareCounterDelta { .. } => value, + _ => return Err("unsupported raw weight expression".into()), + }; + let scalar_frequency = asap_types::accumulator_spec::is_unit_sample_frequency(&self.input) + && !updater.is_keyed(); + let weight = if scalar_frequency { value } else { weight }; + updater.validate_single_input(weight)?; + if updater.is_keyed() { + let labels = super::worker::parse_labels_from_series_key(series); + fn eval( + expr: &SummaryInputExpr, + series: &str, + value: f64, + labels: &HashMap<&str, &str>, + ) -> Result, String> { + Ok(match expr { + SummaryInputExpr::EntityIdentity(_) => vec![series.to_owned()], + SummaryInputExpr::Column(ColumnRef::SampleValue) => vec![value.to_string()], + SummaryInputExpr::Column(ColumnRef::Named(name)) => vec![labels + .get(name.as_str()) + .map(|s| super::worker::decode_label_value(s).into_owned()) + .ok_or_else(|| format!("missing DAG item column {name}"))?], + SummaryInputExpr::Tuple(items) => items + .iter() + .map(|i| eval(i, series, value, labels)) + .collect::, _>>()? + .into_iter() + .flatten() + .collect(), + _ => return Err("unsupported raw item expression".into()), + }) + } + let item = self + .input + .item + .as_ref() + .ok_or("keyed DAG kernel requires an explicit item")?; + let key = KeyByLabelValues::new_with_labels(eval(item, series, value, &labels)?); + updater.update_keyed(&key, weight, timestamp); + } else { + updater.update_single(weight, timestamp); + } + Ok(()) + } +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 751f47e5..01af6231 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -20,7 +20,7 @@ use xxhash_rust::xxh64::xxh64; /// hashing or pane lookup. `group_key` and `policy_fp` still travel /// alongside the sid: `group_key` is consumed at emit-time to render the /// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `AggregationConfig` from the hot-reload snapshot (window +/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window /// shape, late-data policy, etc.). Together they let the worker key state /// by sid without losing the data the legacy `(agg_id, group_key)` shape /// carried. @@ -50,8 +50,8 @@ pub enum WorkerMessage { /// `(metric, attrs_fingerprint, agg_kind_canonical)` — see /// `SeriesIdResolver::resolve`. Worker keys `group_states` on this. sid: u64, - /// Source `AggregationConfig` fingerprint. Worker looks up its - /// `AggregationConfig` (window size, sketch kind/config, late + /// Source `PrecomputeMaterialization` fingerprint. Worker looks up its + /// `PrecomputeMaterialization` (window size, sketch kind/config, late /// data policy, etc.) via `snap.get_aggregation_config(policy_fp.as_u64())`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons (e.g. "constant"). @@ -78,7 +78,7 @@ pub enum WorkerMessage { AccumulatorInput { /// Registry-allocated bucket identity; see `GroupSamples::sid`. sid: u64, - /// Source `AggregationConfig` fingerprint; see + /// Source `PrecomputeMaterialization` fingerprint; see /// `GroupSamples::policy_fp`. policy_fp: PolicyFingerprint, /// Grouping label values joined by semicolons, matching the diff --git a/data_plane/src/precompute_engine/window_manager.rs b/data_plane/src/precompute_engine/window_manager.rs index afccd016..e1214fd9 100644 --- a/data_plane/src/precompute_engine/window_manager.rs +++ b/data_plane/src/precompute_engine/window_manager.rs @@ -18,7 +18,7 @@ pub struct WindowManager { impl WindowManager { /// Create a new WindowManager. /// - /// `window_size_secs` and `slide_interval_secs` come from `AggregationConfig` + /// `window_size_secs` and `slide_interval_secs` come from `PrecomputeMaterialization` /// (which stores them in seconds). They are converted to milliseconds internally. pub fn new(window_size_secs: u64, slide_interval_secs: u64) -> Self { Self::with_origin(window_size_secs, slide_interval_secs, None) diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index 2c48006f..c6aac060 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -1,6 +1,6 @@ -use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, -}; +#[cfg(test)] +use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; +use crate::precompute_engine::accumulator_factory::AccumulatorUpdater; use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::group_key::GroupKey; use crate::precompute_engine::metrics::record_late_input; @@ -11,7 +11,7 @@ use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, }; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; @@ -37,10 +37,11 @@ use tracing::{debug, debug_span, info, warn}; /// producing one output per (sid, window) — exactly like Arroyo's /// `GROUP BY window, key`. struct GroupState { + program: Option>, series_id: u64, catalog_generation: Option>, input_revisions: BTreeMap>, - config: Arc, + config: Arc, /// Source policy fingerprint that minted this sid. Held so /// `evict_orphaned_groups` can check liveness against the streaming /// config snapshot (a sid stays alive only while its source policy is @@ -411,7 +412,7 @@ impl Worker { /// /// B7.6 — buckets are now keyed by `sid` (a single u64) rather than /// `(agg_id, group_key)`. `policy_fp` is the source config's - /// fingerprint, used to fetch the `AggregationConfig` from the + /// fingerprint, used to fetch the `PrecomputeMaterialization` from the /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// @@ -425,12 +426,16 @@ impl Worker { sid: u64, policy_fp: PolicyFingerprint, group_key: &Arc, - ) -> Option<&mut GroupState> { + ) -> Result, String> { if !self.group_states.contains_key(&sid) { let snap = self.hot_reload.snapshot(); - let cfg = snap.get_aggregation_config(policy_fp.as_u64())?; + let Some(cfg) = snap.get_aggregation_config(policy_fp.as_u64()) else { + return Ok(None); + }; + let program = snap.raw_programs.get(&policy_fp.as_u64()).cloned(); let config = Arc::new(cfg.clone()); let gs = GroupState { + program, series_id: sid, catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), @@ -454,7 +459,7 @@ impl Worker { self.group_count .store(self.group_states.len(), Ordering::Relaxed); } - self.group_states.get_mut(&sid) + Ok(self.group_states.get_mut(&sid)) } /// Process a batch of samples for a specific sid bucket. @@ -462,7 +467,7 @@ impl Worker { /// /// This is the core of the Arroyo-equivalent GROUP BY logic. /// B7.6 — buckets are keyed by `sid`; `policy_fp` is the source - /// `AggregationConfig` fingerprint used to resolve the bucket's + /// `PrecomputeMaterialization` fingerprint used to resolve the bucket's /// config on first sight; `group_key` is held on the resulting /// `GroupState` for emit-time label rendering. pub fn process_group_samples( @@ -479,7 +484,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -489,6 +494,10 @@ impl Worker { return Ok(()); } let state = self.group_states.get_mut(&sid).unwrap(); + #[cfg(not(test))] + if state.program.is_none() { + return Err("raw precompute requires an installed post-ASAP DAG producer".into()); + } // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. @@ -537,12 +546,19 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = - if let SampleUpdateRule::CounterDelta { .. } = state.config.sample_update_rule() { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), + ) { + reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) + } else { + Some(*val) + }; for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -589,9 +605,14 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } + if state.program.as_deref().map_or_else( + || { + matches!( + state.config.sample_update_rule(), + SampleUpdateRule::CounterDelta { .. } + ) + }, + |p| p.uses_counter_delta(), ) { if let Some(input) = state.input_revisions.get_mut(&bucket_start) { Arc::make_mut(input).first_revision = 0; @@ -600,8 +621,16 @@ impl Worker { continue; } record_late_input("append_correction", "raw_sample"); - let mut updater = create_accumulator_updater(&state.config); - apply_sample(&mut *updater, series_key, *val, *ts, &state.config); + let mut updater = + installed_updater(state.program.as_deref(), &state.config)?; + apply_installed_sample( + state.program.as_deref(), + &mut *updater, + series_key, + *val, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { @@ -646,12 +675,21 @@ impl Worker { // only closes an idle pane, not a long-running bulk ingest whose // records share one event timestamp. state.touch_pane(bucket_start, now_ms); - let updater = state - .active_panes - .entry(bucket_start) - .or_insert_with(|| create_accumulator_updater(&state.config)); + if let std::collections::btree_map::Entry::Vacant(entry) = + state.active_panes.entry(bucket_start) + { + entry.insert(installed_updater(state.program.as_deref(), &state.config)?); + } + let updater = state.active_panes.get_mut(&bucket_start).unwrap(); if let Some(value) = value { - apply_sample(&mut **updater, series_key, value, *ts, &state.config); + apply_installed_sample( + state.program.as_deref(), + &mut **updater, + series_key, + value, + *ts, + &state.config, + )?; if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) { observer.observe( @@ -767,7 +805,7 @@ impl Worker { let now_ms = (self.now_ms_fn)(); if self - .get_or_create_group_state(sid, policy_fp, group_key) + .get_or_create_group_state(sid, policy_fp, group_key)? .is_none() { warn!( @@ -1349,7 +1387,10 @@ pub fn extract_metric_name(series_key: &str) -> &str { /// aggregation config's `grouping_labels`. /// /// The series key format is: `metric_name{label1="val1",label2="val2",...}` -pub fn extract_key_from_series(series_key: &str, config: &AggregationConfig) -> KeyByLabelValues { +pub fn extract_key_from_series( + series_key: &str, + config: &PrecomputeMaterialization, +) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1492,19 +1533,61 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { std::borrow::Cow::Owned(out) } +fn installed_updater( + program: Option<&super::raw_dag::RawDagProgram>, + config: &PrecomputeMaterialization, +) -> Result, String> { + if let Some(program) = program { + return program.updater(); + } + #[cfg(test)] + { + Ok(create_fixture_accumulator(config)) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + +fn apply_installed_sample( + program: Option<&super::raw_dag::RawDagProgram>, + updater: &mut dyn AccumulatorUpdater, + series: &str, + value: f64, + timestamp: i64, + config: &PrecomputeMaterialization, +) -> Result<(), String> { + if let Some(program) = program { + return program.apply(updater, series, value, timestamp); + } + #[cfg(test)] + { + apply_sample(updater, series, value, timestamp, config); + Ok(()) + } + #[cfg(not(test))] + { + let _ = config; + Err("missing installed Planner producer".into()) + } +} + /// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. /// /// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted /// from the series' **aggregated_labels** — these are the labels that become /// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a MultipleSumAccumulator's HashMap). This matches the Arroyo SQL +/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL /// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +#[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) { if updater.is_keyed() { // Planner's PromQL Top-K item is the series identity. When no @@ -1529,7 +1612,7 @@ pub(crate) fn apply_sample( /// Convert a cumulative counter sample into a non-negative, reset-aware /// increment. Only the immediately preceding sample per series is retained; /// pane rotation therefore cannot lose the boundary increment. -fn reset_aware_counter_delta( +pub(crate) fn reset_aware_counter_delta( previous: &mut HashMap, series_key: &str, value: f64, @@ -1560,7 +1643,7 @@ fn reset_aware_counter_delta( /// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. fn extract_aggregated_key_from_series( series_key: &str, - config: &AggregationConfig, + config: &PrecomputeMaterialization, ) -> KeyByLabelValues { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); @@ -1792,7 +1875,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::operators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; - use crate::precompute_engine::operators::multiple_sum_accumulator::MultipleSumAccumulator; + use crate::precompute_engine::operators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use crate::precompute_engine::operators::sum_accumulator::SumAccumulator; use crate::precompute_engine::output_sink::CapturingOutputSink; use crate::storage_engines::types::StreamingConfig; @@ -1808,7 +1891,7 @@ mod tests { window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { make_agg_config_full( id, metric, @@ -1831,7 +1914,7 @@ mod tests { slide_secs: u64, grouping: Vec<&str>, aggregated: Vec<&str>, - ) -> AggregationConfig { + ) -> PrecomputeMaterialization { // `_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. Callers below build the // streaming-config map by reading `config.policy_fp_u64()` @@ -1841,7 +1924,7 @@ mod tests { } else { WindowKind::Sliding }; - AggregationConfig::new( + PrecomputeMaterialization::new( agg_type, agg_sub_type.to_string(), HashMap::new(), @@ -1861,7 +1944,7 @@ mod tests { } fn make_worker( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1871,7 +1954,7 @@ mod tests { } fn make_worker_with_lateness( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, pass_raw: bool, raw_agg_id: u64, @@ -1900,12 +1983,12 @@ mod tests { } /// Build a fresh `StreamingConfigHandle` from a map of agg_id - /// → AggregationConfig. Worker::new takes this handle instead of - /// the old `HashMap>`. Tests use this + /// → PrecomputeMaterialization. Worker::new takes this handle instead of + /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( - configs: HashMap, + configs: HashMap, ) -> crate::storage_engines::types::StreamingConfigHandle { crate::storage_engines::types::StreamingConfigHandle::new( crate::storage_engines::types::StreamingConfig::new(configs), @@ -1991,7 +2074,7 @@ mod tests { assert_eq!(output.start_timestamp as i64, *ts); assert_eq!(output.end_timestamp as i64, *ts); // Raw mode emits PolicyFingerprint::UNSET (no source - // AggregationConfig in the raw-mode fast path). The sink + // PrecomputeMaterialization in the raw-mode fast path). The sink // drops UNSET outputs with a warn — verified separately // via integration tests. assert!(output.policy_fp.is_unset()); @@ -2452,7 +2535,7 @@ mod tests { #[test] fn test_keyed_accumulator_aggregated_labels() { // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside MultipleSumAccumulator) + // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) let config = make_agg_config_full( 3, "cpu", @@ -2504,10 +2587,10 @@ mod tests { let (_output, acc) = &captured[0]; let ms_acc = acc .as_any() - .downcast_ref::() - .expect("should be MultipleSumAccumulator"); + .downcast_ref::() + .expect("should be KeyedSumCountAccumulator"); - // The MultipleSumAccumulator should have two internal keys: "A" and "B" + // The KeyedSumCountAccumulator should have two internal keys: "A" and "B" assert_eq!(ms_acc.sums.len(), 2, "two host keys inside one accumulator"); let mut found_a = false; @@ -2680,100 +2763,15 @@ mod tests { // ----------------------------------------------------------------------- #[test] - fn test_worker_from_streaming_config_yaml() { - let yaml = r#" -aggregations: -- aggregationType: SingleSubpopulation - aggregationSubType: Sum - labels: - grouping: [] - rollup: [] - aggregated: [] - metric: requests_total - parameters: {} - tumblingWindowSize: 10 - windowSize: 10 - windowType: tumbling - slideInterval: 0 - spatialFilter: '' -"#; - - let data: serde_yaml::Value = serde_yaml::from_str(yaml).expect("valid YAML"); - let streaming_config = - StreamingConfig::from_yaml_data(&data).expect("valid streaming config"); - - // PR 5: the streaming-config key is the policy fingerprint. - let agg_id = *streaming_config - .materializations() - .keys() - .next() - .expect("one agg"); - assert!(streaming_config.contains(agg_id)); - - let agg_configs = streaming_config.materializations().clone(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - let pf = PolicyFingerprint(agg_id); - let sid = 1_u64; - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(1_000, 3.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(5_000, 4.0)]), - ) - .unwrap(); - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(9_000, 5.0)]), - ) - .unwrap(); - assert_eq!(sink.len(), 0); - - worker - .process_group_samples( - sid, - pf, - &test_group_key(""), - group_samples("requests_total", vec![(10_000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - - let (output, acc) = &captured[0]; - let _ = agg_id; - assert!(!output.policy_fp.is_unset()); - assert_eq!(output.start_timestamp, 0); - assert_eq!(output.end_timestamp, 10_000); - - let sum_acc = acc - .as_any() - .downcast_ref::() - .expect("should be SumAccumulator"); - assert!( - (sum_acc.sum - 12.0).abs() < 1e-10, - "sum should be 3+4+5=12, got {}", - sum_acc.sum - ); + fn test_worker_rejects_flat_streaming_config_yaml() { + let data = + serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); + assert!(StreamingConfig::from_yaml_data(&data).is_err()); } #[test] fn test_extract_key_from_series() { - let config = AggregationConfig::new( + let config = PrecomputeMaterialization::new( AggregationType::SingleSubpopulation, "Sum".to_string(), HashMap::new(), @@ -3413,7 +3411,7 @@ aggregations: /// Build a worker with explicit wall-clock closure grace values. fn make_worker_with_wall_clock_policy( - agg_configs: HashMap, + agg_configs: HashMap, sink: Arc, late_data_policy: LateDataPolicy, idle_grace_period_ms: i64, @@ -4278,3 +4276,179 @@ aggregations: ); } } + +#[cfg(test)] +mod dag_execution_tests { + use super::*; + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::precompute_engine::output_sink::CapturingOutputSink; + use crate::storage_engines::types::StreamingConfig; + use asap_types::query_plan::ExactReadout; + + fn plan(query: &str) -> control_plane::physical::compiler::CompiledPhysicalPlan { + let mut json: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut item = json["query_workload"]["repeating_queries"][0].clone(); + item["query"] = query.into(); + json["query_workload"]["repeating_queries"] = serde_json::json!([item]); + let snapshot = serde_json::from_value(json).unwrap(); + crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap() + } + + // A selected producer must govern updates, persisted family, and query readout. + #[test] + fn installed_dag_ingestion_persistence_and_readout() { + for (query, readout, answer) in [ + ( + "sum_over_time(asap_demo_gauge[5s])", + ExactReadout::Sum, + 54.0, + ), + ( + "count_over_time(asap_demo_gauge[5s])", + ExactReadout::Count, + 5.0, + ), + ("min_over_time(asap_demo_gauge[5s])", ExactReadout::Min, 3.0), + ( + "max_over_time(asap_demo_gauge[5s])", + ExactReadout::Max, + 20.0, + ), + ("rate(asap_demo_counter_total[5s])", ExactReadout::Rate, 5.5), + ( + "increase(asap_demo_counter_total[5s])", + ExactReadout::Increase, + 27.5, + ), + ] { + let plan = plan(query); + let config = plan + .precompute_plan + .materializations + .first() + .expect("ASAP producer required") + .clone(); + let fp = config.policy_fingerprint(); + let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); + let doc = serde_json::to_value(&streaming).unwrap(); + assert!(doc.get("aggregation_configs").is_none()); + let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let sink = Arc::new(CapturingOutputSink::new()); + let (_tx, rx) = mpsc::channel(8); + let mut worker = Worker::new( + 0, + rx, + sink.clone(), + StreamingConfigHandle::new(streaming), + WorkerRuntimeConfig { + max_buffer_per_series: 100, + allowed_lateness_ms: 10_000, + pass_raw_samples: false, + raw_mode_aggregation_id: 0, + late_data_policy: LateDataPolicy::Drop, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + }, + Arc::new(AtomicUsize::new(0)), + Arc::new(AtomicI64::new(0)), + ); + worker + .process_group_samples( + 1, + fp, + &Arc::new(GroupKey::new([])), + [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, v)| (config.metric.clone(), t, v)) + .collect(), + ) + .unwrap(); + worker.force_close_all().unwrap(); + let mut states = BTreeMap::new(); + for (output, state) in sink.drain() { + let select = if matches!( + config.window_layout, + asap_types::WindowMaterializationLayout::FullWindow + ) { + output.start_timestamp == 0 && output.end_timestamp == 5000 + } else { + output.end_timestamp <= 5000 + }; + if select { + assert_eq!( + state.get_accumulator_type().planner_exact_family(), + Some(readout.planner_family()) + ); + let restored = + ExactAccumulator::deserialize_from_bytes(&state.serialize_to_bytes()) + .unwrap(); + states.insert( + output.end_timestamp as i64, + Arc::new(restored) as Arc, + ); + } + } + assert!(!states.is_empty(), "{query}: no stored states"); + let group = + crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { + entries: vec![std::rc::Rc::new(states)], + agg_type: config.aggregation_type, + }; + assert_eq!( + group.exact_value_for(readout, &None, 0, 5000), + Some(answer), + "{query}" + ); + } + } + + // A flat config and a DAG whose producer no longer matches its binding cannot install. + #[test] + fn execution_requires_matching_dag_producer() { + assert!(serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err()); + let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; + plan.executable_dags.clear(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("DAG producer")); + } + // Changing a raw update must not silently reuse the original summary identity. + #[test] + fn altered_dag_update_cannot_reuse_a_stored_definition() { + let mut plan = plan("sum_over_time(asap_demo_gauge[5s])").precompute_plan; + let installed = plan.executable_dags.values_mut().next().unwrap(); + let mut dag = installed.document.decode().unwrap(); + for node in &mut dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + input, .. + } = &mut node.payload + { + input.weight = planner_types::post_asap::SummaryInputExpr::Constant(99.0); + } + } + installed.document = asap_types::executable_plan::OwnedPostAsapDag::from_executable( + installed.document.query_id.clone(), + &dag, + ) + .unwrap(); + assert!(StreamingConfig::from_precompute_plan(plan) + .unwrap_err() + .to_string() + .contains("update")); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index 19486299..8b454a6b 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -6,7 +6,7 @@ use std::collections::{BTreeMap, BTreeSet}; use asap_types::query_plan::{ExactReadout, QueryPlanEntry, QueryPlanNode, QueryReadout}; use asap_types::sds::{SummaryDefinitionId, SummaryDescriptor, SummaryOperator}; use asap_types::summary_catalog::SummaryCatalog; -use asap_types::AggregationType; +use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; use crate::query_engines::EngineError; @@ -51,64 +51,40 @@ impl ResolvedMaterialization<'_> { matches!( &self.summary.operator, SummaryOperator::Configured { - aggregation_type: AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease - | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleMin - | AggregationType::MultipleMax, + family: SummaryFamilyType::ExactAggregate(..), .. } ) } fn supports(&self, node: &QueryPlanNode) -> bool { - let SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - .. - } = &self.summary.operator - else { + let SummaryOperator::Configured { family, .. } = &self.summary.operator else { // Partial legacy descriptors cannot attest a configured capability. return false; }; - use AggregationType::*; match node { - QueryPlanNode::ExactReadout { readout, .. } => match readout { - ExactReadout::Sum => matches!(aggregation_type, Sum | MultipleSum), - ExactReadout::Count => *aggregation_type == Sum, - ExactReadout::Increase | ExactReadout::Rate => { - matches!(aggregation_type, Increase | MultipleIncrease) - } - // Direction is the family now -- no `aggregation_sub_type` - // cross-check, and a minimum summary can no longer be - // offered up for a maximum readout. - ExactReadout::Min => matches!(aggregation_type, Min | MultipleMin), - ExactReadout::Max => matches!(aggregation_type, Max | MultipleMax), - }, + QueryPlanNode::ExactReadout { readout, .. } => family == &readout.planner_family(), QueryPlanNode::SummaryEstimate { query, .. } => match query { QueryReadout::Quantile { q } => { q.is_finite() && (0.0..=1.0).contains(q) - && matches!(aggregation_type, DatasketchesKLL | HydraKLL | DDSketch) + && matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Kll | SketchAlgorithm::DDSketch)) + } + QueryReadout::Cardinality => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Hll | SketchAlgorithm::UnivMon)) } - QueryReadout::Cardinality => matches!(aggregation_type, HLL | UnivMon), QueryReadout::FrequencyL2 | QueryReadout::FrequencyEntropy => { - *aggregation_type == UnivMon + matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) } - QueryReadout::PointCount { value: None, .. } if *aggregation_type == UnivMon => { + QueryReadout::PointCount { value: None, .. } if matches!(family, SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &SketchAlgorithm::UnivMon) => { true } - QueryReadout::PointCount { .. } => matches!( - aggregation_type, - CountMinSketch | CountMinSketchWithHeap | CountSketch | CountSketchWithHeap - ), - QueryReadout::TopK { .. } => matches!( - aggregation_type, - CountMinSketchWithHeap | CountSketchWithHeap - ), + QueryReadout::PointCount { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::Cms | SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketch | SketchAlgorithm::CountSketchWithHeap)) + } + QueryReadout::TopK { .. } => { + matches!(family, SummaryFamilyType::Sketch(kind, _) if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + } }, _ => false, } diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 0ba9c966..9a992ca2 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -902,9 +902,9 @@ mod tests { sid: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index a0327fb8..15487d36 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1373,9 +1373,9 @@ mod tests { sid: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), - capability: Some(Capability::ExactAgg(asap_types::AggregationType::Increase)), + capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), agg_kind: AggKind::ExactAgg { - agg_type: asap_types::AggregationType::Increase, + agg_type: asap_types::AggregationType::Rate, parameters_canonical: String::new(), spatial_filter_canonical: String::new(), }, diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 13a19d36..058cd8d8 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -65,8 +65,8 @@ use std::sync::Arc; use crate::query_engines::asap_query_engine::summary_exec::SummaryExecutor; use planner_types::post_asap::{ - ExactKind, ExactParams, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, - SummaryFamilyType, SummaryNode, + ExactKind, SketchAlgorithm, SketchParams, SketchQuery, SummaryExpr, SummaryFamilyType, + SummaryNode, }; use planner_types::pre_asap::{ColumnId, ColumnRef, QueryExpr, Reduction, Source}; @@ -203,11 +203,13 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match agg_type { - AggregationType::Sum - | AggregationType::MultipleSum - | AggregationType::Increase - | AggregationType::MultipleIncrease => asap_types::Statistic::Sum, + let stat = match agg_type.planner_exact_family()? { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, _) => asap_types::Statistic::Sum, + SummaryFamilyType::ExactAggregate(ExactKind::Count, _) => asap_types::Statistic::Count, + SummaryFamilyType::ExactAggregate(ExactKind::Increase, _) => { + asap_types::Statistic::Increase + } + SummaryFamilyType::ExactAggregate(ExactKind::Rate, _) => asap_types::Statistic::Rate, _ => return None, }; let mut merged: Option> = None; @@ -224,9 +226,7 @@ impl GroupState { .ok() } - /// Finalize a compiler-declared exact readout. Rate and increase share - /// reset-aware Increase state physically, but remain distinct operations - /// in QueryPlan so serving never infers semantics from PromQL text. + /// Finalize the Planner-declared exact family with its matching readout. pub fn exact_value_for( &self, readout: asap_types::query_plan::ExactReadout, @@ -237,40 +237,32 @@ impl GroupState { let GroupState::ExactAgg { entries, agg_type } = self else { return None; }; - let stat = match (readout, agg_type) { - (asap_types::query_plan::ExactReadout::Count, AggregationType::Sum) => { - asap_types::Statistic::Count - } - ( - asap_types::query_plan::ExactReadout::Sum, - AggregationType::Sum | AggregationType::MultipleSum, - ) => asap_types::Statistic::Sum, - ( - asap_types::query_plan::ExactReadout::Increase, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Increase, - ( - asap_types::query_plan::ExactReadout::Rate, - AggregationType::Increase | AggregationType::MultipleIncrease, - ) => asap_types::Statistic::Rate, - ( - asap_types::query_plan::ExactReadout::Min, - AggregationType::Min | AggregationType::MultipleMin, - ) => asap_types::Statistic::Min, - ( - asap_types::query_plan::ExactReadout::Max, - AggregationType::Max | AggregationType::MultipleMax, - ) => asap_types::Statistic::Max, - _ => return None, + if agg_type.planner_exact_family().as_ref() != Some(&readout.planner_family()) { + return None; + } + let stat = match readout { + asap_types::query_plan::ExactReadout::Count => asap_types::Statistic::Count, + asap_types::query_plan::ExactReadout::Sum => asap_types::Statistic::Sum, + asap_types::query_plan::ExactReadout::Increase => asap_types::Statistic::Increase, + asap_types::query_plan::ExactReadout::Rate => asap_types::Statistic::Rate, + asap_types::query_plan::ExactReadout::Min => asap_types::Statistic::Min, + asap_types::query_plan::ExactReadout::Max => asap_types::Statistic::Max, }; + let planner_state = entries.iter().flat_map(|w| w.values()).any(|a| { + a.as_any() + .is::() + }); // Temporal exact summaries are the hot path for long-window // dashboards. Merge their concrete, fixed-size states in one batch // instead of allocating a boxed trait object for every pane. - if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease - ) { + if !planner_state + && matches!( + readout, + asap_types::query_plan::ExactReadout::Increase + | asap_types::query_plan::ExactReadout::Rate + ) + { let accumulators = entries .iter() .flat_map(|windows| windows.values()) @@ -286,11 +278,7 @@ impl GroupState { ]); return merged.query_statistic(stat, key, &query_kwargs).ok(); } - if matches!( - agg_type, - AggregationType::Min | AggregationType::MultipleMin - ) && readout == asap_types::query_plan::ExactReadout::Min - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Min { return entries .iter() .flat_map(|windows| windows.values()) @@ -303,11 +291,7 @@ impl GroupState { .into_iter() .reduce(f64::min); } - if matches!( - agg_type, - AggregationType::Max | AggregationType::MultipleMax - ) && readout == asap_types::query_plan::ExactReadout::Max - { + if !planner_state && readout == asap_types::query_plan::ExactReadout::Max { return entries .iter() .flat_map(|windows| windows.values()) @@ -334,9 +318,6 @@ impl GroupState { ("range_end_ms".to_string(), range_end_ms.to_string()), ]); let merged = merged?; - if readout == asap_types::query_plan::ExactReadout::Count { - return merged.aux_stats().count.map(|count| count as f64); - } merged.query_statistic(stat, key, &query_kwargs).ok() } @@ -598,9 +579,13 @@ impl QueryExecutionContext<'_> { }); } Candidate::ExactAgg(agg_type) => { + let exact_family = agg_type.planner_exact_family(); if matches!( - agg_type, - AggregationType::Increase | AggregationType::MultipleIncrease + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Increase | ExactKind::Rate, + _ + )) ) { // Counter pane statistics are sufficient for Prometheus // extrapolatedRate only when no query boundary cuts a @@ -616,12 +601,12 @@ impl QueryExecutionContext<'_> { )); } } - if let Some((reduction, is_min)) = match agg_type { - AggregationType::Min | AggregationType::MultipleMin => Some(( + if let Some((reduction, is_min)) = match exact_family.as_ref() { + Some(SummaryFamilyType::ExactAggregate(ExactKind::Min, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Min, true, )), - AggregationType::Max | AggregationType::MultipleMax => Some(( + Some(SummaryFamilyType::ExactAggregate(ExactKind::Max, _)) => Some(( crate::storage_engines::sketch_db::index::RollupReduction::Max, false, )), @@ -673,8 +658,11 @@ impl QueryExecutionContext<'_> { // counter and extrema state. Additive pane summaries must // remain contiguous because a missing pane is not zero. if matches!( - agg_type, - AggregationType::Sum | AggregationType::MultipleSum + exact_family.as_ref(), + Some(SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count, + _ + )) ) { check_panes(windows.keys().copied().collect())?; } @@ -919,9 +907,15 @@ impl<'a> SummaryExecutor for QueryExecutionContext<'a> { entries.extend(more); } ( - GroupState::ExactAgg { entries, .. }, - GroupState::ExactAgg { entries: more, .. }, + GroupState::ExactAgg { entries, agg_type }, + GroupState::ExactAgg { + entries: more, + agg_type: incoming, + }, ) => { + if agg_type.planner_exact_family() != incoming.planner_exact_family() { + return Err(SummaryExecutorError::UnsupportedFamily); + } entries.extend(more); } // `find_candidates`'s exact-match contract never produces a @@ -1263,29 +1257,19 @@ fn summary_family_matches_sketch( /// parameters, so this is a pure `ExactKind` identity check against the sid's /// `AggregationType`, mirroring the canonical `AggregationType -> /// ExactKind` mapping `asap_types::accumulator_spec` uses on the write -/// side (`Sum|MultipleSum -> ExactKind::Sum`, `Increase|MultipleIncrease -/// -> ExactKind::Increase` — confirmed against that module's own -/// dispatch table rather than invented here). +/// side. Count and Rate remain distinct families even though their runtime +/// accumulators share implementations with Sum and Increase. /// -/// `ExactKind::Count`/`Rate`/`Min`/`Max` are not matched by this legacy -/// family-discovery path. For `Count`/`Rate` the final operation is ambiguous -/// from the stored accumulator alone. `Min`/`Max` were excluded for a reason -/// that no longer holds -- direction used to be unrecoverable once a summary -/// reached `AggKind::ExactAgg`, and is now the family itself -- but admitting -/// them here widens candidate discovery beyond the family split and is left -/// as follow-up. Installed QueryPlans carry an explicit `ExactReadout`, and +/// Installed QueryPlans carry an explicit `ExactReadout`, and /// `read_bound_materialization` serves those forms safely. fn summary_family_matches_exact(family: &SummaryFamilyType, agg_type: AggregationType) -> bool { matches!( - (family, agg_type), - ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - AggregationType::Sum | AggregationType::MultipleSum, - ) | ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - AggregationType::Increase | AggregationType::MultipleIncrease, + family, + SummaryFamilyType::ExactAggregate( + ExactKind::Sum | ExactKind::Count | ExactKind::Increase | ExactKind::Rate, + _ ) - ) + ) && agg_type.planner_exact_family().as_ref() == Some(family) } /// Project a full label-values map down to the requested `by` columns -- @@ -1455,6 +1439,58 @@ mod tests { use planner_types::pre_asap::{Column, DataType, Schema}; use std::rc::Rc; + #[test] + fn keyed_count_state_follows_planner_family_and_query_readout() { + use crate::precompute_engine::operators::KeyedSumCountAccumulator; + use asap_types::query_plan::ExactReadout; + + let key = KeyByLabelValues::new_with_labels(vec!["web".to_string()]); + let mut payload = KeyedSumCountAccumulator::for_family(ExactKind::Count); + payload.update(key.clone(), 10.0); + payload.update(key.clone(), 20.0); + let state = GroupState::ExactAgg { + entries: vec![Rc::new(BTreeMap::from([( + 60_000, + Arc::new(payload) as Arc, + )]))], + agg_type: AggregationType::Count, + }; + assert_eq!( + state.exact_value_for(ExactReadout::Count, &Some(key.clone()), 0, 60_000), + Some(2.0) + ); + assert_eq!( + state.exact_value_for(ExactReadout::Sum, &Some(key), 0, 60_000), + None + ); + } + + #[test] + fn state_merge_rejects_different_planner_families() { + let index = SketchStore::new(); + let context = QueryExecutionContext { + index: &index, + t0_ms: 0, + t1_ms: 60_000, + is_cumulative: true, + allowed_materializations: None, + }; + let states = vec![ + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Rate, + }, + GroupState::ExactAgg { + entries: vec![], + agg_type: AggregationType::Increase, + }, + ]; + assert!(matches!( + context.merge_states(states), + Err(SummaryExecutorError::UnsupportedFamily) + )); + } + #[test] fn pane_only_reads_require_the_planned_evaluation_phase() { let binding = asap_types::query_plan::MaterializationBinding { diff --git a/data_plane/src/query_engines/query_result.rs b/data_plane/src/query_engines/query_result.rs index 08eb75e0..0d46b252 100644 --- a/data_plane/src/query_engines/query_result.rs +++ b/data_plane/src/query_engines/query_result.rs @@ -102,7 +102,7 @@ impl QueryResult { /// Attach an accuracy envelope. Chainable so engine paths /// can build the bare result first and decorate once the - /// `agg_id → AggregationConfig → AccuracyProfile` lookup + /// `agg_id → PrecomputeMaterialization → AccuracyProfile` lookup /// has resolved. pub fn with_accuracy(mut self, envelope: AccuracyEnvelope) -> Self { match &mut self { diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index f3780b04..084c87cc 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -1,5 +1,5 @@ //! `AccuracyProfile` — derived error / confidence bound for each -//! `AggregationConfig`. +//! `PrecomputeMaterialization`. //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. @@ -12,7 +12,7 @@ //! //! ## Scope of this module //! -//! Pure derivation: `derive(&AggregationConfig)` +//! Pure derivation: `derive(&PrecomputeMaterialization)` //! looks at `aggregation_type` and the relevant entries in //! `config.parameters` and returns an `AccuracyProfile`. No //! runtime measurement, no sampling — just the textbook bound. @@ -31,14 +31,14 @@ use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::AggregationType; pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; use planner_types::post_asap::SketchParams as PlannerParams; /// Derive an [`AccuracyProfile`] from a pinned -/// [`AggregationConfig`]. Reads `aggregation_type` and any +/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any /// necessary entries in `parameters`; falls back to exact for /// unknown / legacy variants (harmless — the caller just gets /// "0 error" rather than a panic). @@ -52,7 +52,7 @@ use planner_types::post_asap::SketchParams as PlannerParams; /// ε_st`; the random parts compose in quadrature but the staleness part is /// adversarial, so linear addition is the honest envelope). δ is /// unchanged (staleness is not probabilistic). -pub fn derive(config: &AggregationConfig) -> AccuracyProfile { +pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { let mut profile = derive_sketch_only(config); let eps_st = config .parameters @@ -67,20 +67,20 @@ pub fn derive(config: &AggregationConfig) -> AccuracyProfile { /// Source adapter for installed aggregation configs. pub trait BackendAccuracyProfile { - fn derive(config: &AggregationConfig) -> Self; - fn derive_sketch_only(config: &AggregationConfig) -> Self; + fn derive(config: &PrecomputeMaterialization) -> Self; + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; } impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &AggregationConfig) -> Self { + fn derive(config: &PrecomputeMaterialization) -> Self { derive(config) } - fn derive_sketch_only(config: &AggregationConfig) -> Self { + fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { derive_sketch_only(config) } } /// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { +fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { match config.aggregation_type { AggregationType::UnivMon => AccuracyProfile { epsilon: f64::MAX, @@ -91,13 +91,11 @@ fn derive_sketch_only(config: &AggregationConfig) -> AccuracyProfile { // `DeltaSetAggregator` exact-set-membership family lived // here too before its retirement.) AggregationType::Sum + | AggregationType::Count | AggregationType::Increase + | AggregationType::Rate | AggregationType::Min - | AggregationType::Max - | AggregationType::MultipleSum - | AggregationType::MultipleIncrease - | AggregationType::MultipleMin - | AggregationType::MultipleMax => AccuracyProfile::exact(), + | AggregationType::Max => AccuracyProfile::exact(), AggregationType::CountMinSketch => { let (rows, cols) = cms_params(config); @@ -220,7 +218,7 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { // authority on *accuracy*, not on *construction*. /// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &AggregationConfig) -> (u64, u64) { +fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { let rows = config .parameters .get("d") @@ -234,7 +232,7 @@ fn cms_params(config: &AggregationConfig) -> (u64, u64) { (rows, cols) } -fn hll_precision(config: &AggregationConfig) -> u32 { +fn hll_precision(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("precision") @@ -244,7 +242,7 @@ fn hll_precision(config: &AggregationConfig) -> u32 { .unwrap_or(14) } -fn kll_k(config: &AggregationConfig) -> u32 { +fn kll_k(config: &PrecomputeMaterialization) -> u32 { config .parameters .get("K") @@ -254,7 +252,7 @@ fn kll_k(config: &AggregationConfig) -> u32 { .unwrap_or(200) } -fn ddsketch_alpha(config: &AggregationConfig) -> f64 { +fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { config .parameters .get("alpha") @@ -266,7 +264,7 @@ fn ddsketch_alpha(config: &AggregationConfig) -> f64 { /// from `parameters["heap_size"]` with a default of 100 — /// matches the default the control plane's planner uses when the /// caller didn't override. -fn cms_heap_size(config: &AggregationConfig) -> u64 { +fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { config .parameters .get("heap_size") @@ -373,8 +371,11 @@ mod tests { use serde_json::{json, Value}; use std::collections::HashMap; - fn base_config(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( + fn base_config( + agg_type: AggregationType, + params: HashMap, + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index 0e970b82..ce8e476d 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -11,7 +11,7 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::RwLock; use std::time::{SystemTime, UNIX_EPOCH}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use serde::{Deserialize, Serialize}; use tracing::{debug, warn}; @@ -507,12 +507,12 @@ impl BackfillRegistry { /// control-plane-facing HTTP endpoint can return specific 404 / /// 409 / 400 statuses. `CreateError::UnknownAgg` is no longer /// returned from this method — the caller proves the agg - /// exists by holding the `AggregationConfig` — but the variant + /// exists by holding the `PrecomputeMaterialization` — but the variant /// is kept on the enum for HTTP error-mapping compatibility /// (the handler still produces it when its own lookup misses). pub fn create_checked( &self, - config: &AggregationConfig, + config: &PrecomputeMaterialization, created_at_ms: u64, time_range: (u64, u64), source: BackfillSource, @@ -883,7 +883,9 @@ pub use service::{ default_reader_factory, noop_reader_factory, BackfillService, BackfillServiceConfig, BackfillServiceHandle, ReaderFactory, }; +#[cfg(test)] pub use window_builder::build_backfilled_accumulator; +pub use window_builder::build_dag_accumulator; pub use worker::{BackfillWorker, BackfillWorkerError, WindowProcessor}; #[cfg(test)] diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 430aca74..8bfcf0cf 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -22,10 +22,11 @@ use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; use super::raw_sample_reader::RawSample; +#[cfg(test)] use super::window_builder::build_backfilled_accumulator; use super::worker::WindowProcessor; use super::BackfillRegistry; @@ -39,7 +40,7 @@ use super::BackfillRegistry; /// Kept local to the backfill module (not shared with live) per /// the §5e separation ask; the implementations must stay identical /// by convention. -fn extract_group_key(series_key: &str, config: &AggregationConfig) -> String { +fn extract_group_key(series_key: &str, config: &PrecomputeMaterialization) -> String { let labels = parse_labels_from_series_key(series_key); let mut values = Vec::new(); for label_name in &config.grouping_labels.names() { @@ -71,7 +72,7 @@ fn build_group_key_label_values(group_key: &str) -> KeyByLabelValues { /// live windows occupy the same storage row. fn resolve_backfill_bucket_sid( resolver: &SeriesIdResolver, - config: &AggregationConfig, + config: &PrecomputeMaterialization, series_key: &str, store: Option<&crate::storage_engines::sketch_db::index::SketchStore>, captured_generation: Option<&asap_types::sds::CatalogGeneration>, @@ -124,7 +125,7 @@ fn fallback_bucket_id(group_key: &str) -> u64 { pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest /// `StreamingConfig` at each window to find the - /// `AggregationConfig` for `agg_id`. The snapshot is cheap + /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. config: StreamingConfigHandle, /// Destination for rebuilt windows. Tests may omit it to record registry @@ -185,22 +186,6 @@ impl BackfillWindowProcessor { self.series_resolver = Some(series_resolver); self } - - /// Look up the `AggregationConfig` for `agg_id` in the current - /// `StreamingConfig` snapshot. Returns an error string if the - /// agg has been removed from the config since the job was - /// created — rare but worth handling (e.g. operator retired - /// the agg mid-backfill; the `BackfillWorker` will - /// `mark_failed` the job with this message). - fn config_for_agg( - &self, - agg_id: u64, - ) -> Result> { - let snap = self.config.snapshot(); - snap.get_aggregation_config(agg_id).cloned().ok_or_else(|| { - format!("agg_id {agg_id} not in current StreamingConfig — retired mid-backfill?").into() - }) - } } /// One per-sid bucket assembled by [`BackfillWindowProcessor::process_window`]. @@ -219,7 +204,16 @@ impl WindowProcessor for BackfillWindowProcessor { window_range: (u64, u64), samples: Vec, ) -> Result<(), Box> { - let config = self.config_for_agg(agg_id)?; + let snapshot = self.config.snapshot(); + let config = snapshot + .get_aggregation_config(agg_id) + .cloned() + .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + let program = snapshot.raw_programs.get(&agg_id).cloned(); + #[cfg(not(test))] + if program.is_none() { + return Err("backfill requires a post-ASAP DAG installation".into()); + } // B7.7 — sid-keyed bucketing. Per the schema-retirement #5 // step 6 plan, the backfill processor's per-window grouping is @@ -289,7 +283,18 @@ impl WindowProcessor for BackfillWindowProcessor { for (sid, bucket) in by_bucket { let SidBucket { group_key, samples } = bucket; - let accumulator = build_backfilled_accumulator(&config, &samples); + let accumulator = if let Some(program) = &program { + super::window_builder::build_dag_accumulator(program, &samples)? + } else { + #[cfg(test)] + { + build_backfilled_accumulator(&config, &samples) + } + #[cfg(not(test))] + { + return Err("missing backfill DAG producer".into()); + } + }; // Keyed accumulators (MultipleSubpopulation) carry their // subpopulation keys internally; the PrecomputedOutput's // `key` represents the *group* key (grouping_labels @@ -374,7 +379,7 @@ mod tests { use asap_types::KeyByLabelNames; use std::sync::Arc; - fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str, grouping: Vec<&str>) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed // via `PolicyFingerprint::from_config`. let grouping_labels = if grouping.is_empty() { @@ -382,7 +387,7 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -401,7 +406,7 @@ mod tests { ) } - fn streaming_config_with(config: AggregationConfig) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); Arc::new(StreamingConfig::new(map)) @@ -567,7 +572,7 @@ mod tests { /// when given the same ordered samples. #[test] fn backfill_builds_bit_identical_sum_accumulator_to_live() { - use crate::precompute_engine::accumulator_factory::create_accumulator_updater; + use crate::precompute_engine::accumulator_factory::create_fixture_accumulator; let cfg = sum_config(1, "m", vec![]); @@ -592,7 +597,7 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_accumulator_updater(&cfg); + let mut updater = create_fixture_accumulator(&cfg); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -612,7 +617,7 @@ mod tests { serialisations for SumAccumulator. \ If this test fails, something diverged — check:\n\ (1) Is `build_backfilled_accumulator` still calling \ - `create_accumulator_updater`?\n\ + `create_fixture_accumulator`?\n\ (2) Did a recent change to `SumAccumulator` introduce \ non-deterministic state (e.g. a seed)?\n\ (3) Does `serialize_to_bytes` include any timestamp \ diff --git a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs index 7f9208a1..79d9ccdb 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/raw_sample_reader.rs @@ -43,7 +43,7 @@ pub struct RawSample { /// honour. Exactly one metric name plus zero or more equality /// matchers on grouping labels — no regex, no negation, no /// lexicographic ranges. The control plane picks the subset of -/// `AggregationConfig.grouping_labels` that should gate the read. +/// `PrecomputeMaterialization.grouping_labels` that should gate the read. /// /// Rationale: every supported exact-DB backend (Prometheus, /// ClickHouse, S3+Gorilla) can evaluate this filter efficiently, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index c0b35971..86d64fac 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -342,15 +342,15 @@ mod tests { MockRawSampleReader, RawSample, }; use crate::storage_engines::types::StreamingConfig; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::sync::Mutex; - fn sum_config(_agg_id: u64, metric: &str) -> AggregationConfig { + fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - AggregationConfig::new( + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), std::collections::HashMap::new(), @@ -369,7 +369,7 @@ mod tests { ) } - fn streaming_with(cfg: AggregationConfig) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); Arc::new(StreamingConfig::new(m)) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index 04a4eed3..6bf028c9 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -4,13 +4,16 @@ //! It shares the pure accumulator factory and update primitives with live ingest //! so both paths use the same sketch semantics. +#[cfg(test)] use crate::precompute_engine::accumulator_factory::{ - create_accumulator_updater, AccumulatorUpdater, + create_fixture_accumulator, AccumulatorUpdater, }; +#[cfg(test)] use crate::precompute_engine::worker::apply_sample; use crate::storage_engines::sketch_db::backfill::raw_sample_reader::RawSample; use crate::storage_engines::types::AggregateCore; -use asap_types::aggregation_config::AggregationConfig; +#[cfg(test)] +use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -25,11 +28,12 @@ use asap_types::aggregation_config::AggregationConfig; /// The function is synchronous + pure (no I/O, no async, no global /// state). Suitable to call from inside a `WindowProcessor` /// implementation without worrying about the async runtime. +#[cfg(test)] pub fn build_backfilled_accumulator( - config: &AggregationConfig, + config: &PrecomputeMaterialization, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_accumulator_updater(config); + let mut updater: Box = create_fixture_accumulator(config); for sample in samples { apply_sample( &mut *updater, @@ -45,14 +49,14 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_config() -> AggregationConfig { - AggregationConfig::new( + fn sum_config() -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), @@ -156,3 +160,28 @@ mod tests { assert!(aux.sum == Some(0.0) || aux.sum.is_none()); } } + +/// Backfill uses the same selected DAG producer and update expressions as live input. +pub fn build_dag_accumulator( + program: &crate::precompute_engine::raw_dag::RawDagProgram, + samples: &[RawSample], +) -> Result, String> { + let mut updater = program.updater()?; + let mut previous = std::collections::HashMap::new(); + for sample in samples { + let value = if program.uses_counter_delta() { + crate::precompute_engine::worker::reset_aware_counter_delta( + &mut previous, + &sample.labels, + sample.value, + sample.timestamp_ms, + ) + } else { + Some(sample.value) + }; + if let Some(value) = value { + program.apply(&mut *updater, &sample.labels, value, sample.timestamp_ms)?; + } + } + Ok(updater.take_accumulator()) +} diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index e8f3d873..e696fe52 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -137,7 +137,7 @@ pub enum AggKind { /// Complete resolver identity for a configured materialization. All live and /// replay paths must include policy semantics, not just the sketch family. pub(crate) fn materialization_kind_for_config( - config: &asap_types::aggregation_config::AggregationConfig, + config: &asap_types::aggregation_config::PrecomputeMaterialization, ) -> String { format!( "{}|{}", @@ -149,7 +149,9 @@ pub(crate) fn materialization_kind_for_config( /// Resolve the physical state family produced by a precompute policy. This is /// shared by SID minting and store registration so a sketch policy can never /// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config(config: &asap_types::aggregation_config::AggregationConfig) -> AggKind { +pub fn agg_kind_for_config( + config: &asap_types::aggregation_config::PrecomputeMaterialization, +) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; // HLL is intentionally absent from raw-value accumulator dispatch because @@ -592,7 +594,7 @@ mod tests { #[test] fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::AggregationConfig::new( + let config = asap_types::aggregation_config::PrecomputeMaterialization::new( AggregationType::HLL, String::new(), HashMap::from([("precision".to_string(), serde_json::json!(12))]), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 751b3877..098c9077 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -92,11 +92,12 @@ fn reconstruct_exact_agg( bytes: &[u8], ) -> Option> { use crate::precompute_engine::operators::{ - IncreaseAccumulator, MaxAccumulator, MinAccumulator, MultipleIncreaseAccumulator, - MultipleSumAccumulator, SumAccumulator, + IncreaseAccumulator, KeyedCounterState, KeyedSumCountAccumulator, MaxAccumulator, + MinAccumulator, SumAccumulator, }; use crate::storage_engines::types::AggregateCore; match type_name { + "PlannerExactAccumulatorV1" => crate::precompute_engine::operators::exact_accumulator::ExactAccumulator::deserialize_from_bytes(bytes).ok().map(|a|Box::new(a) as Box), "SumAccumulator" => SumAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), @@ -109,10 +110,12 @@ fn reconstruct_exact_agg( "MaxAccumulator" => MaxAccumulator::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), - "MultipleSumAccumulator" => MultipleSumAccumulator::deserialize_from_bytes(bytes) - .ok() - .map(|a| Box::new(a) as Box), - "MultipleIncreaseAccumulator" => MultipleIncreaseAccumulator::deserialize_from_bytes(bytes) + "KeyedSumCountAccumulator" => { + KeyedSumCountAccumulator::deserialize_from_bytes(bytes) + .ok() + .map(|a| Box::new(a) as Box) + } + "KeyedCounterState" => KeyedCounterState::deserialize_from_bytes(bytes) .ok() .map(|a| Box::new(a) as Box), // The keyed `MultipleMin`/`MultipleMax` forms and the @@ -138,7 +141,7 @@ fn reconstruct_exact_agg( /// so the mint-driven path (B7.6) and the sid-direct path (B7.7) stay /// byte-identical on the values they hand to the index. fn build_attrs_fp_and_label_map( - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Result<(String, BTreeMap), String> { if let Some(labels) = &output.population_labels { @@ -232,7 +235,7 @@ pub struct SummarySeriesMetadata { /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source - /// `AggregationConfig` (test fixtures + the early-Phase-5 sketch + /// `PrecomputeMaterialization` (test fixtures + the early-Phase-5 sketch /// ingest path that didn't thread the config through); the index /// skips those entries — they're reachable through the legacy /// `instances_matching(metric, gbk)` walk if a query needs them. @@ -2301,7 +2304,7 @@ impl SketchStore { } /// Phase 5 M2.3.5 — query the precompute payloads across every sid - /// belonging to one `AggregationConfig` (identified by `metric` + + /// belonging to one `PrecomputeMaterialization` (identified by `metric` + /// `agg_cfg.aggregation_type`), shaped as the legacy `Store` /// trait's `TimestampedBucketsMap`. Lets the query engine swap /// `Store::query_precomputed_output` for `SketchStore` without @@ -3005,7 +3008,7 @@ impl SketchStore { } /// Phase 5 M2.3.6e — write-side helper. Given an - /// `AggregationConfig` and one `(PrecomputedOutput, AggregateCore)` + /// `PrecomputeMaterialization` and one `(PrecomputedOutput, AggregateCore)` /// pair (the shape both the live worker AND the backfill processor /// emit), compute the precompute sid, register a metadata entry on /// first sight, and append the payload window. Used by @@ -3025,7 +3028,7 @@ impl SketchStore { pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3157,7 +3160,7 @@ impl SketchStore { pub fn ingest_precompute_with_series_id( &self, sid: u64, - agg_cfg: &asap_types::aggregation_config::AggregationConfig, + agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3175,6 +3178,18 @@ impl SketchStore { output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + let expected = agg_cfg.accumulator_spec().ok()?.family; + if matches!( + expected, + planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) + ) && accumulator + .get_accumulator_type() + .planner_exact_family() + .as_ref() + != Some(&expected) + { + return None; + } let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; // Keep the physical lifetime alive through publication. Removal takes @@ -6225,4 +6240,91 @@ mod tests { ); assert_eq!(idx.series.len(), 2); } + // Flush and reopen must preserve Planner family rather than reconstructing Rate as Increase. + #[test] + fn planner_exact_families_survive_disk_eviction_and_restart() { + use crate::precompute_engine::operators::exact_accumulator::ExactAccumulator; + use crate::storage_engines::types::{AggregateCore, AggregationType}; + let kinds = [ + AggregationType::Sum, + AggregationType::Count, + AggregationType::Min, + AggregationType::Max, + AggregationType::Rate, + AggregationType::Increase, + ]; + let stats = [ + asap_types::Statistic::Sum, + asap_types::Statistic::Count, + asap_types::Statistic::Min, + asap_types::Statistic::Max, + asap_types::Statistic::Rate, + asap_types::Statistic::Increase, + ]; + let expected = [16.0, 3.0, 2.0, 8.0, 3.0, 6.0]; + let temp = tempfile::tempdir().unwrap(); + { + let store = Arc::new(SketchStore::new()); + for (i, kind) in kinds.iter().enumerate() { + let mut metadata = meta(9000 + i as u64); + metadata.agg_kind = AggKind::ExactAgg { + agg_type: *kind, + parameters_canonical: String::new(), + spatial_filter_canonical: String::new(), + }; + metadata.capability = Some(Capability::ExactAgg(*kind)); + metadata.accuracy = None; + store.register(metadata); + } + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + for window in 0..10u64 { + let mut state = + ExactAccumulator::new(kind.planner_exact_family().unwrap(), false).unwrap(); + for (time, value) in [(1000, 8.0), (2000, 2.0), (3000, 6.0)] { + state.update(None, value, time); + } + store.append_precompute( + 9000 + i as u64, + BTreeMap::new(), + (window * 30000, (window + 1) * 30000), + Box::new(state), + ); + } + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.approx_memory_bytes() == 0 + && store.list_sealed_epochs_len() == 0, + std::time::Duration::from_secs(5) + )); + persistence.shutdown(); + } + let store = Arc::new(SketchStore::new()); + let mut persistence = store + .start_persistence(durable_cfg(temp.path().to_path_buf())) + .unwrap(); + for (i, kind) in kinds.iter().enumerate() { + let series = store.query_exact_agg_range(9000 + i as u64, 0, 30001); + assert_eq!(series.len(), 1, "{kind:?}"); + let state = &series[0].1[&30000]; + assert_eq!(state.get_accumulator_type(), *kind); + assert_eq!( + state + .query_statistic(stats[i], &None, &HashMap::new()) + .unwrap(), + expected[i] + ); + for (j, stat) in stats.iter().enumerate() { + if i != j { + assert!(state + .query_statistic(*stat, &None, &HashMap::new()) + .is_err()); + } + } + } + persistence.shutdown(); + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index d7de8c2b..768f0946 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -195,13 +195,13 @@ mod tests { use super::*; use crate::precompute_engine::operators::SumAccumulator; use crate::storage_engines::types::{AggregationType, StreamingConfig}; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> AggregationConfig { - AggregationConfig { + fn sum_agg_config(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: AggregationType::Sum, aggregation_sub_type: String::new(), diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index f9a83493..0b712f45 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -11,7 +11,7 @@ use std::sync::Arc; use std::time::Duration; use crate::storage_engines::types::StreamingConfig; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; use crate::storage_engines::sketch_db::index::SketchStore; @@ -87,7 +87,7 @@ pub fn reconcile_from_streaming_config( } // `live_signatures` is built exclusively from // `signature_from_agg_config`, which canonicalizes every - // streaming-config `AggregationConfig` to an `AggKind::ExactAgg` + // streaming-config `PrecomputeMaterialization` to an `AggKind::ExactAgg` // signature (`P`-prefixed). An `AggKind::Sketch` sid (OTLP // modified-sketch ingest path: KLL / HLL / DDSketch / CMS / // CountSketch) always produces an `S`-prefixed signature, so it @@ -166,7 +166,7 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &AggregationConfig) -> Vec { +fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { let agg_kind = AggKind::ExactAgg { agg_type: cfg.aggregation_type, parameters_canonical: canonical_parameters(&cfg.parameters), @@ -264,7 +264,7 @@ mod tests { use super::*; use std::collections::HashMap; - use asap_types::aggregation_config::AggregationConfig; + use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -276,8 +276,8 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> AggregationConfig { - AggregationConfig::new( + ) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), HashMap::new(), @@ -321,7 +321,7 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> StreamingConfig { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); diff --git a/data_plane/src/storage_engines/sketch_db/mod.rs b/data_plane/src/storage_engines/sketch_db/mod.rs index e2aff195..0ef39870 100644 --- a/data_plane/src/storage_engines/sketch_db/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/mod.rs @@ -16,12 +16,12 @@ pub mod sds; pub use accuracy::{AccuracyEnvelope, AccuracyKind, AccuracyProfile, PerSegmentAccuracy}; pub use backfill::{ - build_backfilled_accumulator, clickhouse_reader_factory, default_reader_factory, - noop_reader_factory, BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, - BackfillServiceHandle, BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, - BackfillWorkerError, ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, - MockRawSampleReader, PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, - ReaderFactory, WindowProcessor, + build_dag_accumulator, clickhouse_reader_factory, default_reader_factory, noop_reader_factory, + BackfillJob, BackfillRegistry, BackfillService, BackfillServiceConfig, BackfillServiceHandle, + BackfillSource, BackfillStatus, BackfillWindowProcessor, BackfillWorker, BackfillWorkerError, + ClickHouseReaderConfig, Coverage, CreateError, LabelFilter, MockRawSampleReader, + PrometheusReader, RawSample, RawSampleReader, RawSampleReaderError, ReaderFactory, + WindowProcessor, }; pub use lifecycle::{ warn_if_retention_inverted, AggStatus, SchemaEvictionConfig, SchemaEvictionHandle, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index d42457fe..b81cc6a9 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -680,7 +680,7 @@ impl ActivePhysicalPlanHandle { #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::AggregationConfig; + use crate::storage_engines::types::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -755,8 +755,8 @@ mod tests { } } - fn dummy_agg(id: u64) -> AggregationConfig { - AggregationConfig::new( + fn dummy_agg(id: u64) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( AggregationType::Sum, String::new(), HashMap::new(), diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 57d47470..a91e8314 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -25,7 +25,7 @@ pub use streaming_config::*; pub use traits::*; // Cross-module re-export of asap_types data types so callers can -// write `crate::storage_engines::types::AggregationConfig` instead of +// write `crate::storage_engines::types::PrecomputeMaterialization` instead of // reaching across crates. pub use asap_types::aggregation_config::*; diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 4d268dc4..28e71b4c 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -62,7 +62,7 @@ pub struct PrecomputedOutput { #[serde(default)] pub origin: Origin, /// Content-addressed policy identity. The data plane's only handle - /// on which source `AggregationConfig` produced this output. + /// on which source `PrecomputeMaterialization` produced this output. /// `#[serde(default)]` on read preserves forward-compat with /// PR-3 / PR-4-era records that may not have carried the field; /// sinks treat `PolicyFingerprint::UNSET` as "skip this output" @@ -75,7 +75,7 @@ impl PrecomputedOutput { /// Construct a `Native` precompute. /// /// `policy_fp` is the content-addressed handle on the source - /// [`asap_types::AggregationConfig`]; sinks use it to look up the + /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the /// config via `PolicyRegistry::get(policy_fp)`. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs index 430dd10b..95055c2e 100644 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ b/data_plane/src/storage_engines/types/streaming_config.rs @@ -6,31 +6,22 @@ use std::fs::File; use std::io::BufReader; use std::ops::Index; -use asap_types::enums::QueryLanguage; -use asap_types::{AggregationConfig, MonitorSpec, PolicyRegistry}; +use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; use super::storage_backend::StorageBackend; -/// The backend's active streaming policy config: every `AggregationConfig` -/// currently pushed by the controller, plus the storage-backend pin and CDM -/// monitor specs. -/// -/// Formerly `asap_types::streaming_config::StreamingConfig` — moved here -/// (see `scratchpad/artifacts/enum-unification-plan.md`) because -/// `control_plane` never actually depended on this type: its own -/// `StreamingConfigEmitter` hand-builds wire-compatible JSON independently, -/// and `PolicyRegistry::from_streaming_config` (the only thing that made -/// `asap_types::PolicyRegistry` -- genuinely shared -- look coupled to this -/// type) had exactly one real caller, this struct's own `policy_registry()` -/// method below. `asap_types` keeps the lower-level `PolicyRegistry:: -/// from_configs` primitive this method now calls directly. -#[derive(Debug, Clone, Serialize, Deserialize)] +/// DAG installation plus a derived in-memory routing index. The flat index is +/// never serialized as executable configuration. Raw programs are validated +/// and shared once per installed producer across all of its population states. +#[derive(Debug, Clone, Serialize)] pub struct StreamingConfig { - #[serde( - rename = "aggregation_configs", - alias = "materializations_by_policy_fingerprint" - )] - pub materializations_by_policy_fingerprint: HashMap, + #[serde(skip)] + pub(crate) raw_programs: + HashMap>, + /// Authoritative execution configuration: Planner DAGs and physical bindings. + pub precompute_plan: Option, + #[serde(skip)] + pub materializations_by_policy_fingerprint: HashMap, /// Phase-5 capability-routing axis: which storage tier serves this /// per-metric runtime config. The controller pushes this when planning /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 @@ -45,15 +36,60 @@ pub struct StreamingConfig { pub monitors: Vec, } +// Flat aggregation lists are deliberately not an accepted execution document. +impl<'de> Deserialize<'de> for StreamingConfig { + fn deserialize>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Document { + precompute_plan: asap_types::precompute_plan::PrecomputePlan, + #[serde(default)] + storage_backend: StorageBackend, + #[serde(default)] + monitors: Vec, + } + let doc = Document::deserialize(deserializer)?; + let mut config = + Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; + config.storage_backend = doc.storage_backend; + config.monitors = doc.monitors; + Ok(config) + } +} + impl StreamingConfig { - pub fn new(materializations_by_policy_fingerprint: HashMap) -> Self { + pub fn new( + materializations_by_policy_fingerprint: HashMap, + ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: StorageBackend::default(), monitors: Vec::new(), } } + /// Build the routing projection only after validating the DAG installation. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|c| { + c.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::new(materializations); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + /// CDM monitor specs the data-plane coordinator should serve (may be empty). pub fn monitors(&self) -> &[MonitorSpec] { &self.monitors @@ -63,10 +99,12 @@ impl StreamingConfig { /// Used by the controller-driven plan-push path; tests typically /// stay on `Self::new(...)` and let the default land. pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, + materializations_by_policy_fingerprint: HashMap, storage_backend: StorageBackend, ) -> Self { Self { + raw_programs: HashMap::new(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend, monitors: Vec::new(), @@ -80,12 +118,15 @@ impl StreamingConfig { self.storage_backend } - pub fn get_aggregation_config(&self, aggregation_id: u64) -> Option<&AggregationConfig> { + pub fn get_aggregation_config( + &self, + aggregation_id: u64, + ) -> Option<&PrecomputeMaterialization> { self.materializations_by_policy_fingerprint .get(&aggregation_id) } - pub fn materializations(&self) -> &HashMap { + pub fn materializations(&self) -> &HashMap { &self.materializations_by_policy_fingerprint } @@ -126,56 +167,12 @@ impl StreamingConfig { /// (operator-authored query→agg_ids YAML feeding a retention_map) /// is gone — the controller drives capability matching dynamically. pub fn from_yaml_data(data: &Value) -> Result { - let mut materializations_by_policy_fingerprint: HashMap = - HashMap::new(); - - if let Some(aggregations) = data.get("aggregations").and_then(|v| v.as_sequence()) { - for aggregation_data in aggregations { - // Retention comes from each aggregation entry; identity is derived from - // its configuration content. - let num_aggregates_to_retain = aggregation_data - .get("numAggregatesToRetain") - .and_then(|v| v.as_u64()); - let config = AggregationConfig::from_yaml_data( - aggregation_data, - num_aggregates_to_retain, - QueryLanguage::PromQl, - )?; - if !config.population_key_encoding.is_legacy() { - anyhow::bail!( - "legacy streaming input does not support this population key encoding" - ); - } - if config.derived_input.is_some() { - anyhow::bail!( - "legacy streaming input cannot execute a derived summary program" - ); - } - // PR 5: the map key IS the policy-fingerprint u64. - // `AggregationConfig::policy_fp_u64()` is the canonical - // accessor for this value. - materializations_by_policy_fingerprint.insert(config.policy_fp_u64(), config); - } - } - - let mut config = Self::new(materializations_by_policy_fingerprint); - // Continuous-monitoring (CDM) specs: a top-level `monitors:` array, each - // entry deserializing into a MonitorSpec. Absent → empty (the common - // case). The data-plane monitor coordinator reads these. - if let Some(monitors) = data.get("monitors").and_then(|v| v.as_sequence()) { - for m in monitors { - let spec: MonitorSpec = serde_yaml::from_value(m.clone()).map_err(|e| { - anyhow::anyhow!("invalid monitor spec in streaming-config: {e}") - })?; - config.monitors.push(spec); - } - } - Ok(config) + serde_yaml::from_value(data.clone()).map_err(Into::into) } } impl Index for StreamingConfig { - type Output = AggregationConfig; + type Output = PrecomputeMaterialization; fn index(&self, aggregation_id: u64) -> &Self::Output { &self.materializations_by_policy_fingerprint[&aggregation_id] @@ -190,7 +187,7 @@ impl Default for StreamingConfig { impl StreamingConfig { #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { + pub fn get_all_aggregation_configs(&self) -> &HashMap { self.materializations() } } @@ -199,153 +196,16 @@ impl StreamingConfig { mod tests { use super::*; - /// Pre-Phase-5 deploys serialize `StreamingConfig` without the - /// `storage_backend` field; deserialize must default to `SketchStore` - /// so the router keeps dispatching to `ASAPQueryEngine` unchanged. - #[test] - fn deserialize_legacy_yaml_defaults_to_asap_tier() { - let yaml = "{\"aggregation_configs\":{}}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("legacy decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::SketchStore); - } - - #[test] - fn deserialize_with_explicit_double_write_pin() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"double_write\"}"; - let cfg: StreamingConfig = serde_json::from_str(yaml).expect("Phase-5 decode"); - assert_eq!(cfg.storage_backend(), StorageBackend::DoubleWrite); - } - - /// #746 deleted the archive tier; its storage-axis spelling is no longer - /// a known variant, so a stale config naming it fails to decode rather - /// than silently pinning some other tier. + // Old flat lists cannot become execution authority through JSON or YAML. #[test] - fn deserialize_rejects_the_removed_archive_axis() { - let yaml = "{\"aggregation_configs\":{},\"storage_backend\":\"gorilla_object_store\"}"; - assert!(serde_json::from_str::(yaml).is_err()); - } - - #[test] - fn legacy_yaml_rejects_derived_summary_input() { - let data = serde_yaml::from_str::(&format!( - "aggregations:\n- aggregationType: Sum\n aggregationSubType: ''\n metric: outer\n labels: {{grouping: [], rollup: [], aggregated: []}}\n parameters: {{}}\n windowSize: 10\n windowType: tumbling\n spatialFilter: ''\n derived_input:\n inputs: [1]\n program_sha256: '{}'\n", "a".repeat(64) - )).unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!(error - .to_string() - .contains("legacy streaming input cannot execute")); - } - - #[test] - fn legacy_yaml_rejects_canonical_population_key_encoding() { - let data: Value = serde_yaml::from_str( - r#" -aggregations: -- aggregationType: Sum - aggregationSubType: '' - metric: m - population_key_encoding: canonical_labels_v1 - labels: - grouping: [host] - rollup: [] - aggregated: [] - parameters: {} - windowSize: 60 - windowType: tumbling - spatialFilter: '' -"#, - ) - .unwrap(); - let error = StreamingConfig::from_yaml_data(&data).unwrap_err(); - assert!( - error.to_string().contains("population key encoding"), - "{error}" - ); - } - - /// PR 5: a streaming-config YAML that omits `aggregationId` - /// parses correctly — the backend derives identity from content - /// via `PolicyFingerprint::from_config`. The map key is the - /// fingerprint's u64 form. - #[test] - fn from_yaml_data_accepts_entry_without_aggregation_id() { - let yaml = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode without id"); - assert_eq!(cfg.materializations_by_policy_fingerprint.len(), 1); - let (k, v) = cfg - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_ne!(*k, 0, "derived id is not the 0 sentinel"); - assert_eq!(*k, v.policy_fp_u64(), "map key equals fingerprint u64"); - assert_eq!(v.metric, "cpu_seconds"); - } - - /// PR 5: a streaming-config YAML that still spells out - /// `aggregationId: N` parses the SAME as one without — the field - /// is silently dropped. - #[test] - fn from_yaml_data_ignores_explicit_aggregation_id() { - let with = "\ -aggregations:\n\ -- aggregationId: 42\n aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let without = "\ -aggregations:\n\ -- aggregationType: DDSketch\n aggregationSubType: ''\n metric: cpu_seconds\n labels:\n grouping: [host]\n rollup: []\n aggregated: []\n parameters:\n relative_accuracy: 0.01\n windowSize: 30\n windowType: tumbling\n spatialFilter: ''\n"; - let w: Value = serde_yaml::from_str(with).expect("with yaml ok"); - let wo: Value = serde_yaml::from_str(without).expect("without yaml ok"); - let cw = StreamingConfig::from_yaml_data(&w).expect("with"); - let cwo = StreamingConfig::from_yaml_data(&wo).expect("without"); - let (kw, _) = cw - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - let (kwo, _) = cwo - .materializations_by_policy_fingerprint - .iter() - .next() - .unwrap(); - assert_eq!( - kw, kwo, - "explicit aggregationId in YAML must not change identity" - ); - assert_ne!( - *kw, 42, - "the explicit value must NOT leak through as the map key" - ); - } - - #[test] - fn from_yaml_data_parses_monitors_section() { - // CDM monitor specs: a top-level `monitors:` array must populate - // StreamingConfig.monitors (the data-plane coordinator reads these). - let yaml = "\ -aggregations: []\n\ -monitors:\n\ -- agg_id: 16346598078036168951\n key: \"\"\n tau: 5000.0\n epsilon: 0.05\n window_ms: 10000\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode monitors"); - assert_eq!(cfg.monitors().len(), 1, "monitors: section must be parsed"); - let m = &cfg.monitors()[0]; - assert_eq!(m.agg_id, 16346598078036168951); - assert_eq!(m.tau, 5000.0); - assert_eq!(m.window_ms, 10000); - assert_eq!(m.epsilon, 0.05); - } - - #[test] - fn from_yaml_data_absent_monitors_is_empty() { - let yaml = "aggregations: []\n"; - let data: Value = serde_yaml::from_str(yaml).expect("yaml ok"); - let cfg = StreamingConfig::from_yaml_data(&data).expect("decode"); - assert!( - cfg.monitors().is_empty(), - "no monitors: → empty (byte-compat)" - ); + fn rejects_flat_aggregation_documents() { + for text in [ + r#"{"aggregation_configs":{}}"#, + "aggregations: []", + "aggregations: [{aggregationType: Sum, metric: m}]", + ] { + let yaml = serde_yaml::from_str(text).unwrap(); + assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); + } } } diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index ed88c51d..21cd2b11 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,7 +27,7 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::AggregationConfig; +use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -35,8 +35,8 @@ use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> AggregationConfig { - AggregationConfig::new( +fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { + PrecomputeMaterialization::new( agg_type, String::new(), params, diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 5158398b..895df1b4 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -2,14 +2,14 @@ //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore //! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build AggregationConfig with +//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with //! the correct aggregation_type string. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationConfig, AggregationType, KeyByLabelValues, PrecomputedOutput, QueryLanguage, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, StreamingConfig, WindowKind, }; use crate::AggregateCore; @@ -23,7 +23,7 @@ use std::collections::HashMap; fn ingest_with_fresh_resolver( summary_store: &crate::storage_engines::sketch_db::index::SketchStore, resolver: &std::sync::Arc, - agg_cfg: &AggregationConfig, + agg_cfg: &PrecomputeMaterialization, output: &PrecomputedOutput, accumulator: &dyn AggregateCore, ) -> Option { @@ -89,7 +89,7 @@ pub fn create_engine_single_pop_with_aggregated( .collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -119,6 +119,8 @@ pub fn create_engine_single_pop_with_aggregated( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -175,7 +177,7 @@ pub fn create_engine_dual_input( let mut materializations_by_policy_fingerprint = HashMap::new(); // Value aggregation - let value_agg_config = AggregationConfig { + let value_agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: value_agg_type, aggregation_sub_type: String::new(), @@ -205,7 +207,7 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(value_id, value_agg_config); // Keys aggregation - let keys_agg_config = AggregationConfig { + let keys_agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: key_agg_type, aggregation_sub_type: String::new(), @@ -235,6 +237,8 @@ pub fn create_engine_dual_input( materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -300,7 +304,7 @@ pub fn create_engine_two_metrics( let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config_a = AggregationConfig { + let agg_config_a = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: aggregation_type_a, aggregation_sub_type: String::new(), @@ -329,7 +333,7 @@ pub fn create_engine_two_metrics( let id_a = agg_config_a.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_a, agg_config_a); - let agg_config_b = AggregationConfig { + let agg_config_b = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: aggregation_type_b, aggregation_sub_type: String::new(), @@ -359,6 +363,8 @@ pub fn create_engine_two_metrics( materializations_by_policy_fingerprint.insert(id_b, agg_config_b); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -434,7 +440,7 @@ pub fn create_engine_three_metrics( (aggregation_type_b, &labels_b, metric_b), (aggregation_type_c, &labels_c, metric_c), ] { - let cfg = AggregationConfig { + let cfg = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type: agg_type, aggregation_sub_type: String::new(), @@ -466,6 +472,8 @@ pub fn create_engine_three_metrics( } let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -516,7 +524,7 @@ pub fn create_engine_multi_timestamp( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -546,6 +554,8 @@ pub fn create_engine_multi_timestamp( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), @@ -574,7 +584,7 @@ pub fn create_engine_multi_timestamp( /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the AggregationConfig (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -590,7 +600,7 @@ pub fn create_engine_multi_timestamp_with_window( grouping_labels.iter().map(|s| s.to_string()).collect(); let mut materializations_by_policy_fingerprint = HashMap::new(); - let agg_config = AggregationConfig { + let agg_config = PrecomputeMaterialization { population_key_encoding: Default::default(), aggregation_type, aggregation_sub_type: String::new(), @@ -620,6 +630,8 @@ pub fn create_engine_multi_timestamp_with_window( materializations_by_policy_fingerprint.insert(agg_id, agg_config); let streaming_config = Arc::new(StreamingConfig { + raw_programs: Default::default(), + precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), monitors: Vec::new(), diff --git a/data_plane/src/tests/trait_design_tests.rs b/data_plane/src/tests/trait_design_tests.rs index b56408a2..a10cd3d1 100644 --- a/data_plane/src/tests/trait_design_tests.rs +++ b/data_plane/src/tests/trait_design_tests.rs @@ -1,4 +1,4 @@ -use crate::precompute_engine::operators::{MultipleSumAccumulator, SumAccumulator}; +use crate::precompute_engine::operators::{KeyedSumCountAccumulator, SumAccumulator}; #[cfg(test)] use crate::storage_engines::types::{ KeyByLabelValues, MultipleSubpopulationAggregate, SingleSubpopulationAggregate, @@ -18,7 +18,7 @@ fn test_single_subpopulation_interface() { #[test] fn test_multiple_subpopulation_interface() { // Multiple accumulator - matches Python behavior exactly - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let mut key = KeyByLabelValues::new(); key.insert("web".to_string()); @@ -43,7 +43,7 @@ fn test_interface_prevents_misuse() { let single_acc: Box = Box::new(SumAccumulator::with_sum(42.0)); let multi_acc: Box = - Box::new(MultipleSumAccumulator::new()); + Box::new(KeyedSumCountAccumulator::new()); // ✅ These work - correct usage let _result1 = single_acc.query(Statistic::Sum, None); @@ -68,7 +68,7 @@ fn test_python_alignment() { // Python: multiple_accumulator.query(Statistic.SUM, key) // Rust: multiple_accumulator.query(Statistic::Sum, &key) - let mut multi_acc = MultipleSumAccumulator::new(); + let mut multi_acc = KeyedSumCountAccumulator::new(); let key = KeyByLabelValues::new(); multi_acc.add_sum(key.clone(), 100.0); let multi_trait: Box = Box::new(multi_acc); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs index 5150ddf0..6f33ca87 100644 --- a/data_plane/src/utils/file_io.rs +++ b/data_plane/src/utils/file_io.rs @@ -20,7 +20,7 @@ mod tests { use tempfile::NamedTempFile; #[test] - fn test_read_streaming_config() { + fn flat_streaming_file_is_rejected() { // PR 5: `aggregationId: 1` is silently dropped on read — the // streaming-config map key is the policy fingerprint derived // from content. The legacy field stays in this fixture to @@ -47,9 +47,6 @@ aggregations: let mut streaming_temp_file = NamedTempFile::new().unwrap(); write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - let config = read_streaming_config(streaming_temp_file.path().to_str().unwrap()).unwrap(); - assert!(!config.materializations_by_policy_fingerprint.is_empty()); - let agg = config.materializations().values().next().expect("one agg"); - assert_eq!(agg.num_aggregates_to_retain, Some(6)); + assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); } } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index daa4d60f..2afb897c 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -957,7 +957,14 @@ async fn run_shared_dashboard(multi_pane: bool) { snapshot = serde_json::to_value(&typed).unwrap(); let plan = typed.compile_promql().unwrap(); assert!(plan.cost_comparison.is_some()); - assert_eq!(plan.precompute_plan.materializations.len(), 1); + assert_eq!(plan.precompute_plan.materializations.len(), 2); + let families = plan + .precompute_plan + .materializations + .iter() + .map(|m| m.aggregation_type.as_str()) + .collect::>(); + assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); assert!(plan .precompute_plan diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index 1a291c49..b44c3ebe 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -37,7 +37,7 @@ //! the GET endpoint reflects the registered aggregation. //! * Test 2 — same shape with `group_by_labels: ["zone"]`; verifies //! #245's grouping plumb survives the round-trip into the backend's -//! `AggregationConfig.grouping_labels`. +//! `PrecomputeMaterialization.grouping_labels`. //! * Test 3 — full controller-to-query roundtrip: harness simulates //! the agent (builds DDSketch state with `asap_sketchlib`, encodes //! as a modified-OTLP `DdSketchDataPoint`), POSTs sketches to the @@ -45,7 +45,7 @@ //! PromQL, asserts the response is well-formed for the planned //! metric. -use asap_types::AggregationConfig; +use asap_types::PrecomputeMaterialization; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -76,7 +76,7 @@ fn phase_aligned_now_ns() -> u64 { async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[AggregationConfig], + materializations: &[PrecomputeMaterialization], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the @@ -171,7 +171,7 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( @@ -641,7 +641,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { // Verifies #245's grouping plumb survives the controller → backend // round-trip. The workload carries `group_by_labels: ["zone"]`; the // emitted JSON must surface `["zone"]` in `labels.grouping`, the -// backend's parser must materialise it into `AggregationConfig. +// backend's parser must materialise it into `PrecomputeMaterialization. // grouping_labels`, and the active-config snapshot must reflect that. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -1261,7 +1261,7 @@ async fn controller_plan_to_query_full_roundtrip_count_min_sketch() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &AggregationConfig) -> (u32, u32) { +fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { let w = agg.parameters["w"] .as_u64() .expect("materialization must carry parameters.w") as u32; diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md new file mode 100644 index 00000000..7504c948 --- /dev/null +++ b/docs/design_docs/precompute-dag-execution.md @@ -0,0 +1,24 @@ +# Precompute execution from post-ASAP IR + +Audience: backend developers and reviewers of issue #762. + +The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. + +```mermaid +flowchart LR + P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] + I --> R[Raw source → SummaryAgg streaming kernel] + I --> M[Maintenance dependency scheduler] + R --> S[Stored summary frontier] + S --> M + S --> Q[Query projection and readout] + M --> S +``` + +For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. + +`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. + +Catalog schema version 3 carries Planner family in SDS. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. + +The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats.