diff --git a/control_plane/Cargo.toml b/control_plane/Cargo.toml index 510d3aa9..63b86609 100644 --- a/control_plane/Cargo.toml +++ b/control_plane/Cargo.toml @@ -12,7 +12,6 @@ name = "control_plane" path = "src/main.rs" [dependencies] -asap-physical-operators.workspace = true tokio = { version = "1", features = ["full"] } axum = { version = "0.7", features = ["ws"] } futures-util = "0.3" @@ -33,6 +32,7 @@ parking_lot = "0.12" prometheus = { version = "0.13", default-features = false, features = ["process"] } tonic = { version = "0.12", features = ["gzip"] } asap_types.workspace = true +asap-physical-operators.workspace = true # Planner's IR, replacement search, and frontends share the workspace pin. planner-types.workspace = true diff --git a/control_plane/src/backend_client.rs b/control_plane/src/backend_client.rs index 595f9732..dc356a00 100644 --- a/control_plane/src/backend_client.rs +++ b/control_plane/src/backend_client.rs @@ -1,22 +1,11 @@ -//! HTTP client that pushes a freshly-generated `StreamingConfig` YAML -//! to the ASAPQuery-backend's `POST /api/v1/streaming-config` endpoint. -//! -//! This is the control-plane-side **producer** of the PR E phase 1 / phase 2 -//! hot-reload contract that landed in ASAPQuery-backend PRs #10 and #12. -//! The replanner calls into this module immediately after generating a -//! new plan so the backend's active `StreamingConfig` is updated without -//! a restart and subsequent queries observe the new aggregation layout. -//! -//! The client is **fire-and-forget at the call site** — the replanner -//! awaits the POST but doesn't block its own return on the outcome. -//! Errors are logged at WARN; the control plane is expected to be tolerant -//! of transient backend unavailability because the next replan cycle -//! will try again with the latest plan. +//! Publish and activate complete physical-plan generations on the backend. use std::time::Duration; +#[cfg(test)] use anyhow::{Context, Result}; use reqwest::Client; +#[cfg(test)] use tracing::debug; #[cfg(test)] use tracing::warn; @@ -92,7 +81,7 @@ fn classify_http_status(status: reqwest::StatusCode, body: String, what: &str) - } } -/// Minimal HTTP client for ASAPQuery-backend's streaming-config endpoint. +/// Minimal HTTP client for ASAPQuery-backend's physical-plan endpoint. /// Built once at control-plane startup from the /// `CONTROL_PLANE_BACKEND_ENDPOINT` environment variable and shared /// via `Arc` with the replanner. @@ -105,7 +94,7 @@ pub struct BackendClient { impl BackendClient { /// Construct a client pointing at the backend's plan-push endpoint. /// `endpoint` should be the full URL, e.g. - /// `http://backend.svc:8088/api/v1/streaming-config`. + /// `http://backend.svc:8088/api/v1/physical-plan`. /// /// A 5-second timeout bounds the duration a slow or unreachable /// backend can stall the replanner — consistent with the symmetric @@ -136,11 +125,12 @@ impl BackendClient { &self.endpoint } - /// POST the given `StreamingConfig` YAML to the backend. Returns + /// Test-only transport helper: POST the given YAML to the backend. Returns /// `Ok(())` on any 2xx status, otherwise an error carrying the /// status code and response body. The caller (typically /// [`Replanner::replan_metric`]) logs the error and moves on — the /// next replan cycle will retry with the latest plan. + #[cfg(test)] pub async fn push_streaming_config(&self, yaml: String) -> Result<()> { debug!( endpoint = %self.endpoint, @@ -212,6 +202,7 @@ impl BackendClient { /// that don't need retry semantics keep their `anyhow::Result` /// shape. The retry layer in `emit::backend_push` uses this typed /// variant. + #[cfg(test)] pub async fn post_streaming_config_json_typed( &self, json: String, @@ -245,7 +236,7 @@ impl BackendClient { #[cfg(test)] /// Post backend storage-routing JSON. Derive the URL by replacing the - /// `/api/v1/streaming-config` suffix with `/api/v1/storage_routing`; URLs + /// `/api/v1/physical-plan` suffix with `/api/v1/storage_routing`; URLs /// without that suffix are used verbatim. pub async fn post_storage_routing_json(&self, json: String) -> Result<()> { let url = derive_storage_routing_url(&self.endpoint); @@ -364,25 +355,18 @@ impl BackendClient { } fn derive_physical_plan_url(endpoint: &str) -> String { - const DASH: &str = "/api/v1/streaming-config"; - const UNDERSCORE: &str = "/api/v1/streaming_config"; - const PHYSICAL: &str = "/api/v1/physical-plan"; - endpoint - .strip_suffix(DASH) - .or_else(|| endpoint.strip_suffix(UNDERSCORE)) - .map(|base| format!("{base}{PHYSICAL}")) - .unwrap_or_else(|| endpoint.to_string()) + endpoint.to_string() } -/// Map a streaming-config endpoint URL to the sibling storage-routing +/// Map a physical-plan endpoint URL to the sibling storage-routing /// endpoint by rewriting the trailing path component. URLs that don't -/// end with `/api/v1/streaming-config` (or `/api/v1/streaming_config` — +/// end with `/api/v1/physical-plan` (or `/api/v1/physical_plan` — /// either spelling is supported) pass through unchanged so tests can /// inject a mock-server URL directly. #[cfg(test)] fn derive_storage_routing_url(endpoint: &str) -> String { - const STREAMING_PATH_DASH: &str = "/api/v1/streaming-config"; - const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/streaming_config"; + const STREAMING_PATH_DASH: &str = "/api/v1/physical-plan"; + const STREAMING_PATH_UNDERSCORE: &str = "/api/v1/physical_plan"; const ROUTING_PATH: &str = "/api/v1/storage_routing"; if let Some(stripped) = endpoint.strip_suffix(STREAMING_PATH_DASH) { return format!("{stripped}{ROUTING_PATH}"); @@ -429,7 +413,7 @@ mod tests { async fn start_mock_backend(sink: SharedSink, status: axum::http::StatusCode) -> String { let app = Router::new() .route( - "/api/v1/streaming-config", + "/api/v1/physical-plan", post( move |State(sink): State, body: axum::body::Bytes| async move { let yaml = String::from_utf8_lossy(&body).to_string(); @@ -445,7 +429,7 @@ mod tests { axum::serve(listener, app).await.unwrap(); }); tokio::time::sleep(Duration::from_millis(50)).await; - format!("http://{addr}/api/v1/streaming-config") + format!("http://{addr}/api/v1/physical-plan") } #[tokio::test] @@ -481,7 +465,7 @@ mod tests { #[tokio::test] async fn push_or_log_swallows_errors() { // Point at an unreachable port so the request fails fast. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/streaming-config"); + let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); // Must not panic or propagate — fire-and-forget semantics. push_or_log(&client, "cpu_usage", "content".to_string()).await; } @@ -522,15 +506,15 @@ mod tests { /// storage-routing-URL derivation rewrites the path /// component when the configured endpoint ends in - /// `/api/v1/streaming-config`, leaving everything else untouched. + /// `/api/v1/physical-plan`, leaving everything else untouched. #[test] fn storage_routing_url_rewrites_streaming_path() { assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming-config"), + derive_storage_routing_url("http://backend:8088/api/v1/physical-plan"), "http://backend:8088/api/v1/storage_routing" ); assert_eq!( - derive_storage_routing_url("http://backend:8088/api/v1/streaming_config"), + derive_storage_routing_url("http://backend:8088/api/v1/physical_plan"), "http://backend:8088/api/v1/storage_routing" ); } @@ -578,7 +562,7 @@ mod tests { let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap(); }); - let client = BackendClient::new(format!("http://{addr}/api/v1/streaming-config")); + let client = BackendClient::new(format!("http://{addr}/api/v1/physical-plan")); client .post_catalog_plan_typed(&publication, None, &[]) .await @@ -630,7 +614,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; // Return the streaming-config URL — the client will rewrite // the path before issuing the POST. - format!("http://{addr}/api/v1/streaming-config") + format!("http://{addr}/api/v1/physical-plan") } #[tokio::test] @@ -702,7 +686,7 @@ mod tests { #[tokio::test] async fn typed_connection_refused_is_transient() { // Port 1 on loopback is reserved and refuses connections. - let client = BackendClient::new("http://127.0.0.1:1/api/v1/streaming-config"); + let client = BackendClient::new("http://127.0.0.1:1/api/v1/physical-plan"); let err = client .post_streaming_config_json_typed("{}".to_string()) .await diff --git a/control_plane/src/emit/monitor.rs b/control_plane/src/emit/monitor.rs index 4bd65e6e..59776f3a 100644 --- a/control_plane/src/emit/monitor.rs +++ b/control_plane/src/emit/monitor.rs @@ -9,7 +9,7 @@ //! by `asapedgeprocessor.ThresholdConfig`). The edge derives the agg_id //! from the metric name itself, so this block carries no agg_id. //! 2. [`streaming_config_monitor_entry`] — the `monitors[]` JSON object for the -//! backend `StreamingConfig` (consumed by `asap_types::MonitorSpec`), where +//! backend `InstalledPrecomputePlan` (consumed by `asap_types::MonitorSpec`), where //! `agg_id` IS carried and MUST equal [`agg_id_for_metric`]. //! //! The functions are pure so they can be unit-tested and called from whichever @@ -88,7 +88,7 @@ pub fn edge_threshold_block(intent: &MonitorIntent) -> Value { Value::Mapping(m) } -/// Render the backend `StreamingConfig.monitors[]` JSON entry for this intent, +/// Render the backend `InstalledPrecomputePlan.monitors[]` JSON entry for this intent, /// stamping the cross-language `agg_id`. pub fn streaming_config_monitor_entry(intent: &MonitorIntent) -> serde_json::Value { serde_json::json!({ diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index b0b3bf86..41dd1505 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -41,7 +41,7 @@ struct AppState { /// input identity; incoming telemetry cannot supply its own descriptors. active_summary_catalog: Arc>>>, - /// Shared client for posting streaming configs from HTTP planning and replanning. + /// Shared client for publishing physical plans from HTTP planning and replanning. /// `None` when `CONTROLLER_BACKEND_ENDPOINT` is unset; pushes are then skipped. backend_client: Option>, } @@ -69,13 +69,13 @@ async fn main() { backend_endpoint.as_ref().map(|endpoint| { info!( endpoint = %endpoint, - "ASAPQuery-backend StreamingConfig push enabled" + "ASAPQuery-backend InstalledPrecomputePlan push enabled" ); Arc::new(backend_client::BackendClient::new(endpoint.clone())) }); if backend_client_shared.is_none() { info!( - "ASAPQuery-backend StreamingConfig push disabled \ + "ASAPQuery-backend physical-plan publication disabled \ (set CONTROLLER_BACKEND_ENDPOINT= to enable)" ); } @@ -996,9 +996,9 @@ mod api_tests { #[test] fn app_state_backend_client_some_when_constructed_with_url() { let (state, _router) = - test_app_with_backend(Some("http://127.0.0.1:1/api/v1/streaming-config".into())); + test_app_with_backend(Some("http://127.0.0.1:1/api/v1/physical-plan".into())); let bc = state.backend_client.expect("backend_client must be Some"); - assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/streaming-config"); + assert_eq!(bc.endpoint(), "http://127.0.0.1:1/api/v1/physical-plan"); } // ── POST /api/v1/plan ───────────────────────────────────────────────────── diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index c11eebe4..390934cc 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -7012,15 +7012,22 @@ pub(crate) mod tests { &query_plan, ) .unwrap(); - // V1 has one stored output per definition; arbitrary output IDs are - // rejected before writer/reader agreement is considered. + // Output identity is independent of definition identity, but changing + // only the writer must still invalidate every unchanged reader binding. let mut rebound_writer = bundle.precompute_plan.clone(); rebound_writer.schemas[0] .stored_output_reference .stored_output_id = asap_types::sds::StoredOutputId(123); - assert!(rebound_writer + rebound_writer .validate_against_catalog(&bundle.summary_catalog) - .is_err()); + .unwrap(); + assert!( + asap_types::plan_publication::validate_stored_output_references( + &rebound_writer, + &query_plan, + ) + .is_err() + ); } #[test] diff --git a/control_plane/src/physical/executable_binding.rs b/control_plane/src/physical/executable_binding.rs index ef0609e6..41f12ab2 100644 --- a/control_plane/src/physical/executable_binding.rs +++ b/control_plane/src/physical/executable_binding.rs @@ -2,6 +2,48 @@ pub use asap_types::executable_plan::*; +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum OperatorExecution { + Maintenance, + Query, +} + +/// Keep the backend's ownership decision exhaustive over Planner's physical IR. +/// Adding a payload variant upstream must therefore choose an executor here. +fn operator_execution( + node: &planner_types::post_asap::ExecutableDagNode, +) -> Result { + use planner_types::post_asap::{ExecutableOperatorPayload as Payload, ExecutionTiming}; + + let declared = match &node.payload { + Payload::Binary { timing, .. } + | Payload::Value { timing, .. } + | Payload::SummaryMerge { timing } => *timing, + Payload::MembershipFilter { .. } | Payload::SummaryEstimate { .. } => { + ExecutionTiming::QueryTime + } + Payload::SummaryAgg { .. } + | Payload::SummaryJoin { .. } + | Payload::SummarySubtract + | Payload::SummaryDelete { .. } => ExecutionTiming::IngestionTime, + // These operators can be placed on either side of the stored-state + // boundary. Planner's validated output state is authoritative. + Payload::Fallback { .. } | Payload::RelationalJoin { .. } => node.output_state.timing, + }; + if declared != node.output_state.timing { + return Err(format!( + "post-ASAP node {:?} has operator timing {} but output state {}", + node.id, + declared.as_str(), + node.output_state + )); + } + Ok(match declared { + ExecutionTiming::IngestionTime => OperatorExecution::Maintenance, + ExecutionTiming::QueryTime => OperatorExecution::Query, + }) +} + /// Assign backend phases to a selected semantic DAG without changing its nodes. pub fn install_selected_dag( query_id: String, @@ -15,12 +57,21 @@ pub fn install_selected_dag( let mut nodes = std::collections::BTreeMap::new(); let mut precompute_sinks = Vec::new(); for node in &dag.nodes { + if let planner_types::post_asap::ExecutableOperatorPayload::SummaryAgg { + family, + input, + grouping, + .. + } = &node.payload + { + asap_physical_operators::capability::validate_summary_kernel(family, input, grouping) + .map_err(|reason| format!("post-ASAP node {:?}: {reason}", node.id))?; + } + let execution = operator_execution(node)?; let binding = if let Some(summary_definition) = materialization(node.id) { precompute_sinks.push(node.id); BackendNodeBinding::Materialization { summary_definition } - } else if node.output_state.timing - == planner_types::post_asap::ExecutionTiming::IngestionTime - { + } else if execution == OperatorExecution::Maintenance { BackendNodeBinding::MaintenanceInput } else { query_node(node.id).map_or(BackendNodeBinding::QueryInput, |query_node| { diff --git a/control_plane/src/physical/post_asap/deployment_expr.rs b/control_plane/src/physical/post_asap/deployment_expr.rs index 4012277e..b0372b6a 100644 --- a/control_plane/src/physical/post_asap/deployment_expr.rs +++ b/control_plane/src/physical/post_asap/deployment_expr.rs @@ -59,7 +59,7 @@ pub enum PhysicalExpr { /// Phase ε.1 Mode 2: no sketch processor at the edge — raw OTLP /// forwards to the backend, which builds the sketch at ingest. The /// `family` and `params` are the sketch the backend will build, so - /// the backend's `StreamingConfig` `aggregation_input` is `raw` for + /// the backend's `InstalledPrecomputePlan` `aggregation_input` is `raw` for /// this metric. RawAtEdgeSketchAtBackend { /// Sketch family the backend will build at ingest. diff --git a/control_plane/src/physical/post_asap/tests.rs b/control_plane/src/physical/post_asap/tests.rs index deb1b464..48508861 100644 --- a/control_plane/src/physical/post_asap/tests.rs +++ b/control_plane/src/physical/post_asap/tests.rs @@ -579,7 +579,7 @@ fn pipeline_l1_to_l4(query: &str, accuracy: AccuracyTarget) -> PhysicalExpr { } /// `quantile_over_time.yaml` — the asap-planner-rs `quantile_over_time` -/// fixture maps to a KLL or DDSketch StreamingConfig row. The control plane +/// fixture maps to a KLL or DDSketch InstalledPrecomputePlan row. The control plane /// path: L1 PromQL parse → L3 `Aggregate{Quantile{0.99}}` over `Window` → /// L4 bind picks Kll (default) or DDSketch. Either is functionally /// equivalent — both are quantile sketches. diff --git a/control_plane/src/workload.rs b/control_plane/src/workload.rs index d6b74cd4..395ce65c 100644 --- a/control_plane/src/workload.rs +++ b/control_plane/src/workload.rs @@ -501,7 +501,7 @@ impl WorkloadRegistry { /// Inject (or replace, keyed by `metric_name`) a runtime workload entry. /// Used by the autonomous-allocation apply path to register a synthesized /// monitor so the next replan/repost emits it into the backend - /// `StreamingConfig` (the coordinator then derives the ε-floor `p`). Shared + /// `InstalledPrecomputePlan` (the coordinator then derives the ε-floor `p`). Shared /// across registry clones via the `Arc>` overlay. pub fn insert_runtime(&self, entry: WorkloadEntry) { let mut rt = self diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index f4fe8699..99abb12a 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -334,7 +334,7 @@ impl PrecomputeMaterialization { } /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the - /// policy-fingerprint-keyed call sites (e.g. `StreamingConfig`'s + /// policy-fingerprint-keyed call sites (e.g. `InstalledPrecomputePlan`'s /// `HashMap` keys). **Always** equal to /// `self.policy_fingerprint().as_u64()`. The value is content- /// addressed identity, NOT a controller-allocated counter id. diff --git a/crates/asap_types/src/executable_plan.rs b/crates/asap_types/src/executable_plan.rs index 1b087f3a..32e8f321 100644 --- a/crates/asap_types/src/executable_plan.rs +++ b/crates/asap_types/src/executable_plan.rs @@ -369,7 +369,7 @@ mod tests { fn send_sync() {} send_sync::(); let wire = serde_json::json!({ - "schema_version": 1, "query_id": "q", "nodes": [], "edges": [], "root": 0 + "schema_version": 2, "query_id": "q", "nodes": [], "edges": [], "root": 0 }); let document: OwnedPostAsapDag = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(serde_json::to_value(document).unwrap(), wire); diff --git a/crates/asap_types/src/monitor_spec.rs b/crates/asap_types/src/monitor_spec.rs index a22352be..6aad5ad0 100644 --- a/crates/asap_types/src/monitor_spec.rs +++ b/crates/asap_types/src/monitor_spec.rs @@ -38,9 +38,9 @@ impl MonitorFunctional { /// (empty for Sum / whole-stream). See /// `ASAPCollector/docs/continuous-monitoring-tumbling-cost-analysis.md`. /// -/// Stays here (unlike `data_plane::storage_engines::types::StreamingConfig`, +/// Stays here (unlike `data_plane::storage_engines::types::InstalledPrecomputePlan`, /// which holds a `Vec` field) because `control_plane` genuinely -/// needs it: `emit/monitor.rs` builds the `StreamingConfig.monitors[]` JSON +/// needs it: `emit/monitor.rs` builds the `InstalledPrecomputePlan.monitors[]` JSON /// entry by hand and has a regression test asserting that JSON deserializes /// into this exact type. `control_plane` cannot depend on `data_plane` (the /// dependency runs the other way), so this type has to live somewhere both diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs index dde5e5cc..16f007bb 100644 --- a/crates/asap_types/src/policy_registry.rs +++ b/crates/asap_types/src/policy_registry.rs @@ -3,8 +3,8 @@ //! Derived view over a collection of `PrecomputeMaterialization`s that maps //! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the //! merged-sid-identity-chain replacement for the controller-allocated -//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `StreamingConfig` -//! carries (see `data_plane::storage_engines::types::streaming_config`'s +//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `InstalledPrecomputePlan` +//! carries (see `data_plane::storage_engines::types::installed_precompute_plan`'s //! module doc for why that type lives there, not here). //! //! ## Dual-keyed transition diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs index 40a90fe3..ec20b7ef 100644 --- a/crates/asap_types/src/routing_index.rs +++ b/crates/asap_types/src/routing_index.rs @@ -1,6 +1,6 @@ //! `RoutingIndex` — a metric-bucketed structural index over a //! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `StreamingConfig`'s `PrecomputeMaterialization`s, so it represents planned policy +//! `InstalledPrecomputePlan`'s `PrecomputeMaterialization`s, so it represents planned policy //! rather than a reconstruction from ingest side effects. //! //! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] @@ -23,11 +23,11 @@ //! `streaming_snap.policy_registry()` call) still get the Tier-2 win for //! any query with more than one candidate sharing the same snapshot //! (composed PromQL shapes routinely do). Building it once per -//! `StreamingConfig` hot-reload swap instead of once per query — the same +//! `InstalledPrecomputePlan` hot-reload swap instead of once per query — the same //! "cheap, but call at swap time not per query, if it shows up in -//! profiles" note `StreamingConfig::policy_registry`'s own doc comment +//! profiles" note `InstalledPrecomputePlan::policy_registry`'s own doc comment //! already flags — is a further, larger change (it means threading a -//! cached derived value through `StreamingConfigHandle`'s swap path) +//! cached derived value through `InstalledPrecomputePlanHandle`'s swap path) //! and is not done by this type on its own. use std::collections::{BTreeSet, HashMap}; diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index c19a1819..aca1f6d0 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -55,14 +55,13 @@ impl From for crate::PolicyFingerprint { } /// Identity of one persisted producer output within an installed plan version. -/// V1 derives it from the definition ID because the runtime index is keyed by -/// definition; a future schema may allocate independent output IDs. +/// It is independent of the semantic definition shared by equivalent producers. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(transparent)] pub struct StoredOutputId(pub u64); /// Typed join key carried by both the writer and every bound reader. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct StoredOutputReference { #[serde(alias = "state_slot_id")] @@ -71,7 +70,7 @@ pub struct StoredOutputReference { } impl StoredOutputReference { - /// V1 binding for the single stored output of a definition. + /// Default compiler allocation when one output is selected per definition. pub fn for_definition(definition_id: SummaryDefinitionId) -> Self { Self { stored_output_id: StoredOutputId(definition_id.as_u64()), @@ -80,16 +79,42 @@ impl StoredOutputReference { } pub fn validate(&self) -> Result<(), SdsError> { - if *self == Self::for_definition(self.definition_id) { + if self.stored_output_id.0 != 0 && !self.definition_id.fingerprint().is_unset() { Ok(()) } else { Err(SdsError( - "stored output differs from its V1 definition binding".into(), + "stored output and definition identities must be set".into(), )) } } } +/// Canonical address of one stored DAG output for one population and window. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct StoredSummaryKey { + pub plan_id: u64, + pub plan_version: u64, + pub output: StoredOutputReference, + pub population: BTreeMap, + pub window: HalfOpenTimeRange, +} + +impl StoredSummaryKey { + pub fn validate(&self) -> Result<(), SdsError> { + self.output.validate()?; + if self.window.start_ms >= self.window.end_ms { + return Err(SdsError("stored summary requires a nonempty window".into())); + } + Ok(()) + } + + pub fn storage_key(&self) -> Result { + self.validate()?; + serde_json::to_string(self).map_err(|e| SdsError(e.to_string())) + } +} + descriptor_id!(SummaryDescriptorId); descriptor_id!(DataDescriptorId); @@ -335,13 +360,11 @@ pub struct SummaryInstance { impl SummaryInstance { pub fn validate(&self) -> Result<(), SdsError> { - if self.stored_output_id - != StoredOutputReference::for_definition(self.summary_definition_id).stored_output_id - { - return Err(SdsError( - "summary instance has an invalid stored output".into(), - )); + StoredOutputReference { + stored_output_id: self.stored_output_id, + definition_id: self.summary_definition_id, } + .validate()?; if self.time_range.start_ms >= self.time_range.end_ms { return Err(SdsError( "summary instance time range must be non-empty".into(), @@ -1397,16 +1420,62 @@ mod tests { inventory.validate().unwrap(); } + // Every component of the persisted output address participates in identity. + #[test] + fn stored_summary_key_binds_plan_output_population_and_window() { + let key = StoredSummaryKey { + plan_id: 1, + plan_version: 2, + output: StoredOutputReference { + stored_output_id: StoredOutputId(101), + definition_id: crate::PolicyFingerprint(7).into(), + }, + population: BTreeMap::from([("service".into(), "api".into())]), + window: HalfOpenTimeRange { + start_ms: 0, + end_ms: 1000, + }, + }; + let mut variants = vec![key.clone(); 5]; + variants[0].plan_id += 1; + variants[1].plan_version += 1; + variants[2].output.stored_output_id.0 += 1; + variants[3].population.insert("service".into(), "db".into()); + variants[4].window.end_ms += 1; + let canonical = key.storage_key().unwrap(); + for other in variants { + assert_ne!(canonical, other.storage_key().unwrap()); + } + assert_eq!( + serde_json::from_str::(&canonical).unwrap(), + key + ); + } + + // A DAG output has its own identity even when its definition is shared. + #[test] + fn stored_output_identity_is_independent_of_definition() { + let definition_id = SummaryDefinitionId::from(crate::PolicyFingerprint(7)); + for stored_output_id in [StoredOutputId(101), StoredOutputId(102)] { + StoredOutputReference { + stored_output_id, + definition_id, + } + .validate() + .unwrap(); + } + } + #[test] fn stored_output_and_payload_version_must_match_instance_definition() { let mut instance = observed_instance(InstanceLifecycle::Persistent); - instance.stored_output_id = StoredOutputId(8); + instance.stored_output_id = StoredOutputId(0); assert!(instance.validate().is_err()); instance.stored_output_id = StoredOutputId(7); instance.state_reference.generation = 3; assert!(instance.validate().is_err()); let mut reference = StoredOutputReference::for_definition(instance.summary_definition_id); - reference.stored_output_id = StoredOutputId(8); + reference.stored_output_id = StoredOutputId(0); assert!(reference.validate().is_err()); } diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 6462d255..cc005bda 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -111,7 +111,7 @@ fn sketch_meta( config: SketchConfig, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), capability: Some(match algorithm { @@ -133,7 +133,7 @@ fn sketch_meta( fn precompute_meta(sid: u64, metric: &str, agg_type: AggregationType) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: None, @@ -389,12 +389,14 @@ fn bench_query_precomputes_by_agg(c: &mut Criterion) { g.finish(); } -/// Build a `StreamingConfig` whose single agg-config's content +/// Build a `InstalledPrecomputePlan` whose single agg-config's content /// signature matches every sid registered by `build_precompute_store` /// (metric / `Sum` / no grouping / empty params+filter). With this /// config the reconciler retires nothing — the steady-state ingest /// case, where the per-batch reconcile is pure scan overhead. -fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types::StreamingConfig { +fn matching_streaming_config( + metric: &str, +) -> data_plane::storage_engines::types::InstalledPrecomputePlan { use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; @@ -418,9 +420,14 @@ fn matching_streaming_config(metric: &str) -> data_plane::storage_engines::types None, None, ); - let mut map = HashMap::new(); - map.insert(1u64, cfg); - data_plane::storage_engines::types::StreamingConfig::new(map) + let runtime = data_plane::storage_engines::types::InstalledPrecomputePlan::default(); + let plan = control_plane::physical::compiler::PrecomputePlan::build( + runtime.plan().envelope.clone(), + vec![cfg], + &[], + ) + .unwrap(); + data_plane::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan(plan).unwrap() } /// `reconcile_from_streaming_config` ran on EVERY ingest batch and, in diff --git a/data_plane/examples/sketch_db_diag.rs b/data_plane/examples/sketch_db_diag.rs index 325d72b0..87f719d4 100644 --- a/data_plane/examples/sketch_db_diag.rs +++ b/data_plane/examples/sketch_db_diag.rs @@ -51,7 +51,7 @@ fn dd_meta(sid: u64) -> SummarySeriesMetadata { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "bench_metric".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 985edc6c..bd12e770 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -5,7 +5,7 @@ //! engine via [`OtlpReceiver::with_ingest_state`] — routes both raw metric //! points and pre-built sketches through the precompute engine's worker //! pool. The precompute engine then performs window-aligned aggregation -//! per `StreamingConfig` and writes results to `SketchStore`. +//! per `InstalledPrecomputePlan` and writes results to `SketchStore`. //! //! Architectural flow: //! ```text @@ -621,27 +621,11 @@ fn resolve_bucket_sid_for_agg_config( config.population_key_encoding, &grouping_pairs, )?; - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - let sid = ingest_state.series_resolver.resolve_with_reactivation( - &config.metric, + let sid = ingest_state.summary_store.resolve_output_storage_handle( + &ingest_state.series_resolver, + config.policy_fingerprint().into(), &fp, - &agg_kind_canonical, - |sid| { - ingest_state - .summary_store - .validate_routed_catalog_generation(captured_generation)?; - let activation = ingest_state - .summary_store - .authorize_series_reactivation(sid, config.policy_fingerprint().into())?; - if activation - .as_deref() - .is_some_and(|generation| Some(generation) != captured_generation) - { - return Err("stale OTLP generation cannot reactivate series".into()); - } - Ok(activation) - }, + captured_generation, )?; let policy_fp = asap_types::PolicyFingerprint(config.policy_fp_u64()); Ok((sid, policy_fp)) @@ -663,7 +647,7 @@ async fn route_otlp_to_precompute( .map(Arc::new); let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.streaming_config.clone()) + .map(|plan| plan.installed_precompute_plan.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); let agg_configs = snap.materializations(); // Reconcile sid lifecycle using the current streaming-config snapshot. @@ -914,7 +898,7 @@ async fn route_modified_otlp_sketches_to_precompute( let active_physical_plan_snapshot = ingest_state.active_physical_plan_snapshot(); let snap = active_physical_plan_snapshot .as_ref() - .map(|plan| plan.streaming_config.clone()) + .map(|plan| plan.installed_precompute_plan.clone()) .unwrap_or_else(|| ingest_state.config_snapshot()); let catalog_generation = active_physical_plan_snapshot .as_ref() @@ -1221,7 +1205,7 @@ async fn route_modified_otlp_sketches_to_precompute( // because `fp=""` is a perfectly good mint/lookup key. let resolved_sid: Option = if dp.series_id != 0 && attrs_pairs.is_empty() { let sid = dp.series_id; - if ingest_state.summary_store.instance(sid).is_some() { + if ingest_state.summary_store.is_current_storage_handle(sid) { Some(sid) } else { unknown_sids.push(sid); @@ -1249,45 +1233,36 @@ async fn route_modified_otlp_sketches_to_precompute( // frames would mint distinct sids and the upgrade // could never fire (the analyzer would also see two // candidates for one logical series). - let algorithm_for_sid = base_sketch_algorithm(sketch_algorithm_for(&dp)); - let agg_kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { - algorithm: algorithm_for_sid, - config: dp.container_config.clone(), - // OTel-ingest path: no per-DP spatial filter applies - // at this layer (the agent has already filtered - // before emitting the sketch). The empty filter is - // the canonical value for "no filter on this sid's - // policy". - spatial_filter_canonical: String::new(), - }; - let agg_kind_canonical = agg_kind.canonical_string(); let definition = frame_identity .as_ref() .map(|frame| frame.materialization) .unwrap_or_else(|| asap_types::PolicyFingerprint(0).into()); - let assigned = match ingest_state.series_resolver.resolve_with_reactivation( - &canonical_name, + let resolved = ingest_state.summary_store.resolve_output_storage_handle( + &ingest_state.series_resolver, + definition, &fp, - &agg_kind_canonical, - |sid| { - ingest_state - .summary_store - .validate_routed_catalog_generation( - catalog_generation.as_deref(), - )?; - let activation = ingest_state - .summary_store - .authorize_series_reactivation(sid, definition)?; - if activation.as_deref().is_some_and(|generation| { - Some(generation) != catalog_generation.as_deref() - }) { - return Err( - "stale OTLP generation cannot reactivate series".into() - ); - } - Ok(activation) - }, - ) { + catalog_generation.as_deref(), + ); + // Codec-only unit fixtures have no installed plan. Production + // accepts stored summaries only through an installed output. + #[cfg(test)] + let resolved = if catalog_generation.is_none() + && definition.fingerprint().is_unset() + { + let kind = crate::storage_engines::sketch_db::data::AggKind::Sketch { + algorithm: base_sketch_algorithm(sketch_algorithm_for(&dp)), + config: dp.container_config.clone(), + spatial_filter_canonical: String::new(), + }; + Ok(ingest_state.series_resolver.resolve( + &canonical_name, + &fp, + &kind.canonical_string(), + )) + } else { + resolved + }; + let assigned = match resolved { Ok(sid) => sid, Err(error) => { if dp.series_id != 0 { @@ -1436,7 +1411,7 @@ async fn route_modified_otlp_sketches_to_precompute( .map(|s| s.to_string()) }; ingest_state.summary_store.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: canonical_name.clone(), group_by_keys, capability: Some(cap), @@ -3575,7 +3550,7 @@ mod sid_resolution_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -3588,8 +3563,8 @@ mod sid_resolution_tests { async fn make_state() -> (Arc, tokio::task::JoinHandle<()>) { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); - let streaming = StreamingConfig::new(std::collections::HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(std::collections::HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, samples_ingested: std::sync::atomic::AtomicU64::new(0), @@ -4510,7 +4485,7 @@ mod sid_bucketing_tests { use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::storage_engines::sketch_db::index::SketchStore; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; use asap_otel_proto::tonic::common::v1::{any_value::Value as AnyVal, AnyValue, KeyValue}; use asap_otel_proto::tonic::metrics::v1::{ @@ -4629,8 +4604,8 @@ mod sid_bucketing_tests { let policy_fp = asap_types::PolicyFingerprint(cfg.policy_fp_u64()); let mut configs = HashMap::new(); configs.insert(cfg.policy_fp_u64(), cfg.clone()); - let streaming = StreamingConfig::new(configs); - let hot_reload = StreamingConfigHandle::new(streaming); + let streaming = InstalledPrecomputePlan::new(configs); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); let state = Arc::new(IngestState { @@ -4725,7 +4700,10 @@ mod sid_bucketing_tests { }; let fp = crate::drivers::ingest::canonical_attrs_fingerprint(&[("zone", zone_for_bucket)]); - let resolved = resolver.lookup(metric, &fp, &agg_kind_canonical); + let resolved = state + .summary_store + .resolve_output_storage_handle(&resolver, policy_fp.into(), &fp, None) + .ok(); assert_eq!( resolved, Some(*sid), diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 123c0b83..879fc835 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -233,11 +233,7 @@ impl PrometheusRemoteWriteReceiver { if plan.precompute_plan.summary_catalog.as_ref() != Some(&generation) { return Err("finite maintenance generation changed during drain".into()); } - crate::precompute_engine::maintenance_runtime::execute_finite_maintenance( - &self.inner.ingest.summary_store, - &self.inner.ingest.series_resolver, - &plan.precompute_plan, - )?; + self.inner.ingest.router.complete_dag(plan).await?; if let Some(observer) = self.inner.ingest.router.erp_observer() { let now_ms = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -676,7 +672,7 @@ fn route_messages( Arc, ); type RoutedSample = (String, i64, f64); - let snapshot = physical_plan.streaming_config.clone(); + let snapshot = physical_plan.installed_precompute_plan.clone(); let _ = crate::storage_engines::sketch_db::lifecycle::reconcile_if_config_changed( ingest.summary_store.as_ref(), &snapshot, @@ -766,30 +762,14 @@ fn route_messages( &computed_attrs_fp }; let policy_fp = asap_types::PolicyFingerprint(config.policy_fp_u64()); - // A sketch family is not a complete physical identity. Two - // materializations may use the same family and grouping while - // differing in update semantics (for example count- versus - // value-weighted Top-K). Keep those states on distinct SIDs. - let materialization_kind = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); let sid = ingest - .series_resolver - .resolve_with_reactivation(&config.metric, attrs_fp, &materialization_kind, |sid| { - ingest.summary_store.validate_routed_catalog_generation( - physical_plan.precompute_plan.summary_catalog.as_ref(), - )?; - let activation = ingest - .summary_store - .authorize_series_reactivation(sid, policy_fp.into())?; - if let Some(generation) = &activation { - if physical_plan.precompute_plan.summary_catalog.as_ref() - != Some(generation.as_ref()) - { - return Err("stale routed generation cannot reactivate series".into()); - } - } - Ok(activation) - }) + .summary_store + .resolve_output_storage_handle( + &ingest.series_resolver, + policy_fp.into(), + attrs_fp, + physical_plan.precompute_plan.summary_catalog.as_ref(), + ) .map_err(RemoteWriteError::SeriesIdentity)?; buckets .entry(sid) @@ -936,8 +916,8 @@ mod tests { use crate::precompute_engine::ingest_handler::IngestObservability; use crate::precompute_engine::series_router::SeriesRouter; use crate::storage_engines::types::{ - ActivePhysicalPlanHandle, BackendStorageRouting, RuntimePhysicalPlan, StreamingConfig, - StreamingConfigHandle, + ActivePhysicalPlanHandle, BackendStorageRouting, InstalledPrecomputePlan, + InstalledPrecomputePlanHandle, RuntimePhysicalPlan, }; use tokio::sync::mpsc; @@ -947,7 +927,7 @@ mod tests { .unwrap() } - fn physical_config(streaming: StreamingConfig) -> StreamingConfigHandle { + fn physical_config(streaming: InstalledPrecomputePlan) -> InstalledPrecomputePlanHandle { use asap_types::producer_plan::{FrameIdentityContract, SequenceScope, TransmissionPlan}; use control_plane::physical::compiler::{ IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, @@ -1013,11 +993,13 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: Arc::new(streaming), + installed_precompute_plan: Arc::new(streaming), query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; - StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new(active)) + InstalledPrecomputePlanHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( + active, + )) } fn receiver(config: PrometheusRemoteWriteConfig) -> PrometheusRemoteWriteReceiver { @@ -1026,7 +1008,7 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: physical_config(StreamingConfig::default()), + hot_reload_config: physical_config(InstalledPrecomputePlan::default()), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), @@ -1066,7 +1048,7 @@ mod tests { value_source_column: None, }; let policy_fp = aggregation.policy_fp_u64(); - let streaming = StreamingConfig::new(HashMap::from([(policy_fp, aggregation)])); + let streaming = InstalledPrecomputePlan::new(HashMap::from([(policy_fp, aggregation)])); let (sender, receiver) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1104,7 +1086,7 @@ mod tests { let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let hot = physical_config(StreamingConfig::new(HashMap::from([( + let hot = physical_config(InstalledPrecomputePlan::new(HashMap::from([( config.policy_fp_u64(), config.clone(), )]))); @@ -1218,7 +1200,7 @@ mod tests { assert_ne!(kll_fp, pooled_kll_fp); let cms_fp = cms.policy_fingerprint(); let counter_fp = counter.policy_fingerprint(); - let streaming = StreamingConfig::new(HashMap::from([ + let streaming = InstalledPrecomputePlan::new(HashMap::from([ (cms_fp.0, cms), (counter_fp.0, counter), (kll_fp.0, kll), @@ -1384,7 +1366,9 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: StreamingConfigHandle::new(StreamingConfig::default()), + hot_reload_config: InstalledPrecomputePlanHandle::new( + InstalledPrecomputePlan::default(), + ), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index ddfaeecd..e4b31355 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -134,7 +134,7 @@ pub struct HttpServer { /// `QueryEngine`) and extended via [`Self::with_query_engine`] — /// e.g. to plug in a `GorillaQueryEngine` for the cold archive /// tier. Instant-query dispatch consults this for metrics whose - /// `StreamingConfig::storage_backend()` is anything other than + /// `InstalledPrecomputePlan::storage_backend()` is anything other than /// `SketchStore`; ASAP-tier queries still take the direct /// `ASAPQueryEngine::handle_query` path so they keep the /// `KeyByLabelNames` Prometheus needs to populate the `metric` @@ -142,9 +142,9 @@ pub struct HttpServer { query_router: Arc, /// Sketch storage for runtime diagnostics. summary_store: Arc, - /// Hot-reloadable `StreamingConfig` source. `None` when hot-reload + /// Hot-reloadable `InstalledPrecomputePlan` source. `None` when hot-reload /// is not wired up by the caller (unit tests, legacy binaries). - hot_reload_config: Option, + hot_reload_config: Option, /// Per-metric storage-backend routing table consulted by the HTTP /// instant-query handler at request time. When `Some(..)` and the /// query parses, the handler extracts the metric name from the @@ -201,7 +201,7 @@ struct AppState { summary_store: Arc, adapter: Arc, fallback: Option>, - hot_reload_config: Option, + hot_reload_config: Option, /// See [`HttpServer::backend_storage_routing`]. backend_storage_routing: Option, /// Backfill registry (sketch DB §10). See `HttpServer::backfill`. @@ -289,13 +289,11 @@ impl HttpServer { self } - /// Attach a `StreamingConfigHandle` handle so the - /// `GET/POST /api/v1/streaming-config` endpoints can read and - /// swap the currently active config. Without this handle the - /// endpoints return `503 Service Unavailable`. + /// Attach the precompute projection used by installed-plan consumers. + /// Computation changes are published through the complete physical plan. pub fn with_hot_reload_config( mut self, - handle: crate::storage_engines::types::StreamingConfigHandle, + handle: crate::storage_engines::types::InstalledPrecomputePlanHandle, ) -> Self { self.hot_reload_config = Some(handle); self @@ -328,7 +326,7 @@ impl HttpServer { /// and dispatches through `EngineRouter` for any per-metric /// override. Without the wrapper the handler falls back to the /// pre-Phase-5 single-axis behaviour driven by - /// `StreamingConfig::storage_backend()`. + /// `InstalledPrecomputePlan::storage_backend()`. pub fn with_backend_storage_routing( mut self, routing: Arc, @@ -484,17 +482,6 @@ impl HttpServer { "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), ); - // Legacy partial configuration is available only when the distributed - // profile explicitly supplies its hot-reload handle. The ASAPQuery - // profile exposes only generation-scoped physical-plan publication. - let app = if app_state.hot_reload_config.is_some() { - app.route( - "/api/v1/streaming-config", - get(handle_get_streaming_config).post(handle_post_streaming_config), - ) - } else { - app - }; let app = if adapter.query_language() == asap_types::QueryLanguage::MetricsQl { app.route( "/select/:tenant/prometheus/api/v1/query", @@ -610,14 +597,6 @@ impl HttpServer { "/api/v1/db/backfill/jobs/:job_id", get(handle_get_backfill_job).delete(handle_delete_backfill_job), ); - let app = if app_state.hot_reload_config.is_some() { - app.route( - "/api/v1/streaming-config", - get(handle_get_streaming_config).post(handle_post_streaming_config), - ) - } else { - app - }; let app = if adapter.query_language() == asap_types::QueryLanguage::MetricsQl { app.route( "/select/:tenant/prometheus/api/v1/query", @@ -783,7 +762,7 @@ async fn process_query_request( // AST and looked up in the table. This is the production // path the issue-46 MVP relies on so non-default metrics // actually route through the `EngineRouter`. - // (b) Single-axis `StreamingConfig::storage_backend()` from the + // (b) Single-axis `InstalledPrecomputePlan::storage_backend()` from the // hot-reload config (the pre-Phase-5 fallback). Pre-control-plane // deploys ride this path; it always lands on `SketchStore` // unless the YAML was hand-patched. @@ -961,12 +940,12 @@ fn resolve_metric_storage(state: &AppState, query: &str, tenant: &str) -> Storag return backend; } debug!( - "resolve_metric_storage: PromQL parsed but no metric name found in AST; falling back to streaming-config axis", + "resolve_metric_storage: PromQL parsed but no metric name found in AST; falling back to installed storage axis", ); } Err(e) => { debug!( - "resolve_metric_storage: PromQL parse failed ({}); falling back to streaming-config axis", + "resolve_metric_storage: PromQL parse failed ({}); falling back to installed storage axis", e, ); } @@ -1379,7 +1358,7 @@ async fn process_via_named_engine( /// pinned by the request: /// /// * `query` — straight from the parsed request. -/// * `metric_storage` — looked up from the hot-reload `StreamingConfig` +/// * `metric_storage` — looked up from the hot-reload `InstalledPrecomputePlan` /// by the caller. /// /// `Statistic` remains a shape-derived follow-up. `AccuracyTarget` is @@ -2480,7 +2459,7 @@ mod tests { use crate::precompute_engine::ingest_handler::{IngestObservability, IngestState}; use crate::precompute_engine::series_router::SeriesRouter; use crate::query_engines::ASAPQueryEngine; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlan, InstalledPrecomputePlanHandle}; use prost::Message; use reqwest::Client; use std::sync::atomic::AtomicU64; @@ -2497,7 +2476,7 @@ mod tests { IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, TimestampUnit, PLANNER_REVISION, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let envelope = PlanEnvelope { plan_id: 7, plan_version: 1, @@ -2547,7 +2526,7 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: streaming_config.clone(), + installed_precompute_plan: installed_precompute_plan.clone(), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id: 7, plan_version: 1, @@ -2560,7 +2539,7 @@ mod tests { ), }, ); - let hot_reload = StreamingConfigHandle::from_active_physical_plan(active.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let (sender, _worker) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -2649,7 +2628,9 @@ mod tests { ); } - async fn setup_test_server_with_hot_reload(hot_reload: Option) -> u16 { + async fn setup_test_server_with_hot_reload( + hot_reload: Option, + ) -> u16 { let adapter_config = AdapterConfig::prometheus_promql( "http://127.0.0.1:9999".to_string(), // Unused for this test false, // forward_unsupported_queries @@ -2661,7 +2642,7 @@ mod tests { adapter_config, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( @@ -2841,15 +2822,12 @@ mod tests { assert!(status.is_success() || status == reqwest::StatusCode::OK); } - // ── StreamingConfig hot-reload (PR E) ──────────────────────────────── + // ── InstalledPrecomputePlan hot-reload (PR E) ──────────────────────────────── - /// POST a YAML streaming-config and verify the active state via - /// GET reflects the swap. Covers the full round-trip through - /// `HttpServer::with_hot_reload_config`, the POST parse+swap, and - /// the GET snapshot emission. + /// A retired partial-configuration route cannot mutate the active plan. #[tokio::test] async fn flat_streaming_config_is_rejected_without_mutating_active_state() { - let handle = StreamingConfigHandle::new(StreamingConfig::default()); + let handle = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let port = setup_test_server_with_hot_reload(Some(handle.clone())).await; let before = handle.snapshot(); let response = Client::new() @@ -2858,7 +2836,7 @@ mod tests { .send() .await .unwrap(); - assert_eq!(response.status(), reqwest::StatusCode::GONE); + assert_eq!(response.status(), reqwest::StatusCode::NOT_FOUND); assert!(Arc::ptr_eq(&before, &handle.snapshot())); } @@ -2890,7 +2868,7 @@ mod tests { #[tokio::test] async fn test_streaming_config_hot_reload_rejects_bad_yaml() { - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -2902,9 +2880,7 @@ mod tests { .send() .await .unwrap(); - assert_eq!(resp.status(), reqwest::StatusCode::GONE); - let body: serde_json::Value = resp.json().await.unwrap(); - assert_eq!(body["status"], "error"); + assert_eq!(resp.status(), reqwest::StatusCode::NOT_FOUND); } /// Set up a test server wired with a hot-reload handle and a @@ -2915,7 +2891,7 @@ mod tests { /// legacy `SchemaRegistry` is gone, so there is no longer a /// `schemas` parameter — every reconcile decision is sid-level. async fn setup_test_server_with_hot_reload_and_sketch_index( - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, summary_store: Arc, ) -> u16 { let adapter_config = @@ -2925,7 +2901,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let server = HttpServer::new(config, query_engine, summary_store).with_hot_reload_config(hot_reload); @@ -2950,7 +2926,7 @@ mod tests { use std::collections::BTreeSet; let group_by_keys: BTreeSet = group_by.iter().map(|s| s.to_string()).collect(); store.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, @@ -2975,7 +2951,7 @@ mod tests { // handler force-retires it. use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); register_precompute_sid(&summary_store, 1, "m1", &[]); register_precompute_sid(&summary_store, 2, "m2", &[]); @@ -3005,11 +2981,11 @@ mod tests { assert_eq!(body["count"], 2); let entries = body["schemas"].as_array().unwrap(); // Sorted by sid — first is active, second is retired. - assert_eq!(entries[0]["sid"], 1); + assert_eq!(entries[0]["storage_handle"], 1); assert_eq!(entries[0]["status"], "active"); assert_eq!(entries[0]["metric_name"], "m1"); assert!(entries[0]["retired_at_ms"].is_null()); - assert_eq!(entries[1]["sid"], 2); + assert_eq!(entries[1]["storage_handle"], 2); assert_eq!(entries[1]["status"], "retired"); assert!(entries[1]["retired_at_ms"].is_u64()); @@ -3023,7 +2999,7 @@ mod tests { .unwrap(); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["count"], 1); - assert_eq!(body["schemas"][0]["sid"], 1); + assert_eq!(body["schemas"][0]["storage_handle"], 1); // Filter: retired only. let resp = client @@ -3035,7 +3011,7 @@ mod tests { .unwrap(); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["count"], 1); - assert_eq!(body["schemas"][0]["sid"], 2); + assert_eq!(body["schemas"][0]["storage_handle"], 2); // Bogus filter → 400. let resp = client @@ -3054,7 +3030,7 @@ mod tests { // attached (every `HttpServer` carries one). With no // registered sids the endpoint reports an empty array, not // a 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3080,7 +3056,7 @@ mod tests { use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::AggStatus; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); register_precompute_sid(&summary_store, 11, "cpu", &["host"]); register_precompute_sid(&summary_store, 22, "mem", &["host"]); @@ -3102,7 +3078,7 @@ mod tests { assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); - assert_eq!(body["schema"]["sid"], 11); + assert_eq!(body["schema"]["storage_handle"], 11); assert_eq!(body["schema"]["status"], "retired"); assert_eq!( summary_store.instance(11).unwrap().status(), @@ -3120,7 +3096,7 @@ mod tests { assert!(resp.status().is_success()); let body: serde_json::Value = resp.json().await.unwrap(); assert_eq!(body["status"], "success"); - assert_eq!(body["schema"]["sid"], 22); + assert_eq!(body["schema"]["storage_handle"], 22); assert_eq!(body["schema"]["status"], "expired"); assert_eq!( summary_store.instance(22).unwrap().status(), @@ -3145,7 +3121,7 @@ mod tests { async fn test_get_timeline_missing_param_returns_400() { use crate::storage_engines::sketch_db::index::SketchStore; - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let summary_store = Arc::new(SketchStore::new()); let server_port = setup_test_server_with_hot_reload_and_sketch_index( hot_reload.clone(), @@ -3191,7 +3167,7 @@ mod tests { // reads from the sid catalog (always attached) instead of the // optional `SchemaRegistry`. Empty catalog → empty segments, // not a 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); let resp = client @@ -3241,7 +3217,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - // Build a StreamingConfig with one Sum agg per `active_agg_ids` + // Build a InstalledPrecomputePlan with one Sum agg per `active_agg_ids` // so the backfill handler's agg-config lookup (post-schema- // retirement) can find them. The matching sid in the catalog // is registered via the canonical ingest path so its content @@ -3283,8 +3259,8 @@ mod tests { marker_to_fp.insert(*marker, fp); agg_map.insert(fp, cfg); } - let streaming_config = Arc::new(StreamingConfig::new(agg_map)); - let hot_reload = StreamingConfigHandle::from_arc(streaming_config.clone()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::new(agg_map)); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(installed_precompute_plan.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); for marker in active_agg_ids { @@ -3443,7 +3419,7 @@ mod tests { async fn test_backfill_endpoints_503_without_registry() { // Build a server with NO backfill registry attached — every // backfill endpoint should 503. - let hot_reload = StreamingConfigHandle::new(StreamingConfig::default()); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::default()); let server_port = setup_test_server_with_hot_reload(Some(hot_reload)).await; let client = Client::new(); @@ -3632,10 +3608,10 @@ mod tests { } /// Build an `HttpServer` whose router holds the supplied set of - /// `QueryEngine`s. The hot-reload `StreamingConfig` is pinned at + /// `QueryEngine`s. The hot-reload `InstalledPrecomputePlan` is pinned at /// `metric_storage_backend` so query dispatch follows the /// requested capability axis. Returns the bound port + the - /// `StreamingConfigHandle` handle so tests can swap the + /// `InstalledPrecomputePlanHandle` handle so tests can swap the /// `storage_backend` mid-flight if they need to. async fn setup_test_server_with_router( metric_storage_backend: StorageBackend, @@ -3661,10 +3637,12 @@ mod tests { }; // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. - let streaming_cfg = - StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); + let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( + Default::default(), + metric_storage_backend, + ); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -3706,9 +3684,9 @@ mod tests { // — exactly what the production deploy looks like (the YAML // loader doesn't parse `storage_backend`). All routing // decisions must come from the per-metric routing table. - let streaming_cfg = StreamingConfig::default(); + let streaming_cfg = InstalledPrecomputePlan::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -3928,9 +3906,9 @@ mod tests { #[tokio::test] async fn http_query_with_no_storage_config_defaults_to_asap_tier() { - // `StreamingConfig::default()` has `storage_backend = + // `InstalledPrecomputePlan::default()` has `storage_backend = // SketchStore` (per the `#[serde(default)]` on the - // field — see `streaming_config.rs`). A server set up + // field — see `installed_precompute_plan.rs`). A server set up // without a hot-reload handle still infers ASAP-tier and // takes the ASAPQueryEngine direct path. Back-compat for // pre-Phase-5 deploys whose YAML doesn't include the new @@ -4120,10 +4098,8 @@ mod tests { // mock the routing decision by pinning `streaming_cfg.storage_backend // = GorillaObjectStore` directly. That proves the dispatch BRANCH is // wired, but not the production code path — in real deploys the - // streaming-config YAML loader drops `storage_backend` (it always - // defaults to `SketchStore`), so the issue-46 v2 demo's queries - // never reached the EngineRouter. The tests below exercise the - // **production path** end-to-end: streaming config stays default, + // installed precompute view uses the default storage backend. + // The tests below exercise per-metric routing end-to-end: // a per-metric `BackendStorageRouting` table is loaded at startup // (mirroring `--backend-storage-routing` on `precompute_engine`), // and the handler must consult the table on every request. @@ -4610,7 +4586,9 @@ mod tests { crate::query_engines::routing::HotReloadBackendStorageRouting, ) { use crate::query_engines::routing::HotReloadBackendStorageRouting; - use crate::storage_engines::types::{StreamingConfig, StreamingConfigHandle}; + use crate::storage_engines::types::{ + InstalledPrecomputePlan, InstalledPrecomputePlanHandle, + }; let adapter_config = AdapterConfig::prometheus_promql("http://127.0.0.1:9999".to_string(), false); @@ -4619,9 +4597,9 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = StreamingConfig::default(); + let streaming_cfg = InstalledPrecomputePlan::default(); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let routing_handle = HotReloadBackendStorageRouting::empty(); let server = HttpServer::new( @@ -4941,10 +4919,12 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = - StreamingConfig::with_storage_backend(Default::default(), metric_storage_backend); + let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( + Default::default(), + metric_storage_backend, + ); let streaming_arc = Arc::new(streaming_cfg); - let hot_reload = StreamingConfigHandle::from_arc(streaming_arc.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let mut server = HttpServer::new( config, @@ -4990,7 +4970,7 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_arc = Arc::new(StreamingConfig::default()); + let streaming_arc = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let cache = Arc::new(crate::query_engines::routing::FreshnessProbeCache::new()); let server = HttpServer::new( @@ -5310,36 +5290,6 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: (StatusCode::OK, axum::Json(body)).into_response() } -// Streaming configuration endpoints: GET returns the active snapshot; POST -// parses a replacement and atomically publishes it to new snapshot readers. -// In-flight operations retain their existing snapshot. - -async fn handle_get_streaming_config(State(state): State) -> axum::response::Response { - use axum::http::StatusCode; - use axum::response::IntoResponse; - - let Some(handle) = state.hot_reload_config else { - let body = serde_json::json!({ - "status": "error", - "error": "hot-reload handle not attached; backend was built without HttpServer::with_hot_reload_config"}); - return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); - }; - let snap = handle.snapshot(); - let body = serde_json::json!({ - "status": "success", - "aggregation_count": snap.materializations_by_policy_fingerprint.len(), - "aggregation_ids": snap.materializations_by_policy_fingerprint.keys().copied().collect::>(), - "streaming_config": &*snap}); - (StatusCode::OK, axum::Json(body)).into_response() -} - -async fn handle_post_streaming_config() -> axum::response::Response { - use axum::response::IntoResponse; - (axum::http::StatusCode::GONE, axum::Json(serde_json::json!({ - "status":"error", "error":"install the complete DAG through /api/v1/physical-plan and activate its generation; partial aggregation config updates have been removed" - }))).into_response() -} - pub use asap_types::plan_publication::PhysicalPlanInstallRequest; /// Decode and cross-validate every backend view before it can become visible. @@ -5366,43 +5316,6 @@ pub fn validate_and_build_runtime_plan( .validate_against_catalog(&request.summary_catalog) .map_err(|error| format!("CollectorPlan catalog validation error: {error}"))?; } - for entry in request.query_plan.entries.values() { - for binding in entry.materialization_bindings() { - let materialization = request - .precompute_plan - .materializations - .iter() - .find(|config| config.policy_fingerprint() == binding.materialization.fingerprint()) - .ok_or_else(|| "query binding has no precompute definition".to_string())?; - if binding.window_ms != materialization.stored_window_ms() { - return Err( - "query physical pane duration differs from installed precompute definition" - .into(), - ); - } - if binding.pane_origin_ms != materialization.pane_origin_ms { - return Err( - "query physical pane origin differs from installed precompute definition" - .into(), - ); - } - // `full_window_slide_ms` is `#[serde(default)]`, so a publication from an - // older controller -- or one replayed from a stored artifact -- arrives as - // `None` on a FullWindow materialization. Without this gate the readout - // silently takes the overlap-merging path and counts observations twice, - // which is exactly what the full-window binding exists to prevent. - let full_window_slide_ms = matches!( - materialization.window_layout, - asap_types::WindowMaterializationLayout::FullWindow - ) - .then_some(materialization.slide_interval.saturating_mul(1_000)); - if binding.full_window_slide_ms != full_window_slide_ms { - return Err( - "query window layout differs from installed precompute definition".into(), - ); - } - } - } asap_types::plan_publication::validate_stored_output_references( &request.precompute_plan, &request.query_plan, @@ -5422,11 +5335,12 @@ pub fn validate_and_build_runtime_plan( { return Err("physical subplans have different plan identity/version".into()); } - let streaming_config = crate::storage_engines::types::StreamingConfig::from_precompute_plan( - request.precompute_plan.clone(), - ) - .map_err(|error| format!("DAG execution installation failed: {error}"))?; - let typed_fps: BTreeSet<_> = streaming_config + let installed_precompute_plan = + crate::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan( + request.precompute_plan.clone(), + ) + .map_err(|error| format!("DAG execution installation failed: {error}"))?; + let typed_fps: BTreeSet<_> = installed_precompute_plan .materializations_by_policy_fingerprint .keys() .copied() @@ -5452,7 +5366,7 @@ pub fn validate_and_build_runtime_plan( summary_catalog: Some(Arc::new(request.summary_catalog)), precompute_plan: request.precompute_plan, transmission_plan: request.transmission_plan, - streaming_config: Arc::new(streaming_config), + installed_precompute_plan: Arc::new(installed_precompute_plan), query_plan: Arc::new(request.query_plan), storage_routing, }) @@ -5599,7 +5513,7 @@ async fn handle_activate_physical_plan( move |plan| match plan.summary_catalog.as_ref() { Some(catalog) => { store - .install_summary_catalog(Arc::clone(catalog)) + .install_precompute_plan(Arc::clone(catalog), &plan.precompute_plan) .map_err(|error| format!("SummaryCatalog install error: {error}"))?; if let Some(receiver) = &remote_write { receiver.install_erp_observation_generation( @@ -5643,7 +5557,7 @@ async fn handle_activate_physical_plan( } }); } - let snap = activated.streaming_config.clone(); + let snap = activated.installed_precompute_plan.clone(); let retired = crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config( state.summary_store.as_ref(), snap.as_ref(), @@ -5757,6 +5671,10 @@ async fn handle_physical_plan_status(State(state): State) -> axum::res ) .into_response(); }; + let precompute_plan = state + .active_physical_plan + .as_ref() + .map(|active| active.active_snapshot().precompute_plan.clone()); let materializations = state .active_physical_plan .as_ref() @@ -5767,6 +5685,7 @@ async fn handle_physical_plan_status(State(state): State) -> axum::res axum::Json(serde_json::json!({ "status": "success", "plans": lifecycle.statuses(), + "precompute_plan": precompute_plan, "materializations": materializations })), ) @@ -5787,9 +5706,7 @@ fn unix_time_ms() -> u64 { /// /// Useful for operators to confirm a control plane push landed with the /// expected entries. Returns 503 when the backend wasn't built with a -/// routing-table handle (legacy deploys that loaded the YAML directly -/// can still hit `/api/v1/streaming-config` — this endpoint is for -/// the Phase α JSON path). +/// routing-table handle. async fn handle_get_storage_routing( State(state): State, headers: axum::http::HeaderMap, @@ -5836,8 +5753,7 @@ async fn handle_get_storage_routing( /// * 503 — backend wasn't built with a routing-table handle. /// /// The swap is atomic: in-flight queries either see the entire old -/// table or the entire new table; never a half-applied state. Mirrors -/// the existing `POST /api/v1/streaming-config` swap contract. +/// table or the entire new table; never a half-applied state. async fn handle_post_storage_routing( State(state): State, headers: axum::http::HeaderMap, @@ -5946,11 +5862,23 @@ async fn handle_get_schemas( .iter() .filter(|m| allowed.contains(&m.status())) .map(|metadata| { - let descriptors = state.summary_store.descriptors_for_series_id(metadata.sid); - sid_instance_to_json(metadata, descriptors.as_ref()) + let descriptors = state + .summary_store + .descriptors_for_series_id(metadata.storage_handle); + stored_output_instance_to_json( + metadata, + descriptors.as_ref(), + state + .summary_store + .stored_output_for_handle(metadata.storage_handle), + ) }) .collect(); - entries.sort_by_key(|v| v.get("sid").and_then(|x| x.as_u64()).unwrap_or(0)); + entries.sort_by_key(|v| { + v.get("storage_handle") + .and_then(|x| x.as_u64()) + .unwrap_or(0) + }); let body = serde_json::json!({ "status": "success", @@ -5969,17 +5897,19 @@ fn status_str(s: crate::storage_engines::sketch_db::AggStatus) -> &'static str { } /// Encode sid metadata, including identity, lifecycle timestamps, and status. -fn sid_instance_to_json( +fn stored_output_instance_to_json( m: &crate::storage_engines::sketch_db::index::SummarySeriesMetadata, descriptors: Option<&( std::sync::Arc, std::sync::Arc, )>, + output: Option, ) -> serde_json::Value { let summary_descriptor_id = descriptors.map(|(summary, _)| summary.id.canonical()); let data_descriptor_id = descriptors.map(|(_, data)| data.id.canonical()); serde_json::json!({ - "sid": m.sid, + "storage_handle": m.storage_handle, + "stored_output_reference": output, "metric_name": m.metric_name, "status": status_str(m.status()), "first_seen_unix_ms": m.first_seen_unix_ms, @@ -6009,7 +5939,7 @@ async fn handle_post_schema_retire( let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", - "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); + "schema": stored_output_instance_to_json(&meta, descriptors.as_ref(), state.summary_store.stored_output_for_handle(meta.storage_handle))}); (StatusCode::OK, axum::Json(body)).into_response() } None => { @@ -6035,7 +5965,7 @@ async fn handle_post_schema_expire( let descriptors = state.summary_store.descriptors_for_series_id(sid); let body = serde_json::json!({ "status": "success", - "schema": sid_instance_to_json(&meta, descriptors.as_ref())}); + "schema": stored_output_instance_to_json(&meta, descriptors.as_ref(), state.summary_store.stored_output_for_handle(meta.storage_handle))}); (StatusCode::OK, axum::Json(body)).into_response() } None => { @@ -6228,13 +6158,13 @@ async fn handle_post_backfill_job( return (StatusCode::BAD_REQUEST, axum::Json(body)).into_response(); } - // Resolve the aggregation from the current streaming-config snapshot, or + // Resolve the aggregation from the current installed precompute plan, or // return 404 `UnknownAgg`. Its earliest matching sid timestamp bounds the // backfill range; with no ingested sid, use the current time. let Some(handle) = state.hot_reload_config.as_ref() else { let body = serde_json::json!({ "status": "error", - "error": "hot-reload streaming-config handle not attached; backfill agg lookup requires HttpServer::with_hot_reload_config"}); + "error": "hot-reload installed precompute handle not attached; backfill agg lookup requires HttpServer::with_hot_reload_config"}); return (StatusCode::SERVICE_UNAVAILABLE, axum::Json(body)).into_response(); }; let snapshot = handle.snapshot(); @@ -6243,7 +6173,7 @@ async fn handle_post_backfill_job( None => { let body = serde_json::json!({ "status": "error", - "error": format!("unknown agg_id {} (not in streaming-config snapshot)", req.agg_id)}); + "error": format!("unknown agg_id {} (not in installed precompute plan)", req.agg_id)}); return (StatusCode::NOT_FOUND, axum::Json(body)).into_response(); } }; diff --git a/data_plane/src/lib.rs b/data_plane/src/lib.rs index fd8b5014..116429c2 100644 --- a/data_plane/src/lib.rs +++ b/data_plane/src/lib.rs @@ -59,8 +59,6 @@ pub use precompute_engine::config::{LateDataPolicy, PrecomputeEngineConfig}; pub use precompute_engine::output_sink::SketchStoreSink; pub use precompute_engine::PrecomputeEngine; -pub use utils::read_streaming_config; - pub type Result = std::result::Result>; #[cfg(test)] diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 91c109e8..6e3d38bd 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -9,7 +9,6 @@ use data_plane::drivers::AdapterConfig; use data_plane::precompute_engine::config::LateDataPolicy; use data_plane::precompute_engine::PrecomputeWorkerDiagnostics; use data_plane::storage_engines::types::enums::{CleanupPolicy, LockStrategy}; -use data_plane::utils::file_io::read_streaming_config; use data_plane::{ ASAPQueryEngine, HttpServer, HttpServerConfig, OtlpReceiver, OtlpReceiverConfig, PrecomputeEngine, PrecomputeEngineConfig, PrometheusRemoteWriteConfig, @@ -41,9 +40,9 @@ struct Args { #[arg(long, value_enum, default_value = "distributed")] profile: RuntimeProfile, - /// Legacy bootstrap streaming config (distributed profile only). + /// Monitor coordinator settings, separate from executable computation plans. #[arg(long)] - streaming_config: Option, + monitor_specs: Option, /// JSON physical-plan artifact. Required by the backend-local profile; /// all runtime/query views are validated and installed as one snapshot. @@ -67,7 +66,7 @@ struct Args { /// the e2e harness's 30s window. ASAPQueryEngine uses this as the /// instant-query lookback window — for tumbling-window /// aggregations it must be ≥ the window size in - /// `streaming-config`. + /// the installed precompute plan. #[arg(long, default_value = "30")] prometheus_scrape_interval: u64, @@ -235,7 +234,7 @@ struct Args { otel_http_port: u16, /// Enable the continuous-monitoring (CDM) coordinator gRPC server. Serves - /// the `monitors:` specs from the streaming-config; no-op if that list is + /// the specs from `--monitor-specs`; no-op if that list is /// empty. #[arg(long)] enable_monitor_coordinator: bool, @@ -378,7 +377,7 @@ struct Args { /// consulted by the HTTP query handler on every PromQL request to /// pick the right engine (`ASAPQueryEngine` for ASAP-tier /// sketches). Without - /// this flag the handler falls back to the streaming-config + /// this flag the handler falls back to the installed storage view /// single axis (always `SketchStore`) and the EngineRouter is /// effectively bypassed — the issue-46 v2 demo's criterion ⑤ /// failure mode. Mirrors the `precompute_engine` binary's flag @@ -422,20 +421,14 @@ fn validate_query_forwarding_configuration( fn validate_profile(args: &Args) -> Result<()> { validate_query_forwarding_configuration(args)?; if args.profile != RuntimeProfile::Asapquery { - if args.streaming_config.is_none() { - return Err("the distributed profile requires --streaming-config".into()); + if args.physical_plan.is_none() { + return Err("the distributed profile requires --physical-plan".into()); } if args.planning_snapshot.is_some() { return Err("--planning-snapshot is available only with --profile asapquery".into()); } return Ok(()); } - if args.streaming_config.is_some() { - return Err( - "--profile asapquery rejects --streaming-config; use --planning-snapshot or --physical-plan" - .into(), - ); - } if args.physical_plan.is_some() == args.planning_snapshot.is_some() { return Err( "--profile asapquery requires exactly one of --planning-snapshot or --physical-plan" @@ -623,19 +616,12 @@ async fn main() -> Result<()> { } else { None }; - let streaming_config = match startup_physical_plan.as_ref() { - Some(active) => active.streaming_config.clone(), - None => Arc::new(read_streaming_config( - args.streaming_config - .as_deref() - .expect("validated distributed streaming config"), - )?), - }; + let startup_physical_plan = startup_physical_plan.ok_or("startup requires a physical plan")?; + let installed_precompute_plan = startup_physical_plan.installed_precompute_plan.clone(); info!( - "Loaded streaming config with {} entries", - streaming_config.materializations().len() + "Installed precompute DAG with {} stored outputs", + installed_precompute_plan.materializations().len() ); - info!("Streaming config: {:?}", streaming_config); // Share a hot-reload handle with HTTP configuration endpoints and consumers. @@ -672,12 +658,9 @@ async fn main() -> Result<()> { Arc::new(data_plane::drivers::ingest::series_resolver::SeriesIdResolver::new()) }; let summary_store = Arc::new(data_plane::storage_engines::sketch_db::index::SketchStore::new()); - if let Some(catalog) = startup_physical_plan - .as_ref() - .and_then(|plan| plan.summary_catalog.as_ref()) - { + if let Some(catalog) = startup_physical_plan.summary_catalog.as_ref() { summary_store - .install_summary_catalog(Arc::clone(catalog)) + .install_precompute_plan(Arc::clone(catalog), &startup_physical_plan.precompute_plan) .map_err(std::io::Error::other)?; } @@ -732,65 +715,11 @@ async fn main() -> Result<()> { None }; - // Bootstrap projections share one immutable physical-plan envelope. - let initial_precompute_plan = asap_types::precompute_plan::PrecomputePlan { - summary_catalog: None, - envelope: asap_types::precompute_plan::PlanEnvelope { - plan_id: 0, - plan_version: 0, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "bootstrap".into(), - planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "bootstrap".into(), - }, - ingest: asap_types::precompute_plan::IngestContract { - protocol: asap_types::precompute_plan::IngestProtocol::ModifiedOtlpMetricsV1, - endpoint_path: "/v1/metrics".into(), - timestamp_unit: asap_types::precompute_plan::TimestampUnit::UnixNanoseconds, - require_plan_identity: false, - require_summary_definition_identity: false, - require_registered_producer: false, - }, - schemas: Vec::new(), - producers: Vec::new(), - materializations: streaming_config - .materializations_by_policy_fingerprint - .values() - .cloned() - .collect(), - executable_dags: Default::default(), - }; - let initial_transmission_plan = asap_types::producer_plan::TransmissionPlan { - summary_catalog: None, - envelope: initial_precompute_plan.envelope.clone(), - frame_identity: asap_types::producer_plan::FrameIdentityContract { - identity_version: 1, - sequence_scope: - asap_types::producer_plan::SequenceScope::MaterializationSeriesProducerEpoch, - require_checkpoint_for_full: true, - require_base_checkpoint_for_delta: true, - }, - rules: Vec::new(), - }; - let initial_active_plan = startup_physical_plan.unwrap_or_else(|| { - data_plane::storage_engines::types::RuntimePhysicalPlan { - envelope: initial_precompute_plan.envelope.clone(), - summary_catalog: None, - precompute_plan: initial_precompute_plan, - transmission_plan: initial_transmission_plan, - streaming_config: streaming_config.clone(), - query_plan: Arc::new(asap_types::query_plan::QueryPlan::empty()), - storage_routing: Arc::new( - data_plane::storage_engines::types::BackendStorageRouting::empty(), - ), - } - }); + let initial_active_plan = startup_physical_plan; let active_physical_plan = data_plane::storage_engines::types::ActivePhysicalPlanHandle::new(initial_active_plan); let hot_reload_config = - data_plane::storage_engines::types::StreamingConfigHandle::from_active_physical_plan( + data_plane::storage_engines::types::InstalledPrecomputePlanHandle::from_active_physical_plan( active_physical_plan.clone(), ); @@ -972,69 +901,23 @@ async fn main() -> Result<()> { use data_plane::update_sampling::{ Functional, MonitorConfig, MonitorCoordinator, MonitorServiceImpl, }; - let specs: Vec = streaming_config - .monitors() - .iter() + let monitor_specs: Vec = match &args.monitor_specs { + Some(path) => serde_yaml::from_slice(&fs::read(path)?)?, + None => Vec::new(), + }; + let specs = monitor_specs + .into_iter() .map(|m| MonitorConfig { agg_id: m.agg_id, - key: m.key.clone().into_bytes(), + key: m.key.into_bytes(), tau: m.tau, epsilon: m.epsilon, window_ms: m.window_ms, functional: Functional::from_name(&m.functional), }) .collect(); - if specs.is_empty() { - warn!("--enable-monitor-coordinator set but streaming-config has no `monitors:` yet — the coordinator will pick them up live when the control plane pushes a config (hot-reload)"); - } let coord = MonitorCoordinator::new(specs); - // Hot-reload watcher: the coordinator reads `monitors:` once at boot, but - // the control plane pushes the real config slightly AFTER boot via the - // `/api/v1/streaming-config` POST (an ArcSwap in `hot_reload_config`). - // Without this, a monitor that arrives post-boot never reaches the - // coordinator and every edge registering for it is rejected as - // "unconfigured". Watch the ArcSwap and re-apply its `monitors:` to the - // live coordinator on each swap (cheap: an atomic load + pointer compare - // every 2s; `reconfigure` is a no-op unless the spec set actually - // changed). The same path covers controller-driven monitor add/remove. - { - let coord = coord.clone(); - let hot = hot_reload_config.clone(); - tokio::spawn(async move { - let mut last = hot.snapshot(); - loop { - tokio::time::sleep(std::time::Duration::from_secs(2)).await; - let cur = hot.snapshot(); - if Arc::ptr_eq(&last, &cur) { - continue; - } - last = cur.clone(); - let specs: Vec = cur - .monitors() - .iter() - .map(|m| MonitorConfig { - agg_id: m.agg_id, - key: m.key.clone().into_bytes(), - tau: m.tau, - epsilon: m.epsilon, - window_ms: m.window_ms, - functional: Functional::from_name(&m.functional), - }) - .collect(); - let (added, changed, removed) = coord.reconfigure(specs).await; - if added + changed + removed > 0 { - info!( - added, - changed, - removed, - "CDM monitor coordinator hot-reloaded monitors from pushed streaming-config" - ); - } - } - }); - } - let svc = MonitorServiceImpl::new(coord).into_server(); let port = args.monitor_grpc_port; info!("Starting CDM monitor coordinator gRPC on 0.0.0.0:{port}"); @@ -1111,7 +994,7 @@ async fn main() -> Result<()> { // from `--backend-storage-routing` (or its env-var alias) so the // HTTP handler consults a per-metric `StorageBackend` map on // every PromQL query instead of bypassing the EngineRouter when - // the streaming-config single axis defaults to `SketchStore`. + // the installed storage view single axis defaults to `SketchStore`. // // even when no static YAML is loaded, install an // empty hot-reload handle so the control plane's first @@ -1153,7 +1036,7 @@ async fn main() -> Result<()> { summary_catalog: current.summary_catalog.clone(), precompute_plan: current.precompute_plan.clone(), transmission_plan: current.transmission_plan.clone(), - streaming_config: current.streaming_config.clone(), + installed_precompute_plan: current.installed_precompute_plan.clone(), query_plan: current.query_plan.clone(), storage_routing: Arc::new(bootstrap_routing), }); @@ -1184,7 +1067,7 @@ async fn main() -> Result<()> { args.enable_backfill_worker, precompute_ingest_state.as_ref(), ) { - // Backfill uses the shared streaming-config snapshot and sketch store. + // Backfill uses the installed precompute plan and sketch store. let reader_factory = match args.clickhouse_backfill_table.as_ref() { Some(table) => data_plane::storage_engines::sketch_db::clickhouse_reader_factory( data_plane::storage_engines::sketch_db::ClickHouseReaderConfig { @@ -1488,6 +1371,13 @@ mod tests { use clap::Parser; use data_plane::drivers::AdapterConfig; + // Every profile must bootstrap from the same validated physical artifact. + #[test] + fn distributed_accepts_physical_plan_without_streaming_config() { + let args = Args::try_parse_from(["data_plane", "--physical-plan", "plan.json"]).unwrap(); + assert!(validate_profile(&args).is_ok()); + } + #[test] fn asapquery_requires_atomic_physical_plan_not_streaming_config() { let valid = Args::try_parse_from([ @@ -1501,21 +1391,9 @@ mod tests { .unwrap(); assert!(validate_profile(&valid).is_ok()); - let legacy = Args::try_parse_from([ - "data_plane", - "--profile", - "asapquery", - "--streaming-config", - "streaming.yaml", - "--physical-plan", - "plan.json", - "--forward-unsupported-queries", - ]) - .unwrap(); - assert!(validate_profile(&legacy) - .unwrap_err() - .to_string() - .contains("rejects --streaming-config")); + assert!( + Args::try_parse_from(["data_plane", "--streaming-config", "streaming.yaml"]).is_err() + ); let planned = Args::try_parse_from([ "data_plane", @@ -1573,7 +1451,7 @@ mod tests { fn disable_query_forwarding_rejects_conflicting_flags() { let args = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--forward-unsupported-queries", @@ -1589,7 +1467,7 @@ mod tests { fn disable_query_forwarding_rejects_forwarding_listeners() { let args = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--victoriametrics-http-port", @@ -1606,7 +1484,7 @@ mod tests { fn disable_query_forwarding_rejects_clickhouse_listener() { let clickhouse = Args::try_parse_from([ "data_plane", - "--streaming-config", + "--physical-plan", "streaming.yaml", "--disable-query-forwarding", "--clickhouse-http-port", diff --git a/data_plane/src/precompute_engine/engine.rs b/data_plane/src/precompute_engine/engine.rs index fa451ca3..d6b912e8 100644 --- a/data_plane/src/precompute_engine/engine.rs +++ b/data_plane/src/precompute_engine/engine.rs @@ -3,7 +3,7 @@ use crate::precompute_engine::ingest_handler::IngestState; use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::{SeriesRouter, WorkerMessage}; use crate::precompute_engine::worker::{Worker, WorkerRuntimeConfig}; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; use std::sync::atomic::{AtomicI64, AtomicUsize}; use std::sync::Arc; use tokio::sync::mpsc; @@ -27,7 +27,7 @@ pub struct PrecomputeEngine { output_sink: Arc, diagnostics: Arc, ingest_state: Arc, - hot_reload_config: StreamingConfigHandle, + hot_reload_config: InstalledPrecomputePlanHandle, /// Worker receivers, one per worker. Taken by `run()` when spawning workers. receivers: Vec>, } @@ -35,7 +35,7 @@ pub struct PrecomputeEngine { impl PrecomputeEngine { pub fn new( config: PrecomputeEngineConfig, - hot_reload_config: StreamingConfigHandle, + hot_reload_config: InstalledPrecomputePlanHandle, output_sink: Arc, series_resolver: Arc, summary_store: Arc, @@ -63,7 +63,7 @@ impl PrecomputeEngine { } // Build the router that owns the senders; it will be shared via IngestState. - let router = SeriesRouter::new(senders); + let router = SeriesRouter::new(senders).with_plan(hot_reload_config.clone()); // Snapshot the hot-reload configuration on each ingest batch so policy // changes are visible immediately. Sid lifecycle reconciliation uses @@ -107,13 +107,6 @@ impl PrecomputeEngine { /// submit through the shared `IngestState` returned by `ingest_state()`. pub async fn run(mut self) -> Result<(), Box> { let num_workers = self.config.num_workers; - let output_sink: Arc = Arc::new( - crate::precompute_engine::maintenance_runtime::MaintenanceDagSink::new( - Arc::clone(&self.output_sink), - self.hot_reload_config.clone(), - ), - ); - // Take ownership of receivers (they can only be used once). let receivers = std::mem::take(&mut self.receivers); @@ -122,6 +115,13 @@ impl PrecomputeEngine { // ConfigReload messages needed. let mut worker_handles = Vec::with_capacity(num_workers); for (id, rx) in receivers.into_iter().enumerate() { + let output_sink: Arc = Arc::new( + crate::precompute_engine::maintenance_runtime::MaintenanceDagSink::new( + Arc::clone(&self.output_sink), + self.hot_reload_config.clone(), + ), + ); + let mut worker = Worker::new( id, rx, @@ -141,6 +141,10 @@ impl PrecomputeEngine { self.diagnostics.worker_group_counts[id].clone(), self.diagnostics.worker_watermarks[id].clone(), ); + worker.set_maintenance_storage( + Arc::clone(&self.ingest_state.summary_store), + Arc::clone(&self.ingest_state.series_resolver), + ); worker.set_erp_observer(self.ingest_state.router.erp_observer()); let handle = tokio::spawn(async move { worker.run().await; @@ -155,7 +159,7 @@ impl PrecomputeEngine { // Start flush timer — pure flush, no config polling. let flush_state = ingest_state.clone(); let flush_interval_ms = self.config.flush_interval_ms; - tokio::spawn(async move { + let flush_task = tokio::spawn(async move { let mut interval = tokio::time::interval(tokio::time::Duration::from_millis(flush_interval_ms)); loop { @@ -168,10 +172,16 @@ impl PrecomputeEngine { }); // Wait for workers to finish (this only happens on shutdown). + let mut worker_error = None; for handle in worker_handles { - let _ = handle.await; + if let Err(error) = handle.await { + worker_error = Some(error); + } + } + flush_task.abort(); + if let Some(error) = worker_error { + return Err(Box::new(error)); } - Ok(()) } } diff --git a/data_plane/src/precompute_engine/group_key.rs b/data_plane/src/precompute_engine/group_key.rs index 44212278..dd3c5684 100644 --- a/data_plane/src/precompute_engine/group_key.rs +++ b/data_plane/src/precompute_engine/group_key.rs @@ -45,7 +45,6 @@ impl GroupKey { } } - #[cfg(test)] pub fn canonical_bytes(&self) -> &[u8] { &self.canonical } diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index 77bc6c38..6d59810e 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -1,6 +1,6 @@ use crate::precompute_engine::series_router::SeriesRouter; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; use asap_types::aggregation_config::PrecomputeMaterialization; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; @@ -121,7 +121,7 @@ pub struct IngestState { /// Hot-reloadable streaming config. On each ingest batch, the /// router snapshots the latest config to derive agg_configs. /// This replaces the old frozen `Vec>`. - pub hot_reload_config: StreamingConfigHandle, + pub hot_reload_config: InstalledPrecomputePlanHandle, /// When true, skip group-key extraction and pass raw samples through. pub pass_raw_samples: bool, /// Per-series reconstructed sketch bases, keyed by series identity. Full frames @@ -156,10 +156,10 @@ impl IngestState { /// configs from a `POST /api/v1/streaming-config` swap are /// visible immediately without restart. /// - /// Returns the shared `Arc` — no cloning of + /// Returns the shared `Arc` — no cloning of /// individual PrecomputeMaterialization objects, just an atomic refcount /// increment (~5ns). - pub fn config_snapshot(&self) -> Arc { + pub fn config_snapshot(&self) -> Arc { self.hot_reload_config.snapshot() } @@ -293,7 +293,7 @@ fn extract_group_key( mod tests { use super::*; use crate::precompute_engine::series_router::SeriesRouter; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; @@ -336,9 +336,9 @@ mod tests { let mut map = std::collections::HashMap::new(); map.insert(agg_id, make_config(agg_id, metric)); - let streaming = StreamingConfig::new(map); + let streaming = InstalledPrecomputePlan::new(map); let hot_reload = - crate::storage_engines::types::StreamingConfigHandle::new(streaming.clone()); + crate::storage_engines::types::InstalledPrecomputePlanHandle::new(streaming.clone()); let state = Arc::new(IngestState { router, diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index 569efc5e..59aa5ba6 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -2,10 +2,13 @@ use super::output_sink::OutputSink; use super::subdag_scheduler::{ - execute_precompute_sink, IdempotentCommitSink, MaterializationCommitKey, - PrecomputeOperatorRegistry, ScheduleError, + execute_precompute_sink, execute_precompute_sinks, IdempotentCommitSink, + MaterializationCommitKey, PrecomputeOperatorRegistry, ScheduleError, }; -use crate::storage_engines::types::{AggregateCore, PrecomputedOutput, StreamingConfigHandle}; +use crate::storage_engines::types::{ + AggregateCore, InstalledPrecomputePlanHandle, PrecomputedOutput, +}; +use asap_physical_operators::dag::RunContext; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; use planner_types::post_asap::{ExecutableDagNode, ExecutableOperatorPayload, PostAsapNodeId}; use sha2::{Digest, Sha256}; @@ -161,16 +164,45 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } } + fn output_bytes(&self, value: &MaintenanceValue) -> usize { + fn labels(group: &Population) -> usize { + group.iter().map(|(k, v)| k.len() + v.len()).sum() + } + match value { + MaintenanceValue::Summary { state, .. } => state.approx_memory_bytes(), + MaintenanceValue::SummaryWindows { states, .. } => states + .iter() + .map(|(group, windows)| { + labels(group) + + windows + .iter() + .map(|(_, state)| 8 + state.approx_memory_bytes()) + .sum::() + }) + .sum(), + MaintenanceValue::Rows { values, name, .. } => { + name.len() + + values + .iter() + .map(|(group, rows)| { + labels(group) + rows.len() * std::mem::size_of::<(i64, f64)>() + }) + .sum::() + } + } + } + fn execute( &self, node: &ExecutableDagNode, inputs: &[Arc], + context: RunContext, ) -> Result { if node.output_state.timing != planner_types::post_asap::ExecutionTiming::IngestionTime { return Err("ingestion executor received a query-time node".into()); } match &node.payload { - ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs), + ExecutableOperatorPayload::SummaryMerge => merge_inputs(inputs, &context), ExecutableOperatorPayload::Binary { operator } => { if !self.inputs.frozen_inputs().is_some() || node.output_state @@ -178,7 +210,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { { return Err("maintenance binary requires immutable completed row inputs".into()); } - evaluate_aligned_binary(node, operator, inputs) + evaluate_aligned_binary(node, operator, inputs, &context) } ExecutableOperatorPayload::Value { @@ -190,7 +222,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { .into(), ); } - finalize_exact(node, inputs) + finalize_exact(node, inputs, &context) } ExecutableOperatorPayload::SummaryAgg { family, @@ -255,12 +287,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "keyed maintenance updates require explicit row identity routing".into(), ); } - let mut updater = asap_physical_operators::factory::create_planner_accumulator( - family, input, grouping, - )?; - if updater.is_keyed() { - return Err("keyed maintenance accumulator requires an item expression".into()); - } + let _ = grouping; let output_groups = values .keys() .map(|group| { @@ -290,11 +317,39 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { "maintenance sink requires one explicitly reduced output population".into(), ); } - for (timestamp_ms, value) in values.values().flatten() { - let weight = evaluate_weight(&input.weight, *value, name)?; - updater.validate_single_input(weight)?; - updater.update_single(weight, *timestamp_ms); - } + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![ + ( + "value", + planner_types::post_asap::SummaryFamilyType::Plain( + planner_types::pre_asap::DataType::Float64, + ), + ), + ( + "time", + planner_types::post_asap::SummaryFamilyType::Plain( + planner_types::pre_asap::DataType::Timestamp, + ), + ), + ]); + let rows = values + .values() + .flatten() + .map(|(time, value)| { + Ok(vec![ + Value::Float64(evaluate_weight(&input.weight, *value, name)?), + Value::Timestamp(*time), + ]) + }) + .collect::, String>>()?; + let builder = + Operator::summary_build(schema.clone(), family.clone(), 0, Some(1), vec![]) + .map_err(|e| e.to_string())?; + let mut result = native_rows(schema, rows, vec![builder], &context)?; + let Some(Value::Summary { state, .. }) = result.pop().and_then(|mut row| row.pop()) + else { + return Err("native summary builder did not return state".into()); + }; let timestamp = values .values() .flatten() @@ -304,7 +359,7 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { Ok(MaintenanceValue::SummaryWindows { states: BTreeMap::from([( output_groups.into_iter().next().unwrap(), - vec![(timestamp, Arc::from(updater.into_accumulator()))].into(), + vec![(timestamp, state)].into(), )]), family: family.clone(), }) @@ -316,6 +371,92 @@ impl PrecomputeOperatorRegistry for OperatorAdapter<'_> { } } +// These functions translate deployment values into native batches. Window and +// catalog checks stay here; the library owns all computation on batch values. +fn native_schema( + fields: Vec<(&str, planner_types::post_asap::SummaryFamilyType)>, +) -> asap_physical_operators::dag::values::Schema { + Arc::new(planner_types::post_asap::SummarySchema { + fields: fields + .into_iter() + .map(|(name, dtype)| planner_types::post_asap::SummaryField { + name: name.into(), + dtype, + nullable: false, + }) + .collect(), + time_index: None, + }) +} +fn native_rows( + schema: asap_physical_operators::dag::values::Schema, + rows: Vec>, + operators: Vec, + context: &RunContext, +) -> Result>, String> { + use asap_physical_operators::dag::{batch_execution::evaluate_batch, values::Batch}; + let input = Batch::try_new(schema, rows).map_err(|e| e.to_string())?; + Ok(evaluate_batch(input, operators, context.clone()) + .map_err(|e| e.to_string())? + .into_iter() + .flat_map(|b| b.rows().to_vec()) + .collect()) +} +fn native_arithmetic( + op: &planner_types::post_asap::BinaryOperator, + inputs: Vec<(i64, f64, f64)>, + context: &RunContext, +) -> Result, String> { + use asap_physical_operators::dag::{ + operators::{Expression, Operator}, + values::Value, + }; + use planner_types::{post_asap::SummaryFamilyType, pre_asap::DataType}; + let schema = native_schema(vec![ + ("time", SummaryFamilyType::Plain(DataType::Timestamp)), + ("left", SummaryFamilyType::Plain(DataType::Float64)), + ("right", SummaryFamilyType::Plain(DataType::Float64)), + ]); + let project = Operator::project( + schema.clone(), + vec![ + ("time".into(), Expression::Column(0)), + ( + "value".into(), + Expression::Binary { + operator: op.clone(), + left: Box::new(Expression::Column(1)), + right: Box::new(Expression::Column(2)), + }, + ), + ], + ) + .map_err(|e| e.to_string())?; + let rows = native_rows( + schema, + inputs + .into_iter() + .map(|(time, left, right)| { + vec![ + Value::Timestamp(time), + Value::Float64(left), + Value::Float64(right), + ] + }) + .collect(), + vec![project], + context, + )?; + rows.into_iter() + .map(|row| match row.as_slice() { + [Value::Timestamp(time), Value::Float64(value)] if value.is_finite() => { + Ok((*time, *value)) + } + _ => Err("maintenance binary produced a non-finite update".into()), + }) + .collect() +} + fn validate_maintenance_grouping( target: &asap_types::PrecomputeMaterialization, source: &asap_types::PrecomputeMaterialization, @@ -408,9 +549,10 @@ fn evaluate_aligned_binary( node: &ExecutableDagNode, operator: &planner_types::post_asap::BinaryOperator, inputs: &[Arc], + context: &RunContext, ) -> Result { use planner_types::pre_asap::BinaryOpKind; - let BinaryOpKind::Arithmetic(arithmetic) = &operator.kind else { + let BinaryOpKind::Arithmetic(_) = &operator.kind else { return Err("maintenance binary currently requires arithmetic".into()); }; if operator.vector_match.is_some() { @@ -468,14 +610,9 @@ fn evaluate_aligned_binary( let right = right_rows .get(×tamp) .ok_or("maintenance binary requires matching timestamp sets")?; - let value = asap_physical_operators::arithmetic::evaluate_float64_arithmetic( - arithmetic, left, *right, - ); - if !value.is_finite() { - return Err("maintenance binary produced a non-finite update".into()); - } - joined.push((timestamp, value)); + joined.push((timestamp, left, *right)); } + let joined = native_arithmetic(operator, joined, context)?; values.insert(group.clone(), joined); } Ok(MaintenanceValue::Rows { @@ -488,6 +625,7 @@ fn evaluate_aligned_binary( fn finalize_exact( node: &ExecutableDagNode, inputs: &[Arc], + context: &RunContext, ) -> Result { use planner_types::post_asap::{ExactKind, SummaryFamilyType}; let [input] = inputs else { @@ -534,12 +672,39 @@ fn finalize_exact( let field = maintenance_float64_column(node)?; let mut values = PopulationRows::new(); for (group, states) in groups { + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![("state", family.clone())]); + let readout = Operator::readout(schema.clone(), 0, statistic, Default::default()) + .map_err(|e| e.to_string())?; + let rows = native_rows( + schema, + states + .iter() + .map(|(_, state)| { + vec![Value::Summary { + family: family.clone(), + state: Arc::clone(state), + }] + }) + .collect(), + vec![readout], + context, + )?; let rows = states .into_iter() - .map(|(timestamp, state)| { - let value = state - .query_statistic(statistic, &None, &std::collections::HashMap::new()) - .map_err(|error| error.to_string())?; + .zip(rows) + .map(|((timestamp, _), row)| { + let value = + match row.first() { + Some(Value::Float64(value)) => *value, + Some(Value::Int64(value)) if value.unsigned_abs() <= (1u64 << 53) => { + *value as f64 + } + _ => return Err( + "exact readout cannot be represented by the installed Float64 schema" + .to_string(), + ), + }; if !value.is_finite() { return Err("exact maintenance finalization produced a non-finite value".into()); } @@ -556,7 +721,10 @@ fn finalize_exact( }) } -fn merge_inputs(inputs: &[Arc]) -> Result { +fn merge_inputs( + inputs: &[Arc], + context: &RunContext, +) -> Result { let mut grouped = BTreeMap::, Option)>::new(); let mut expected_groups = None; let mut family = None; @@ -613,19 +781,36 @@ fn merge_inputs(inputs: &[Arc]) -> Result().map(|state| state.family().clone()) + }).or_else(|| first.as_any().is::().then_some( + planner_types::post_asap::SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, planner_types::post_asap::ExactParams::Sum) + )).ok_or("summary merge requires a registered state family")?; + use asap_physical_operators::dag::{operators::Operator, values::Value}; + let schema = native_schema(vec![("state", state_family.clone())]); + let rows = std::iter::once(*first) + .chain(rest.iter().copied()) + .map(|state| { + vec![Value::Summary { + family: state_family.clone(), + state: Arc::clone(state), + }] + }) + .collect(); + let merge = + Operator::summary_merge(schema.clone(), 0, vec![]).map_err(|e| e.to_string())?; + let mut output = native_rows(schema, rows, vec![merge], context)?; + let Some(Value::Summary { state: merged, .. }) = output.pop().and_then(|mut row| row.pop()) + else { + return Err("native summary merge did not return state".into()); + }; let Some(timestamp) = timestamp else { return Ok(MaintenanceValue::Summary { - state: Arc::from(merged), + state: merged, family, }); }; - result.insert(group, vec![(timestamp, Arc::from(merged))].into()); + result.insert(group, vec![(timestamp, merged)].into()); } Ok(MaintenanceValue::SummaryWindows { states: result, @@ -651,31 +836,33 @@ fn frozen_cohort_lineage( } let mut ordered: Vec<_> = inputs.iter().collect(); ordered.sort_by(|left, right| { - (left.definition, left.sid, &left.group).cmp(&(right.definition, right.sid, &right.group)) + (left.stored_output_reference, &left.group) + .cmp(&(right.stored_output_reference, &right.group)) }); if ordered.windows(2).any(|pair| { - (pair[0].definition, pair[0].sid, &pair[0].group) - == (pair[1].definition, pair[1].sid, &pair[1].group) + (pair[0].stored_output_reference, &pair[0].group) + == (pair[1].stored_output_reference, &pair[1].group) }) { - return Err("immutable lineage repeats a physical population".into()); + return Err("immutable lineage repeats a stored-output population".into()); } let multiple = ordered.len() > 1; let mut lineage = Sha256::new(); - if multiple { - lineage.update(b"immutable-maintenance-input-v2"); - lineage.update((ordered.len() as u64).to_be_bytes()); - } else { - // Preserve the existing durable single-input receipt identity. - lineage.update(b"immutable-maintenance-input-v1"); - } + lineage.update(b"immutable-stored-output-input-v3"); + lineage.update((ordered.len() as u64).to_be_bytes()); for input in ordered { if &input.generation != generation || input.windows.is_empty() { return Err("immutable lineage has mixed generations or empty windows".into()); } - lineage.update(input.sid.to_be_bytes()); - let metadata = - serde_json::to_vec(&(&input.definition, &input.generation, &input.group, expected)) - .map_err(|error| error.to_string())?; + if input.stored_output_reference.definition_id != input.definition { + return Err("immutable input output differs from its definition".into()); + } + let metadata = serde_json::to_vec(&( + &input.stored_output_reference, + &input.generation, + &input.group, + expected, + )) + .map_err(|error| error.to_string())?; if multiple { lineage.update((metadata.len() as u64).to_be_bytes()); } @@ -1198,19 +1385,8 @@ fn execute_finite_source_cohort( config.population_key_encoding, &pairs, )?; - let kind = crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); let target_sid = - resolver.resolve_with_reactivation(&config.metric, &attrs, &kind, |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, *target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation) - { - return Err("finite maintenance generation changed".into()); - } - Ok(activation) - })?; + store.resolve_output_storage_handle(resolver, *target, &attrs, Some(generation))?; if existing .get(&target_sid) .and_then(|groups| groups.get(&output_group)) @@ -1354,20 +1530,8 @@ fn execute_finite_complete_populations( config.population_key_encoding, &[], )?; - let kind = crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - let target_sid = resolver - .resolve_with_reactivation(&config.metric, &attrs, &kind, |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation.as_ref()) - { - return Err("complete maintenance generation changed".into()); - } - Ok(activation) - }) - .map_err(|error| error.to_string())?; + let target_sid = + store.resolve_output_storage_handle(resolver, target, &attrs, Some(generation))?; if store .completed_maintenance_coordinates(target, generation)? .get(&target_sid) @@ -1509,25 +1673,11 @@ pub(crate) fn execute_finite_maintenance( config.population_key_encoding, &pairs, )?; - let kind = - crate::storage_engines::sketch_db::data::materialization_kind_for_config( - config, - ); - let target_sid = resolver.resolve_with_reactivation( - &config.metric, + let target_sid = store.resolve_output_storage_handle( + resolver, + *target, &attrs, - &kind, - |sid| { - store.validate_routed_catalog_generation(Some(generation))?; - let activation = store.authorize_series_reactivation(sid, *target)?; - if activation - .as_deref() - .is_some_and(|actual| actual != generation) - { - return Err("finite maintenance generation changed".into()); - } - Ok(activation) - }, + Some(generation), )?; for (start, _) in &windows { if (*start as i128 - config.pane_origin_ms.unwrap_or(0) as i128) @@ -1617,7 +1767,7 @@ struct CommitRegistry(Mutex); impl CommitRegistry { fn plan_snapshot( &self, - plans: &StreamingConfigHandle, + plans: &InstalledPrecomputePlanHandle, ) -> Result>, String> { let mut state = self.0.lock().map_err(|_| "commit registry poisoned")?; // Read the authoritative generation while holding the registry lock, @@ -1785,13 +1935,13 @@ impl IdempotentCommitSink for CommitRegistry { /// With no matching DAG, the source output is forwarded unchanged. pub struct MaintenanceDagSink { inner: Arc, - plans: StreamingConfigHandle, + plans: InstalledPrecomputePlanHandle, commits: CommitRegistry, batch_guard: Mutex<()>, } impl MaintenanceDagSink { - pub fn new(inner: Arc, plans: StreamingConfigHandle) -> Self { + pub fn new(inner: Arc, plans: InstalledPrecomputePlanHandle) -> Self { Self { inner, plans, @@ -1854,6 +2004,8 @@ impl MaintenanceDagSink { }, configs: &plan.precompute_plan.materializations, }; + let mut selected_outputs = Vec::new(); + let mut horizons = Vec::new(); for sink_node in &installed.binding.precompute_sinks { // Derived summaries consume complete immutable windows at the // completion barrier, never additive worker fragments. @@ -1936,18 +2088,24 @@ impl MaintenanceDagSink { if self.commits.is_published(&key)? { continue; } - let value = execute_precompute_sink( - &dag, - &installed.binding, - *sink_node, - key.clone(), - &adapter, - &self.commits, - ) - .map_err(schedule_error)?; + selected_outputs.push((*sink_node, key)); + horizons.push(horizon_ms); + } + let values = execute_precompute_sinks( + &dag, + &installed.binding, + &selected_outputs, + &adapter, + &self.commits, + ) + .map_err(schedule_error)?; + for (((_, key), horizon_ms), value) in + selected_outputs.into_iter().zip(horizons).zip(values) + { + let target = key.summary_definition; let mut target_output = output.clone(); target_output.policy_fp = target.into(); - target_output.series_id = None; + target_output.storage_handle = None; derived.push(( Some((key, horizon_ms)), target_output, @@ -2152,6 +2310,19 @@ pub(crate) fn affected_materializations( #[cfg(test)] mod tests { + fn test_context() -> asap_physical_operators::dag::RunContext { + use asap_physical_operators::dag::{Limits, RunContext, Scope}; + RunContext::new( + Scope::Ingestion { + window_start_ms: 0, + window_end_ms: 10_000, + revision: 1, + }, + Limits::default(), + ) + .unwrap() + } + use super::*; use asap_physical_operators::accumulators::SumAccumulator; use planner_types::post_asap::{ @@ -2190,7 +2361,10 @@ mod tests { let mut state = asap_physical_operators::accumulators::SumAccumulator::new(); state.update(value); FrozenExactWindows { - sid, + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + definition(id), + ), + storage_handle: sid, definition: definition(id), generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2209,6 +2383,13 @@ mod tests { }; let baseline = frozen_cohort_lineage(&[make(10, 1, 3.0), make(20, 2, 5.0)], &expected).unwrap(); + let mut relocated = make(20, 2, 5.0); + relocated.storage_handle = 999; + assert_eq!( + baseline, + frozen_cohort_lineage(&[make(10, 1, 3.0), relocated], &expected).unwrap(), + "local row relocation must not change stored-output lineage" + ); assert_eq!( baseline, frozen_cohort_lineage(&[make(20, 2, 5.0), make(10, 1, 3.0)], &expected).unwrap() @@ -2221,6 +2402,12 @@ mod tests { baseline, frozen_cohort_lineage(&[make(10, 1, 3.0), make(21, 2, 5.0)], &expected).unwrap() ); + let mut changed_output = make(20, 2, 5.0); + changed_output.stored_output_reference.stored_output_id.0 += 100; + assert_ne!( + baseline, + frozen_cohort_lineage(&[make(10, 1, 3.0), changed_output], &expected).unwrap() + ); let mut changed = make(20, 2, 5.0); changed.group.insert("instance".into(), "other".into()); assert_ne!( @@ -2245,6 +2432,30 @@ mod tests { .is_err()); } + // Ingestion adapters must execute native operators in the parent's scope. + #[test] + fn native_merge_uses_the_parent_budget_and_cancellation() { + let input = Arc::new(MaintenanceValue::summary(Arc::new( + SumAccumulator::with_sum(3.), + ))); + let context = test_context(); + let output = merge_inputs(&[Arc::clone(&input)], &context).unwrap(); + assert_eq!( + output + .state() + .unwrap() + .query_statistic(asap_types::Statistic::Sum, &None, &Default::default()) + .unwrap(), + 3. + ); + assert!(context.peak_bytes() > 0); + context.cancel(); + assert!(merge_inputs(&[input], &context) + .err() + .unwrap() + .contains("cancelled")); + } + fn node(id: u32) -> ExecutableDagNode { ExecutableDagNode { id: PostAsapNodeId(id), @@ -2283,7 +2494,10 @@ mod tests { fn frozen_adapter_resolves_each_materialized_frontier_without_aliasing() { use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType, SummaryField}; let make = |id| crate::storage_engines::sketch_db::index::FrozenExactWindows { - sid: id, + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + definition(id), + ), + storage_handle: id, definition: definition(id), generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2337,7 +2551,11 @@ mod tests { .unwrap() .is_none()); let merged = adapter - .execute(&node(3), &[Arc::new(first), Arc::new(second)]) + .execute( + &node(3), + &[Arc::new(first), Arc::new(second)], + test_context(), + ) .unwrap(); assert_eq!( merged @@ -2422,7 +2640,10 @@ mod tests { }; let frozen_inputs = [ crate::storage_engines::sketch_db::index::FrozenExactWindows { - sid: 1, + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + source_definition, + ), + storage_handle: 1, definition: source_definition, generation: Arc::new(asap_types::sds::CatalogGeneration { schema_version: 2, @@ -2456,7 +2677,7 @@ mod tests { ExactParams::Sum, )), }); - let row = adapter.execute(&read, &[source]).unwrap(); + let row = adapter.execute(&read, &[source], test_context()).unwrap(); let mut aggregate = node(3); aggregate.payload = ExecutableOperatorPayload::SummaryAgg { family: target_family, @@ -2468,7 +2689,9 @@ mod tests { reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), }; - let result = adapter.execute(&aggregate, &[Arc::new(row)]).unwrap(); + let result = adapter + .execute(&aggregate, &[Arc::new(row)], test_context()) + .unwrap(); let mut kwargs = std::collections::HashMap::new(); kwargs.insert("quantile".into(), "0.5".into()); assert_eq!( @@ -3113,7 +3336,7 @@ mod tests { .load_strict() .unwrap() .iter() - .all(|record| record.sid != 1)); + .all(|record| record.storage_handle != 1)); persistence.shutdown(); return; } @@ -3296,7 +3519,7 @@ mod tests { .load_strict() .unwrap() .iter() - .all(|record| record.sid != 2)); + .all(|record| record.storage_handle != 2)); persistence.shutdown(); } @@ -3334,9 +3557,13 @@ mod tests { let left = rows(vec![(2_000, 7.0), (1_000, 5.0)]); let right = rows(vec![(1_000, 2.0), (2_000, 3.0)]); // Arrival order cannot exchange windows, and subtraction retains edge order. - let MaintenanceValue::Rows { values, .. } = - evaluate_aligned_binary(&operation, &operator, &[left.clone(), right.clone()]).unwrap() - else { + let MaintenanceValue::Rows { values, .. } = evaluate_aligned_binary( + &operation, + &operator, + &[left.clone(), right.clone()], + &test_context(), + ) + .unwrap() else { panic!("expected rows") }; assert_eq!(values[&BTreeMap::new()], vec![(1_000, 3.0), (2_000, 4.0)]); @@ -3363,6 +3590,7 @@ mod tests { &operation, &operator, &[grouped_left.clone(), grouped_right], + &test_context(), ) .unwrap() else { panic!("expected grouped rows") @@ -3377,7 +3605,8 @@ mod tests { &[ grouped_left, grouped(BTreeMap::from([(a, vec![(1_000, 2.0)])])) - ] + ], + &test_context() ) .is_err()); let binding = BackendExecutableBinding { @@ -3396,7 +3625,7 @@ mod tests { }; operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; assert!(frozen - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_ok()); let live = OperatorAdapter { binding: &binding, @@ -3407,14 +3636,14 @@ mod tests { configs: &[], }; assert!(live - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_err()); operation.payload = ExecutableOperatorPayload::Binary { operator: operator.clone(), }; operation.output_state = planner_types::post_asap::ExecutionDataState::QUERY_ROWS; assert!(frozen - .execute(&operation, &[left.clone(), right.clone()]) + .execute(&operation, &[left.clone(), right.clone()], test_context()) .is_err()); operation.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; @@ -3431,19 +3660,27 @@ mod tests { }), Arc::new(MaintenanceValue::summary(sum(2.0))), ] { - assert!( - evaluate_aligned_binary(&operation, &operator, &[left.clone(), invalid]).is_err() - ); + assert!(evaluate_aligned_binary( + &operation, + &operator, + &[left.clone(), invalid], + &test_context() + ) + .is_err()); } operator.kind = BinaryOpKind::Arithmetic(ArithmeticOpKind::Div); assert!(evaluate_aligned_binary( &operation, &operator, - &[left.clone(), rows(vec![(1_000, 0.0), (2_000, 3.0)])] + &[left.clone(), rows(vec![(1_000, 0.0), (2_000, 3.0)])], + &test_context() ) .is_err()); operation.output_schema.fields[1].dtype = SummaryFamilyType::Plain(DataType::Int64); - assert!(evaluate_aligned_binary(&operation, &operator, &[left, right]).is_err()); + assert!( + evaluate_aligned_binary(&operation, &operator, &[left, right], &test_context()) + .is_err() + ); } #[test] @@ -3464,7 +3701,7 @@ mod tests { )]), family, }); - assert!(merge_inputs(&[inputs.clone(), different_group]).is_err()); + assert!(merge_inputs(&[inputs.clone(), different_group], &test_context()).is_err()); let mut read = node(2); read.output_schema.fields = vec![SummaryField { name: "value".into(), @@ -3472,15 +3709,16 @@ mod tests { nullable: false, }]; let MaintenanceValue::Rows { values, .. } = - finalize_exact(&read, &[inputs.clone()]).unwrap() + finalize_exact(&read, &[inputs.clone()], &test_context()).unwrap() else { panic!("expected finalized rows") }; assert_eq!(values[&BTreeMap::new()], vec![(1_000, 2.0), (2_000, 7.0)]); // Merge is a semantic DAG operation, not an implicit batch optimization. // Finalizing after it emits exactly one value instead of two updates. - let merged = Arc::new(merge_inputs(&[inputs]).unwrap()); - let MaintenanceValue::Rows { values, .. } = finalize_exact(&read, &[merged]).unwrap() + let merged = Arc::new(merge_inputs(&[inputs], &test_context()).unwrap()); + let MaintenanceValue::Rows { values, .. } = + finalize_exact(&read, &[merged], &test_context()).unwrap() else { panic!("expected finalized row") }; @@ -3495,7 +3733,7 @@ mod tests { )), }); assert!( - matches!(finalize_exact(&read, &[integer_state]), Err(error) if error.contains("Float64")) + matches!(finalize_exact(&read, &[integer_state], &test_context()), Err(error) if error.contains("Float64")) ); } @@ -3522,7 +3760,7 @@ mod tests { ]; read.output_schema.time_index = Some(0); let MaintenanceValue::Rows { values, name, .. } = - finalize_exact(&read, &[Arc::clone(&input)]).unwrap() + finalize_exact(&read, &[Arc::clone(&input)], &test_context()).unwrap() else { panic!("expected typed rows") }; @@ -3550,7 +3788,7 @@ mod tests { copy.output_schema.fields[1].name = "ts".into(); malformed.push(copy); for malformed in malformed { - assert!(finalize_exact(&malformed, &[Arc::clone(&input)]).is_err()); + assert!(finalize_exact(&malformed, &[Arc::clone(&input)], &test_context()).is_err()); } let untimed = Arc::new(MaintenanceValue::Summary { state: sum(10.0), @@ -3559,7 +3797,7 @@ mod tests { ExactParams::Sum, )), }); - assert!(finalize_exact(&read, &[untimed]).is_err()); + assert!(finalize_exact(&read, &[untimed], &test_context()).is_err()); } #[test] @@ -3590,7 +3828,11 @@ mod tests { reduction: Reduction::by(vec![]), grouping: GroupingStrategy::default(), }; - let error = adapter.execute(&aggregate, &[Arc::new(MaintenanceValue::summary(sum(7.0)))]); + let error = adapter.execute( + &aggregate, + &[Arc::new(MaintenanceValue::summary(sum(7.0)))], + test_context(), + ); assert!(matches!(error, Err(reason) if reason.contains("typed update evaluator"))); } @@ -3651,8 +3893,10 @@ mod tests { name: "value".into(), timestamped: true, }; - assert!(matches!(adapter.execute(&aggregate, &[Arc::new(rows)]), - Err(error) if error.contains("one explicitly reduced output population"))); + assert!( + matches!(adapter.execute(&aggregate, &[Arc::new(rows)], test_context()), + Err(error) if error.contains("one explicitly reduced output population")) + ); } #[test] @@ -3717,8 +3961,10 @@ mod tests { name: "value".into(), timestamped: true, }; - assert!(matches!(adapter.execute(&aggregate, &[Arc::new(rows)]), - Err(error) if error.contains("positive representable domain"))); + assert!( + matches!(adapter.execute(&aggregate, &[Arc::new(rows)], test_context()), + Err(error) if error.contains("positive representable domain")) + ); } } @@ -3822,7 +4068,7 @@ mod tests { #[test] fn downstream_failure_does_not_acknowledge_maintenance_publication() { use crate::storage_engines::types::{ - ActivePhysicalPlanHandle, RuntimePhysicalPlan, StreamingConfig, + ActivePhysicalPlanHandle, InstalledPrecomputePlan, RuntimePhysicalPlan, }; use asap_types::executable_plan::{InstalledPostAsapDag, OwnedPostAsapDag}; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -3918,7 +4164,7 @@ mod tests { summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - streaming_config: Arc::new(StreamingConfig::new(Default::default())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Default::default())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3933,9 +4179,9 @@ mod tests { }); let sink = MaintenanceDagSink::new( downstream.clone(), - StreamingConfigHandle::from_active_physical_plan(ActivePhysicalPlanHandle::new( - active.clone(), - )), + InstalledPrecomputePlanHandle::from_active_physical_plan( + ActivePhysicalPlanHandle::new(active.clone()), + ), ); let batch = || { (0..count) diff --git a/data_plane/src/precompute_engine/mod.rs b/data_plane/src/precompute_engine/mod.rs index 7c8176b5..726801a2 100644 --- a/data_plane/src/precompute_engine/mod.rs +++ b/data_plane/src/precompute_engine/mod.rs @@ -18,3 +18,5 @@ pub mod worker; pub use engine::{PrecomputeEngine, PrecomputeWorkerDiagnostics}; pub use ingest_handler::IngestState; + +pub mod partitioning; diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 3713e0b0..33782c92 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -1,7 +1,7 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::ingest_handler::IngestObservability; use crate::storage_engines::sketch_db::index::SketchStore; -use crate::storage_engines::types::hot_reload_config::StreamingConfigHandle; +use crate::storage_engines::types::hot_reload_config::InstalledPrecomputePlanHandle; use crate::storage_engines::types::{AggregateCore, PrecomputedOutput}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; @@ -45,20 +45,14 @@ fn consume_in_order(items: Vec, mut persist: impl FnMut(&T) -> bool) -> us failed } -/// Write completed precompute windows to the sid-keyed sketch store. -/// -/// Read one streaming-config snapshot per batch and resolve each policy through -/// its fingerprint. Consume accumulators in order so catch-up batches release -/// each pane as soon as it is serialized. +/// Publish completed windows using their selected output reference and complete +/// plan/population/window address. Local row handles are checked against this +/// address and cannot redirect a queued output to another stored summary. pub struct SketchStoreSink { summary_store: Arc, - hot_reload: StreamingConfigHandle, - /// Single shared resolver across the ingest + precompute paths. Under - /// the registry-allocated sid model (PR-1..3), this is the canonical - /// mint authority — precompute sids share the same `next_sid` counter - /// as OTel-sketch sids, so the two paths can never collide on identity - /// even when the same `(metric, attrs)` carries both a sketch and an - /// exact precompute. + hot_reload: InstalledPrecomputePlanHandle, + /// Durable local-handle allocation keyed by the complete output population + /// identity. Shared with ingest and backfill so they address the same rows. series_resolver: Arc, /// CQ-6 — optional handle to the shared `IngestObservability` so /// policy-miss drops land in the same counter the OTLP ingest path @@ -73,7 +67,7 @@ pub struct SketchStoreSink { impl SketchStoreSink { pub fn new( summary_store: Arc, - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, series_resolver: Arc, ) -> Self { Self { @@ -145,47 +139,41 @@ impl SketchStoreSink { } }; let agg_cfg = &agg_cfg; - let resolver = self.series_resolver.clone(); + let Some(reference) = cfg.stored_output_reference(output.policy_fp.into()) else { + return false; + }; + let Some(start_ms) = i64::try_from(output.start_timestamp).ok() else { + return false; + }; + let Some(end_ms) = i64::try_from(output.end_timestamp).ok() else { + return false; + }; + let population = output.population_labels.clone().unwrap_or_else(|| { + agg_cfg + .grouping_labels + .iter() + .cloned() + .zip(output.key.clone().unwrap_or_default().labels) + .collect() + }); + let (plan_id, plan_version) = output + .catalog_generation + .as_deref() + .map(|generation| (generation.plan_id, generation.plan_version)) + .unwrap_or((0, 0)); + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + output: reference, + population, + window: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, + }; let persist = |writer: &crate::storage_engines::sketch_db::index::SummaryPublicationWriter<'_>| { - if let Some(sid) = output.series_id { - return writer - .ingest_precompute_with_series_id(sid, agg_cfg, output, accumulator) - .inspect(|_| { - crate::precompute_engine::metrics::record_materialized_outputs(1) - }); - } - self.summary_store - .validate_routed_catalog_generation(output.catalog_generation.as_deref()) - .ok()?; writer - .ingest_precompute_for_agg_config( - |metric, fp, ak| { - resolver - .resolve_with_reactivation(metric, fp, ak, |sid| { - self.summary_store.validate_routed_catalog_generation( - output.catalog_generation.as_deref(), - )?; - let activation = - self.summary_store.authorize_series_reactivation( - sid, - output.policy_fp.into(), - )?; - if let Some(generation) = &activation { - if output.catalog_generation.as_deref() - != Some(generation.as_ref()) - { - return Err( - "unbound or stale output cannot reactivate a series" - .into(), - ); - } - } - Ok(activation) - }) - .map_err(|error| warn!(%error, "series reactivation rejected")) - .ok() - }, + .write_stored_summary( + &address, + &self.series_resolver, agg_cfg, output, accumulator, @@ -336,7 +324,7 @@ impl OutputSink for NoopOutputSink { mod tests { use super::*; use crate::storage_engines::sketch_db::index::{AggKind, SeriesLookup}; - use crate::storage_engines::types::{KeyByLabelValues, StreamingConfig}; + use crate::storage_engines::types::{InstalledPrecomputePlan, KeyByLabelValues}; use asap_physical_operators::accumulators::{DDSketchAccumulator, SumAccumulator}; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; @@ -425,8 +413,8 @@ mod tests { let agg_id = cfg.policy_fp_u64(); let mut configs = HashMap::new(); configs.insert(agg_id, cfg); - let streaming = StreamingConfig::new(configs); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(configs); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( @@ -451,7 +439,7 @@ mod tests { .list_by_status(crate::storage_engines::sketch_db::lifecycle::AggStatus::Active); assert_eq!(instances.len(), 1); let meta = instances[0].clone(); - let sid = meta.sid; + let sid = meta.storage_handle; assert_eq!(summary_store.classify(sid), SeriesLookup::Hit); assert!( matches!( @@ -494,7 +482,10 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(fingerprint.0, cfg)]))), + InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new(HashMap::from([( + fingerprint.0, + cfg, + )]))), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -511,7 +502,7 @@ mod tests { .emit_batch(vec![(output(), Box::new(SumAccumulator::with_sum(11.0)))]) .is_err()); let mut stale_output = output(); - stale_output.series_id = Some(old_sid); + stale_output.storage_handle = Some(old_sid); stale_output.catalog_generation = Some(Arc::new(catalog.reference().unwrap())); let mut next = catalog; next.plan_version += 1; @@ -542,7 +533,7 @@ mod tests { .emit_batch(vec![(output(), Box::new(SumAccumulator::with_sum(101.0)))]) .is_err()); let mut stale_routed_output = output(); - stale_routed_output.series_id = Some(new_sid); + stale_routed_output.storage_handle = Some(new_sid); assert!(sink .emit_batch(vec![( stale_routed_output, @@ -550,7 +541,7 @@ mod tests { )]) .is_err()); let mut missing_generation = PrecomputedOutput::new(1000, 2000, None, fingerprint); - missing_generation.series_id = Some(new_sid); + missing_generation.storage_handle = Some(new_sid); assert!(sink .emit_batch(vec![( missing_generation, @@ -573,8 +564,9 @@ mod tests { cfg.parameters .insert("alpha".into(), serde_json::json!(0.01)); let policy_fp = cfg.policy_fp_u64(); - let hot_reload = - StreamingConfigHandle::new(StreamingConfig::new(HashMap::from([(policy_fp, cfg)]))); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new( + HashMap::from([(policy_fp, cfg)]), + )); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), @@ -602,9 +594,14 @@ mod tests { .. } )); - assert_eq!(summary_store.query_range(meta.sid, 1_000, 2_000).len(), 1); + assert_eq!( + summary_store + .query_range(meta.storage_handle, 1_000, 2_000) + .len(), + 1 + ); assert!(summary_store - .query_exact_agg_range(meta.sid, 1_000, 2_000) + .query_exact_agg_range(meta.storage_handle, 1_000, 2_000) .is_empty()); } @@ -612,8 +609,8 @@ mod tests { fn sketch_index_sink_reports_unknown_policy_as_failure() { // Streaming config does NOT contain agg_id=99 — the sink // reports a recoverable error rather than acknowledging a lost write. - let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), @@ -633,8 +630,8 @@ mod tests { /// handle wired in, the `dropped_policy_miss` counter must tick. #[test] fn sink_increments_policy_miss_counter_on_registry_miss() { - let streaming = StreamingConfig::new(HashMap::new()); - let hot_reload = StreamingConfigHandle::new(streaming.clone()); + let streaming = InstalledPrecomputePlan::new(HashMap::new()); + let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); let obs = Arc::new(IngestObservability::new()); let sink = SketchStoreSink::new( diff --git a/data_plane/src/precompute_engine/partitioning.rs b/data_plane/src/precompute_engine/partitioning.rs new file mode 100644 index 00000000..09ef174f --- /dev/null +++ b/data_plane/src/precompute_engine/partitioning.rs @@ -0,0 +1,115 @@ +//! Data ownership for an entire installed precompute graph. +use std::collections::{BTreeMap, BTreeSet}; + +use asap_types::precompute_plan::PrecomputePlan; +use xxhash_rust::xxh64::xxh64; + +/// A conservative locality proof. Raw producer paths are validated separately +/// at installation. Derived graphs use one owner until their complete reduction +/// paths can establish a finer partition without a cross-worker exchange. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub enum DagPartitioning { + #[default] + SingleWorker, + Labels(Vec), + Population, +} + +impl DagPartitioning { + pub fn from_plan(plan: &PrecomputePlan) -> Self { + if plan + .materializations + .iter() + .any(|output| output.derived_input.is_some()) + { + return Self::SingleWorker; + } + let mut common: Option> = None; + for output in &plan.materializations { + let labels = output + .grouping_labels + .iter() + .cloned() + .collect::>(); + common = Some(match common { + None => labels, + Some(previous) => previous.intersection(&labels).cloned().collect(), + }); + } + match common { + Some(labels) if !labels.is_empty() => Self::Labels(labels.into_iter().collect()), + _ if !plan.materializations.is_empty() + && plan.materializations.iter().all(|output| { + output.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) + }) => + { + Self::Population + } + _ => Self::SingleWorker, + } + } + + pub fn owner( + &self, + population: &BTreeMap, + workers: usize, + ) -> Result { + if workers == 0 { + return Err("precompute requires at least one worker".into()); + } + match self { + Self::SingleWorker => Ok(0), + Self::Population => { + let key = super::group_key::GroupKey::new( + population.iter().map(|(k, v)| (k.as_str(), v.as_str())), + ); + Ok(xxh64(key.canonical_bytes(), 0) as usize % workers) + } + Self::Labels(names) => { + let pairs = names + .iter() + .map(|name| { + population + .get(name) + .map(|value| (name.as_str(), value.as_str())) + .ok_or_else(|| { + format!("DAG partition label {name} is absent from population") + }) + }) + .collect::, _>>()?; + let key = super::group_key::GroupKey::new(pairs); + Ok(xxh64(key.canonical_bytes(), 0) as usize % workers) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + // Upstream series state and downstream grouped state have one owner. + #[test] + fn service_partition_preserves_series_and_reduction_locality() { + let rule = DagPartitioning::Labels(vec!["service".into()]); + let group = BTreeMap::from([("service".into(), "api".into())]); + let expected = rule.owner(&group, 4).unwrap(); + for instance in ["a", "b", "c"] { + let mut series = group.clone(); + series.insert("instance".into(), instance.into()); + assert_eq!(rule.owner(&series, 4).unwrap(), expected); + } + assert!(rule.owner(&BTreeMap::new(), 4).is_err()); + assert!(rule.owner(&group, 0).is_err()); + let owners = (0..64) + .map(|i| { + rule.owner( + &BTreeMap::from([("service".into(), format!("service-{i}"))]), + 4, + ) + .unwrap() + }) + .collect::>(); + assert_eq!(owners.len(), 4); + } +} diff --git a/data_plane/src/precompute_engine/series_router.rs b/data_plane/src/precompute_engine/series_router.rs index 01af6231..c80323e6 100644 --- a/data_plane/src/precompute_engine/series_router.rs +++ b/data_plane/src/precompute_engine/series_router.rs @@ -10,20 +10,8 @@ use tokio::sync::mpsc; use xxhash_rust::xxh64::xxh64; /// A message sent from the router to a worker. -/// -/// B7.6 (schema-retirement #5): the per-group bucket key on `GroupSamples` -/// and `AccumulatorInput` is now a single `sid` (registry-allocated by -/// `SeriesIdResolver`), not the `(agg_id, group_key)` tuple. The grouping -/// label values are already folded into the sid via the -/// `(metric, attrs_fingerprint, agg_kind)` identity contract — so one sid -/// uniquely names one bucket, with no extra discriminator needed for -/// hashing or pane lookup. `group_key` and `policy_fp` still travel -/// alongside the sid: `group_key` is consumed at emit-time to render the -/// output label vector; `policy_fp` is the handle the worker uses to fetch -/// the source `PrecomputeMaterialization` from the hot-reload snapshot (window -/// shape, late-data policy, etc.). Together they let the worker key state -/// by sid without losing the data the legacy `(agg_id, group_key)` shape -/// carried. +/// Data partition ownership is derived from the installed DAG; storage locators +/// carried by individual inputs do not determine their destination worker. pub enum WorkerMessage { /// Immutable producer generation captured before routing. The optional /// receipt proves atomic admission; absent receipts are never fabricated. @@ -96,6 +84,11 @@ pub enum WorkerMessage { Flush, /// Finite-input barrier: acknowledge only after queued input and trailing panes reach the sink. Drain(tokio::sync::oneshot::Sender>), + /// Execute downstream DAG work after all raw windows have been sealed. + CompleteDag { + plan: Arc, + reply: tokio::sync::oneshot::Sender>, + }, /// Graceful shutdown. Shutdown, } @@ -148,6 +141,7 @@ impl fmt::Debug for WorkerMessage { .finish(), Self::Flush => f.write_str("Flush"), Self::Drain(_) => f.write_str("Drain"), + Self::CompleteDag { .. } => f.write_str("CompleteDag"), Self::Shutdown => f.write_str("Shutdown"), } } @@ -158,6 +152,7 @@ pub struct SeriesRouter { erp_observer: std::sync::OnceLock>, senders: Vec>, num_workers: usize, + plan: Option, } impl SeriesRouter { @@ -167,9 +162,25 @@ impl SeriesRouter { erp_observer: std::sync::OnceLock::new(), senders, num_workers, + plan: None, } } + pub fn with_plan( + mut self, + plan: crate::storage_engines::types::InstalledPrecomputePlanHandle, + ) -> Self { + self.plan = Some(plan); + self + } + + fn partition_rule(&self) -> super::partitioning::DagPartitioning { + self.plan + .as_ref() + .map(|plan| plan.snapshot().partitioning.clone()) + .unwrap_or(super::partitioning::DagPartitioning::Population) + } + pub fn enable_erp_observation( &self, endpoint: String, @@ -187,16 +198,15 @@ impl SeriesRouter { /// Route a pre-grouped batch of group messages to workers concurrently. /// - /// Each `GroupSamples` / `AccumulatorInput` message is routed by - /// `worker_for_sid(sid)` — same `sid` always lands on the same worker, - /// so per-bucket state stays single-owner. Messages within a single - /// worker are sent sequentially to preserve ordering. + /// Every producer uses the same DAG partition rule. Messages within one + /// worker are sent sequentially to preserve admission order. pub async fn route_group_batch( &self, messages: Vec, _ingest_received_at: Instant, generation: Option>, ) -> Result<(), Box> { + let partitioning = self.partition_rule(); // Group messages by target worker index let mut per_worker: HashMap> = HashMap::new(); for msg in messages { @@ -204,8 +214,10 @@ impl SeriesRouter { WorkerMessage::BoundInput { .. } => { return Err("input must be admitted by the router".into()) } - WorkerMessage::GroupSamples { sid, .. } => self.worker_for_sid(*sid), - WorkerMessage::AccumulatorInput { sid, .. } => self.worker_for_sid(*sid), + WorkerMessage::GroupSamples { group_key, .. } + | WorkerMessage::AccumulatorInput { group_key, .. } => { + partitioning.owner(&group_key.as_population_labels(), self.num_workers)? + } WorkerMessage::RawSamples { series_key, .. } => self.worker_for(series_key), _ => 0, }; @@ -249,16 +261,22 @@ impl SeriesRouter { String, >, ) -> Result<(), TryRouteError> { + let partitioning = self.partition_rule(); let mut pending = Vec::with_capacity(messages.len()); for message in messages { let worker_idx = match &message { - WorkerMessage::GroupSamples { sid, .. } - | WorkerMessage::AccumulatorInput { sid, .. } => self.worker_for_sid(*sid), + WorkerMessage::GroupSamples { group_key, .. } + | WorkerMessage::AccumulatorInput { group_key, .. } => partitioning + .owner(&group_key.as_population_labels(), self.num_workers) + .map_err(TryRouteError::Admission)?, WorkerMessage::RawSamples { series_key, .. } => self.worker_for(series_key), WorkerMessage::BoundInput { .. } => { return Err(TryRouteError::Admission("input already admitted".into())) } - WorkerMessage::Flush | WorkerMessage::Drain(_) | WorkerMessage::Shutdown => 0, + WorkerMessage::Flush + | WorkerMessage::Drain(_) + | WorkerMessage::Shutdown + | WorkerMessage::CompleteDag { .. } => 0, }; let permit = self.senders[worker_idx] .clone() @@ -311,14 +329,36 @@ impl SeriesRouter { Ok(()) } - /// Determine which worker handles a given sid bucket. - /// - /// Hashes the sid alone — the legacy `(agg_id, group_key)` tuple folded - /// into one u64 by `SeriesIdResolver`, so a single xxh64 over the sid - /// gives the same per-bucket sharding the tuple-hash produced. - fn worker_for_sid(&self, sid: u64) -> usize { - let hash = xxh64(&sid.to_le_bytes(), 0); - (hash as usize) % self.num_workers + pub async fn shutdown(&self) -> Result<(), String> { + for sender in &self.senders { + sender + .send(WorkerMessage::Shutdown) + .await + .map_err(|error| error.to_string())?; + } + Ok(()) + } + + pub async fn complete_dag( + &self, + plan: Arc, + ) -> Result<(), String> { + let mut replies = Vec::with_capacity(self.senders.len()); + for sender in &self.senders { + let (reply, receiver) = tokio::sync::oneshot::channel(); + sender + .send(WorkerMessage::CompleteDag { + plan: Arc::clone(&plan), + reply, + }) + .await + .map_err(|error| error.to_string())?; + replies.push(receiver); + } + for reply in replies { + reply.await.map_err(|error| error.to_string())??; + } + Ok(()) } /// Determine which worker handles a given series key (for raw mode). @@ -342,22 +382,41 @@ pub enum TryRouteError { mod tests { use super::*; - #[test] - fn test_consistent_sid_routing() { - let (senders, _receivers): (Vec<_>, Vec<_>) = - (0..4).map(|_| mpsc::channel::(10)).unzip(); - + // Shared DAG consumers must share partition ownership regardless of locator IDs. + #[tokio::test] + async fn shared_population_is_not_split_by_storage_locator() { + let (senders, mut receivers): (Vec<_>, Vec<_>) = + (0..4).map(|_| mpsc::channel::(128)).unzip(); let router = SeriesRouter::new(senders); - - // Same sid should always go to the same worker. - let w1 = router.worker_for_sid(42); - let w2 = router.worker_for_sid(42); - assert_eq!(w1, w2); - - // All resolved buckets land within the worker count. - assert!(router.worker_for_sid(7) < 4); - assert!(router.worker_for_sid(99) < 4); - assert!(router.worker_for_sid(0) < 4); + let messages = (0..32) + .map(|sid| WorkerMessage::GroupSamples { + sid, + policy_fp: PolicyFingerprint(sid), + group_key: Arc::new(GroupKey::new([("service", "api")])), + samples: vec![("counter".into(), 1, 1.0)], + ingest_received_at: Instant::now(), + }) + .collect(); + router + .route_group_batch(messages, Instant::now(), None) + .await + .unwrap(); + let counts = receivers + .iter_mut() + .map(|rx| { + let mut count = 0; + while rx.try_recv().is_ok() { + count += 1; + } + count + }) + .collect::>(); + assert_eq!(counts.iter().sum::(), 32); + assert_eq!( + counts.iter().filter(|n| **n != 0).count(), + 1, + "one service's complete DAG must stay with one worker: {counts:?}" + ); } #[test] diff --git a/data_plane/src/precompute_engine/subdag_scheduler.rs b/data_plane/src/precompute_engine/subdag_scheduler.rs index c00fab6f..6085fcab 100644 --- a/data_plane/src/precompute_engine/subdag_scheduler.rs +++ b/data_plane/src/precompute_engine/subdag_scheduler.rs @@ -1,8 +1,11 @@ +use asap_physical_operators::dag as execution; use asap_types::executable_plan::{BackendExecutableBinding, BackendNodeBinding}; +use futures::StreamExt; use planner_types::post_asap::PostAsapNodeId; use planner_types::post_asap::{ EdgeRole, ExecutableDag, ExecutableDagNode, ExecutableOperatorPayload, ExecutionDataState, }; +use std::{cell::RefCell, rc::Rc}; use std::{ collections::{BTreeMap, BTreeSet}, sync::Arc, @@ -28,7 +31,15 @@ pub trait PrecomputeOperatorRegistry { fn materialized_input(&self, _node: &ExecutableDagNode) -> Result, Self::Error> { Ok(None) } - fn execute(&self, node: &ExecutableDagNode, inputs: &[Arc]) -> Result; + fn output_bytes(&self, _value: &V) -> usize { + std::mem::size_of::().max(1) + } + fn execute( + &self, + node: &ExecutableDagNode, + inputs: &[Arc], + context: execution::RunContext, + ) -> Result; } /// Atomic persistence boundary. Implementations must return the already @@ -67,29 +78,56 @@ where R: PrecomputeOperatorRegistry, S: IdempotentCommitSink, { - if !matches!(binding.node(sink_node), Some(BackendNodeBinding::Materialization { summary_definition }) if *summary_definition == key.summary_definition) - { - return Err(ScheduleError::Invalid(format!( - "commit key materialization {:?} does not match sink {}", - key.summary_definition, sink_node.0 - ))); + let mut outputs = execute_precompute_sinks(dag, binding, &[(sink_node, key)], registry, sink)?; + Ok(outputs.remove(0)) +} + +/// Evaluate all selected stored outputs with one dependency cache. Keys must +/// describe the same input revision and window; only their output identity may +/// differ. Validation finishes before executing or committing any output. +pub fn execute_precompute_sinks( + dag: &ExecutableDag, + binding: &BackendExecutableBinding, + outputs: &[(PostAsapNodeId, MaterializationCommitKey)], + registry: &R, + sink: &S, +) -> Result>, ScheduleError> +where + R: PrecomputeOperatorRegistry, + S: IdempotentCommitSink, +{ + let mut unique = BTreeSet::new(); + for (node, key) in outputs { + if !unique.insert(node.0) + || !binding.precompute_sinks.contains(node) + || !matches!(binding.node(*node), Some(BackendNodeBinding::Materialization { summary_definition }) if *summary_definition == key.summary_definition) + { + return Err(ScheduleError::Invalid( + "commit key does not match a unique stored output binding".into(), + )); + } + let first = &outputs[0].1; + if ( + key.plan_id, + key.plan_version, + key.window_start_ms, + key.window_end_ms, + &key.input_lineage, + ) != ( + first.plan_id, + first.plan_version, + first.window_start_ms, + first.window_end_ms, + &first.input_lineage, + ) { + return Err(ScheduleError::Invalid( + "stored outputs require one evaluation window and input revision".into(), + )); + } } binding .validate_maintenance(dag) .map_err(ScheduleError::Invalid)?; - if !binding.precompute_sinks.contains(&sink_node) - || !matches!( - binding.node(sink_node), - Some(BackendNodeBinding::Materialization { .. }) - ) - { - return Err(ScheduleError::Invalid( - "node is not a precompute sink".into(), - )); - } - if let Some(committed) = sink.get(&key).map_err(ScheduleError::Sink)? { - return Ok(committed); - } let nodes = dag .nodes .iter() @@ -131,66 +169,170 @@ where }; inputs.insert(consumer, ordered); } - let mut active = BTreeSet::new(); - let mut values = BTreeMap::>::new(); - fn visit( - id: u32, - nodes: &BTreeMap, - inputs: &BTreeMap>, - active: &mut BTreeSet, - values: &mut BTreeMap>, - registry: &R, - ) -> Result<(), ScheduleError> - where - R: PrecomputeOperatorRegistry, - { - if values.contains_key(&id) { - return Ok(()); + if outputs.is_empty() { + return Ok(Vec::new()); + } + let error = Rc::new(RefCell::new(None)); + let mut sources = BTreeMap::new(); + for (node, key) in outputs { + if let Some(value) = sink.get(key).map_err(ScheduleError::Sink)? { + sources.insert(node.0, value); } - if !active.insert(id) { - return Err(ScheduleError::Invalid("precompute subDAG cycle".into())); + } + let committed = sources.keys().copied().collect::>(); + let mut graph = execution::PhysicalDag::default(); + let mut pending = outputs.iter().map(|(id, _)| id.0).collect::>(); + let mut added = BTreeSet::new(); + while let Some(id) = pending.pop() { + if !added.insert(id) { + continue; } - let node = nodes + let node = *nodes .get(&id) .ok_or_else(|| ScheduleError::Invalid(format!("missing node {id}")))?; - if node.output_state == ExecutionDataState::QUERY_ROWS { + if node.output_state.timing == planner_types::post_asap::ExecutionTiming::QueryTime { return Err(ScheduleError::Invalid(format!( "query-time node {id} in precompute dependency path" ))); } - if let Some(value) = registry - .materialized_input(node) - .map_err(ScheduleError::Operator)? - { - values.insert(id, Arc::new(value)); - active.remove(&id); - return Ok(()); - } - let child_ids = inputs.get(&id).cloned().unwrap_or_default(); - for child in &child_ids { - visit(*child, nodes, inputs, active, values, registry)?; - } - let child_values = child_ids + let source = if let Some(value) = sources.remove(&id) { + Some(value) + } else { + registry + .materialized_input(node) + .map_err(ScheduleError::Operator)? + .map(Arc::new) + }; + let children = if source.is_some() { + vec![] + } else { + inputs.get(&id).cloned().unwrap_or_default() + }; + let schemas = children .iter() - .map(|child| Arc::clone(&values[child])) - .collect::>(); - let value = registry - .execute(node, &child_values) - .map_err(ScheduleError::Operator)?; - values.insert(id, Arc::new(value)); - active.remove(&id); - Ok(()) + .map(|child| { + nodes + .get(child) + .map(|n| n.output_schema.clone()) + .ok_or_else(|| ScheduleError::Invalid(format!("missing node {child}"))) + }) + .collect::, _>>()?; + pending.extend(children.iter().copied()); + graph + .add( + u64::from(id), + children.into_iter().map(u64::from).collect(), + IngestionOperator { + node, + registry, + source, + schemas, + error: error.clone(), + }, + ) + .map_err(|e| ScheduleError::Invalid(e.to_string()))?; + } + let key = &outputs[0].1; + let context = execution::RunContext::new( + execution::Scope::Ingestion { + window_start_ms: key.window_start_ms, + window_end_ms: key.window_end_ms, + revision: key.plan_version, + }, + execution::Limits::default(), + ) + .map_err(|e| ScheduleError::Invalid(e.to_string()))?; + let roots = outputs + .iter() + .map(|(id, _)| u64::from(id.0)) + .collect::>(); + let streams = graph + .execute(&roots, context) + .map_err(|e| ScheduleError::Invalid(e.to_string()))?; + futures::executor::block_on(futures::future::try_join_all( + streams + .into_iter() + .zip(outputs) + .map(|(mut stream, (node, key))| { + let error = &error; + let committed = &committed; + async move { + let result = stream.next().await.ok_or_else(|| { + ScheduleError::Invalid("ingestion root produced no value".into()) + })?; + let value = match result { + Ok(value) => Arc::clone(value.value()), + Err(failure) => { + return Err(match error.borrow_mut().take() { + Some(error) => ScheduleError::Operator(error), + None => ScheduleError::Invalid(failure.to_string()), + }) + } + }; + if committed.contains(&node.0) { + Ok(value) + } else { + sink.commit_if_absent(key.clone(), value) + .map_err(ScheduleError::Sink) + } + } + }), + )) +} + +struct IngestionOperator<'a, V, R: PrecomputeOperatorRegistry> { + node: &'a ExecutableDagNode, + registry: &'a R, + source: Option>, + schemas: Vec, + error: Rc>>, +} +impl> + execution::PhysicalOperator, planner_types::post_asap::SummarySchema> + for IngestionOperator<'_, V, R> +{ + fn name(&self) -> &str { + "InstalledIngestionOperator" + } + fn input_schemas(&self) -> Vec { + self.schemas.clone() + } + fn output_schema(&self) -> planner_types::post_asap::SummarySchema { + self.node.output_schema.clone() + } + fn output_bytes(&self, value: &Arc) -> usize { + self.registry.output_bytes(value) + } + fn start<'a>( + &'a self, + inputs: Vec>>, + context: execution::RunContext, + ) -> Result>, execution::Error> { + Ok(futures::stream::once(async move { + if let Some(source) = &self.source { + return Ok(Arc::clone(source)); + } + let inputs = + futures::future::try_join_all(inputs.into_iter().map(|mut input| async move { + input.next().await.ok_or_else(|| { + execution::Error::Operator("ingestion input produced no value".into()) + })? + })) + .await?; + let values = inputs + .iter() + .map(|value| Arc::clone(value.value())) + .collect::>(); + self.registry + .execute(self.node, &values, context) + .map(Arc::new) + .map_err(|e| { + *self.error.borrow_mut() = Some(e); + execution::Error::Operator(format!("ingestion node {} failed", self.node.id.0)) + }) + }) + .boxed_local()) } - visit( - sink_node.0, - &nodes, - &inputs, - &mut active, - &mut values, - registry, - )?; - sink.commit_if_absent(key, values.remove(&sink_node.0).unwrap()) - .map_err(ScheduleError::Sink) } #[cfg(test)] @@ -287,6 +429,7 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + _context: execution::RunContext, ) -> Result { *self.0.lock().unwrap().entry(node.id.0).or_default() += 1; Ok(node.id.0 + inputs.iter().map(|v| **v).sum::()) @@ -325,6 +468,35 @@ mod tests { } } + // Two stored sinks in one evaluation reuse their shared upstream work. + #[test] + fn stored_sinks_share_one_evaluation() { + let mut query = node(4); + query.output_state = ExecutionDataState::QUERY_ROWS; + let dag = ExecutableDag { + nodes: vec![node(0), node(1), node(2), node(3), query], + edges: vec![edge(0, 1), edge(1, 2), edge(1, 3)], + root: PostAsapNodeId(4), + }; + let mut bindings = binding(); + bindings.precompute_sinks = vec![PostAsapNodeId(2), PostAsapNodeId(3)]; + let (dag, bindings) = maintenance_only(dag, bindings, PostAsapNodeId(3)); + let registry = Registry::default(); + let sink = Sink::default(); + execute_precompute_sinks( + &dag, + &bindings, + &[(PostAsapNodeId(2), key(2)), (PostAsapNodeId(3), key(3))], + ®istry, + &sink, + ) + .unwrap(); + assert_eq!( + *registry.0.lock().unwrap(), + BTreeMap::from([(0, 1), (1, 1), (2, 1), (3, 1)]) + ); + } + #[test] fn binary_operand_roles_survive_edge_reordering_and_reject_duplicates() { use planner_types::post_asap::BinaryOperator; @@ -336,6 +508,7 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + _context: execution::RunContext, ) -> Result { match node.id.0 { 0 => Ok(10), @@ -432,13 +605,14 @@ mod tests { &self, node: &ExecutableDagNode, inputs: &[Arc], + context: execution::RunContext, ) -> Result { assert_ne!( node.id, PostAsapNodeId(0), "absorbed source subtree must not execute" ); - self.0.execute(node, inputs) + self.0.execute(node, inputs, context) } } let mut raw = node(0); @@ -503,6 +677,13 @@ mod tests { execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink), Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") )); + let mut summary_dag = dag.clone(); + summary_dag.nodes[0].output_state.primitive = + planner_types::post_asap::DataPrimitive::SummaryState; + assert!(matches!( + execute_precompute_sink(&summary_dag, &invalid_path_binding, PostAsapNodeId(1), key(1), ®istry, &sink), + Err(ScheduleError::Invalid(message)) if message.contains("query-owned node") + )); assert!(matches!( execute_precompute_sink(&dag, &invalid_path_binding, PostAsapNodeId(1), key(0), ®istry, &sink), Err(ScheduleError::Invalid(message)) if message.contains("does not match") diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index de14847a..522e6923 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -5,7 +5,7 @@ use crate::precompute_engine::output_sink::OutputSink; use crate::precompute_engine::series_router::WorkerMessage; use crate::precompute_engine::window_manager::WindowManager; use crate::storage_engines::types::{ - AggregateCore, KeyByLabelValues, PrecomputedOutput, StreamingConfigHandle, + AggregateCore, InstalledPrecomputePlanHandle, KeyByLabelValues, PrecomputedOutput, }; #[cfg(test)] use crate::tests::accumulator_fixture::create_fixture_accumulator; @@ -39,6 +39,7 @@ use tracing::{debug, debug_span, info, warn}; struct GroupState { program: Option>, series_id: u64, + stored_output_reference: Option, catalog_generation: Option>, input_revisions: BTreeMap>, config: Arc, @@ -151,6 +152,10 @@ pub struct WorkerRuntimeConfig { /// `(metric, attrs_fingerprint, agg_kind_canonical)` identity contract on /// `SeriesIdResolver`, so one sid uniquely names one bucket. pub struct Worker { + maintenance_storage: Option<( + Arc, + Arc, + )>, erp_observer: Option>, current_input_revision: Option>, current_catalog_generation: Option>, @@ -163,7 +168,7 @@ pub struct Worker { /// Hot-reload handle — workers read config directly from ArcSwap /// instead of holding a local copy. All components see the same /// config at the same time. - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, /// Allowed lateness in ms. allowed_lateness_ms: i64, /// When true, skip aggregation and pass raw samples through. @@ -189,6 +194,47 @@ pub struct Worker { } impl Worker { + pub fn set_maintenance_storage( + &mut self, + store: Arc, + resolver: Arc, + ) { + self.maintenance_storage = Some((store, resolver)); + } + + fn complete_dag( + &self, + plan: &crate::storage_engines::types::RuntimePhysicalPlan, + ) -> Result<(), String> { + if !plan + .precompute_plan + .materializations + .iter() + .any(|output| output.derived_input.is_some()) + { + return Ok(()); + } + // The installed locality proof assigns derived graphs to one owner. + // Every raw input of this graph was routed to that same worker. + if plan.installed_precompute_plan.partitioning + != super::partitioning::DagPartitioning::SingleWorker + { + return Err("derived DAG has no validated local partitioning rule".into()); + } + if self.id != 0 { + return Ok(()); + } + let (store, resolver) = self + .maintenance_storage + .as_ref() + .ok_or("worker has no maintenance storage bindings")?; + super::maintenance_runtime::execute_finite_maintenance( + store, + resolver, + &plan.precompute_plan, + ) + } + pub fn set_erp_observer( &mut self, observer: Option>, @@ -200,7 +246,7 @@ impl Worker { id: usize, receiver: mpsc::Receiver, output_sink: Arc, - hot_reload: StreamingConfigHandle, + hot_reload: InstalledPrecomputePlanHandle, runtime_config: WorkerRuntimeConfig, group_count: Arc, worker_watermark: Arc, @@ -215,6 +261,7 @@ impl Worker { wall_clock_max_open_grace_period_ms, } = runtime_config; Self { + maintenance_storage: None, erp_observer: None, current_input_revision: None, current_catalog_generation: None, @@ -384,6 +431,16 @@ impl Worker { } let _ = reply.send(processing_error.clone().map_or(Ok(()), Err)); } + WorkerMessage::CompleteDag { plan, reply } => { + let result = match &processing_error { + Some(error) => Err(error.clone()), + None => self.complete_dag(&plan), + }; + if let Err(error) = &result { + processing_error = Some(error.clone()); + } + let _ = reply.send(result); + } WorkerMessage::Shutdown => { info!("Worker {} shutting down", self.id); if let Err(e) = self.flush_all() { @@ -416,7 +473,7 @@ impl Worker { /// hot-reload snapshot the first time we see this sid; `group_key` is /// remembered on the `GroupState` for emit-time label rendering. /// - /// Reads config directly from the `StreamingConfigHandle` + /// Reads config directly from the `InstalledPrecomputePlanHandle` /// ArcSwap handle, so new policies from a config swap are visible /// immediately — no message passing, no delay. /// Returns None if `policy_fp` has no matching config (e.g. arrived @@ -437,6 +494,7 @@ impl Worker { let gs = GroupState { program, series_id: sid, + stored_output_reference: snap.stored_output_reference(policy_fp.into()), catalog_generation: self.current_catalog_generation.clone(), input_revisions: BTreeMap::new(), window_manager: WindowManager::with_layout( @@ -659,6 +717,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, updater.take_accumulator())); debug!( @@ -735,6 +794,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -866,6 +926,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, incoming.clone_boxed_core())); } @@ -914,6 +975,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -932,6 +994,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -1117,6 +1180,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -1134,6 +1198,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -1223,6 +1288,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -1240,6 +1306,7 @@ impl Worker { &state.input_revisions, state.series_id, state.catalog_generation.as_ref(), + state.stored_output_reference, ); emit_batch.push((output, accumulator)); } @@ -1362,10 +1429,12 @@ fn precomputed_output_for_group( input_revisions: &BTreeMap>, series_id: u64, catalog_generation: Option<&Arc>, + stored_output_reference: Option, ) -> PrecomputedOutput { let mut output = PrecomputedOutput::new(start_timestamp, end_timestamp, Some(key), policy_fp) .with_population_labels(population_labels_from_group_key(group_key)); - output.series_id = Some(series_id); + output.storage_handle = Some(series_id); + output.stored_output_reference = stored_output_reference; output.catalog_generation = catalog_generation.cloned(); output.input_revision = i64::try_from(start_timestamp) .ok() @@ -1875,7 +1944,7 @@ mod tests { use crate::precompute_engine::config::LateDataPolicy; use crate::precompute_engine::output_sink::CapturingOutputSink; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_physical_operators::accumulators::datasketches_kll_accumulator::DatasketchesKLLAccumulator; use asap_physical_operators::accumulators::keyed_sum_count_accumulator::KeyedSumCountAccumulator; use asap_physical_operators::accumulators::sum_accumulator::SumAccumulator; @@ -1982,16 +2051,16 @@ mod tests { ) } - /// Build a fresh `StreamingConfigHandle` from a map of agg_id + /// Build a fresh `InstalledPrecomputePlanHandle` from a map of agg_id /// → PrecomputeMaterialization. Worker::new takes this handle instead of /// the old `HashMap>`. Tests use this /// helper instead of constructing the handle inline at every /// callsite. fn make_hot_reload( configs: HashMap, - ) -> crate::storage_engines::types::StreamingConfigHandle { - crate::storage_engines::types::StreamingConfigHandle::new( - crate::storage_engines::types::StreamingConfig::new(configs), + ) -> crate::storage_engines::types::InstalledPrecomputePlanHandle { + crate::storage_engines::types::InstalledPrecomputePlanHandle::new( + crate::storage_engines::types::InstalledPrecomputePlan::new(configs), ) } @@ -2759,14 +2828,16 @@ mod tests { } // ----------------------------------------------------------------------- - // Test: worker from streaming_config YAML + // Test: worker from installed_precompute_plan YAML // ----------------------------------------------------------------------- #[test] fn test_worker_rejects_flat_streaming_config_yaml() { let data = serde_yaml::from_str("aggregations: [{aggregationType: Sum, metric: m}]").unwrap(); - assert!(StreamingConfig::from_yaml_data(&data).is_err()); + assert!( + serde_yaml::from_value::(data).is_err() + ); } #[test] @@ -2829,6 +2900,7 @@ mod tests { &BTreeMap::new(), 1, None, + None, ); assert_eq!( output.population_labels, @@ -4281,7 +4353,7 @@ mod tests { mod dag_execution_tests { use super::*; use crate::precompute_engine::output_sink::CapturingOutputSink; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_physical_operators::accumulators::exact_accumulator::ExactAccumulator; use asap_types::query_plan::ExactReadout; @@ -4299,6 +4371,110 @@ mod dag_execution_tests { .unwrap() } + // Real router/engine queues preserve every partition's result as concurrency changes. + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn installed_dag_results_match_with_one_and_four_workers() { + use crate::precompute_engine::{config::PrecomputeEngineConfig, engine::PrecomputeEngine}; + async fn execute(query: &str, workers: usize) -> BTreeMap<(Vec, u64, u64), Vec> { + let physical = plan(query); + assert_eq!( + physical.precompute_plan.materializations.len(), + 1, + "{query}" + ); + let config = physical.precompute_plan.materializations[0].clone(); + let runtime = + InstalledPrecomputePlan::from_precompute_plan(physical.precompute_plan).unwrap(); + let rule = runtime.partitioning.clone(); + let sink = Arc::new(CapturingOutputSink::new()); + let engine = PrecomputeEngine::new( + PrecomputeEngineConfig { + num_workers: workers, + allowed_lateness_ms: 10_000, + wall_clock_idle_grace_period_ms: 0, + wall_clock_max_open_grace_period_ms: 0, + ..Default::default() + }, + InstalledPrecomputePlanHandle::new(runtime), + sink.clone(), + Arc::new(crate::drivers::ingest::series_resolver::SeriesIdResolver::new()), + Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()), + ); + let ingest = engine.ingest_state(); + let diagnostics = engine.diagnostics(); + let running = tokio::spawn(engine.run()); + let mut messages = Vec::new(); + for population in 0..64 { + let label = format!("service-{population}"); + messages.push(WorkerMessage::GroupSamples { + sid: population + 1, + policy_fp: config.policy_fingerprint(), + group_key: Arc::new(GroupKey::new([("service", label.as_str())])), + samples: [ + (1000, 10.0), + (2000, 20.0), + (3000, 3.0), + (4000, 9.0), + (5000, 12.0), + ] + .into_iter() + .map(|(t, value)| { + ( + format!("{}{{service=\"{}\"}}", config.metric, label), + t, + value, + ) + }) + .collect(), + ingest_received_at: std::time::Instant::now(), + }); + } + ingest + .router + .route_group_batch(messages, std::time::Instant::now(), None) + .await + .unwrap(); + ingest.router.drain().await.unwrap(); + if workers > 1 && rule != super::super::partitioning::DagPartitioning::SingleWorker { + assert!( + diagnostics + .worker_group_counts + .iter() + .filter(|n| n.load(Ordering::Relaxed) > 0) + .count() + > 1, + "the test must exercise multiple workers" + ); + } + ingest.router.shutdown().await.unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(5), running) + .await + .unwrap() + .unwrap() + .unwrap(); + let mut records = BTreeMap::new(); + for (output, state) in sink.drain() { + let key = ( + output.key.unwrap().serialize_to_bytes(), + output.start_timestamp, + output.end_timestamp, + ); + assert!( + records.insert(key, state.serialize_to_bytes()).is_none(), + "duplicate output" + ); + } + assert!(!records.is_empty()); + records + } + for query in [ + "sum_over_time(asap_demo_gauge[5s])", + "rate(asap_demo_counter_total[5s])", + ] { + assert_eq!(execute(query, 1).await, execute(query, 4).await, "{query}"); + } + } + // A selected producer must govern updates, persisted family, and query readout. #[test] fn installed_dag_ingestion_persistence_and_readout() { @@ -4334,17 +4510,15 @@ mod dag_execution_tests { .expect("ASAP producer required") .clone(); let fp = config.policy_fingerprint(); - let streaming = StreamingConfig::from_precompute_plan(plan.precompute_plan).unwrap(); - let doc = serde_json::to_value(&streaming).unwrap(); - assert!(doc.get("aggregation_configs").is_none()); - let streaming: StreamingConfig = serde_json::from_value(doc).unwrap(); + let streaming = + InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan).unwrap(); let sink = Arc::new(CapturingOutputSink::new()); let (_tx, rx) = mpsc::channel(8); let mut worker = Worker::new( 0, rx, sink.clone(), - StreamingConfigHandle::new(streaming), + InstalledPrecomputePlanHandle::new(streaming), WorkerRuntimeConfig { max_buffer_per_series: 100, allowed_lateness_ms: 10_000, @@ -4416,13 +4590,15 @@ mod dag_execution_tests { // A flat config and a DAG whose producer no longer matches its binding cannot install. #[test] fn execution_requires_matching_dag_producer() { - assert!(serde_json::from_value::( - serde_json::json!({"aggregation_configs":{}}) - ) - .is_err()); + assert!( + serde_json::from_value::( + serde_json::json!({"aggregation_configs":{}}) + ) + .is_err() + ); let mut plan = plan("rate(asap_demo_counter_total[5s])").precompute_plan; plan.executable_dags.clear(); - assert!(StreamingConfig::from_precompute_plan(plan) + assert!(InstalledPrecomputePlan::from_precompute_plan(plan) .unwrap_err() .to_string() .contains("DAG producer")); @@ -4448,7 +4624,7 @@ mod dag_execution_tests { .unwrap(); installed.document.schema_version = asap_types::executable_plan::MAINTENANCE_DAG_SCHEMA_VERSION; - let error = StreamingConfig::from_precompute_plan(plan) + let error = InstalledPrecomputePlan::from_precompute_plan(plan) .unwrap_err() .to_string(); assert!(error.contains("update"), "{error}"); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index 10860910..cf8c93cb 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -739,7 +739,7 @@ mod tests { .unwrap(); if seed { store.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -1045,8 +1045,8 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); - let hot = crate::storage_engines::types::StreamingConfigHandle::from_arc(Arc::new( - crate::storage_engines::types::StreamingConfig::new(HashMap::from([( + let hot = crate::storage_engines::types::InstalledPrecomputePlanHandle::from_arc(Arc::new( + crate::storage_engines::types::InstalledPrecomputePlan::new(HashMap::from([( cfg.policy_fp_u64(), cfg.clone(), )])), diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index cb90c72b..528c4806 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -1150,7 +1150,7 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu #[cfg(test)] mod sketch_query_tests { - // use crate::storage_engines::types::{CleanupPolicy, StreamingConfig}; + // use crate::storage_engines::types::{CleanupPolicy, InstalledPrecomputePlan}; // use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; // use crate::storage_engines::promsketch_store::PromSketchStore; // use crate::storage_engines::TimestampedBucketsMap; @@ -1216,13 +1216,13 @@ mod sketch_query_tests { // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // ASAPQueryEngine::new( // Arc::new(NoOpStore), // Some(ps), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ) @@ -1292,11 +1292,11 @@ mod sketch_query_tests { // // Engine with promsketch_store = None // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ); @@ -1363,11 +1363,11 @@ mod sketch_query_tests { // fn test_sketch_range_returns_none_without_store() { // let inference_config = // InferenceConfig::new(::promql, CleanupPolicy::NoCleanup); - // let streaming_config = Arc::new(StreamingConfig::default()); + // let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); // let engine = ASAPQueryEngine::new( // Arc::new(NoOpStore), // inference_config, - // streaming_config, + // installed_precompute_plan, // 15, // ::promql, // ); @@ -1467,11 +1467,11 @@ mod aux_pushdown_tests { fn make_engine() -> ASAPQueryEngine { use crate::storage_engines::types::{ - CleanupPolicy, StreamingConfig, StreamingConfigHandle, + CleanupPolicy, InstalledPrecomputePlan, InstalledPrecomputePlanHandle, }; - let sc = Arc::new(StreamingConfig::new(HashMap::new())); - let hr = StreamingConfigHandle::from_arc(sc.clone()); + let sc = Arc::new(InstalledPrecomputePlan::new(HashMap::new())); + let hr = InstalledPrecomputePlanHandle::from_arc(sc.clone()); let _ = sc; ASAPQueryEngine::new(60) } @@ -1599,7 +1599,7 @@ mod asap_tier_classify_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{CleanupPolicy, StreamingConfigHandle}; + use crate::storage_engines::types::{CleanupPolicy, InstalledPrecomputePlanHandle}; use std::collections::{BTreeMap, BTreeSet}; /// `sum by (zone) (http_requests_total)` end-to-end via the @@ -1631,7 +1631,7 @@ mod asap_tier_classify_tests { for (i, zone) in zones.iter().enumerate() { let sid = 9000 + i as u64; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -1722,7 +1722,7 @@ mod asap_tier_classify_tests { // Latest ASAPPlanner sizes an epsilon=0.01 KLL at k=269. let cfg = SketchConfig::Kll { k: 269 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), @@ -1760,7 +1760,7 @@ mod asap_tier_classify_tests { // Latest ASAPPlanner requires p=14 for a 1% HLL error target. let cfg = SketchConfig::Hll { precision: 14 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), @@ -2147,7 +2147,11 @@ mod asap_tier_classify_tests { query: QueryReadout::Quantile { q: 0.99 }, }, ); - let sids = idx.snapshot_instances().iter().map(|m| m.sid).collect(); + let sids = idx + .snapshot_instances() + .iter() + .map(|m| m.storage_handle) + .collect(); let engine = test_plan::engine(idx, config, sids, entry); engine.execute_at(query, now_ms).await.expect( "quantile_over_time over a Hit KLL sid must NOT capability-miss \ @@ -2277,7 +2281,7 @@ mod asap_tier_classify_tests { for (i, (zone, per_window)) in [("z0", 600.0_f64), ("z1", 900.0)].iter().enumerate() { let sid = 14_000 + i as u64; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_requests_total".to_string(), group_by_keys: ["zone".to_string()].into_iter().collect(), capability: Some(Capability::ExactAgg(AggregationType::Sum)), @@ -2373,7 +2377,7 @@ mod asap_tier_classify_tests { // Matches ControlPlaneCostModel's epsilon=0.01 CMS sizing. let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -2488,7 +2492,7 @@ mod outer_agg_integration_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::StreamingConfigHandle; + use crate::storage_engines::types::InstalledPrecomputePlanHandle; use asap_sketchlib::DdSketch; use asap_sketchlib::MessagePackCodec; use std::collections::{BTreeMap, BTreeSet}; @@ -2509,7 +2513,7 @@ mod outer_agg_integration_tests { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -2616,7 +2620,7 @@ mod range_stitch_tests { AccuracyBound, Capability, SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchStore, SummarySeriesMetadata, }; - use crate::storage_engines::types::{KeyByLabelValues, StreamingConfigHandle}; + use crate::storage_engines::types::{InstalledPrecomputePlanHandle, KeyByLabelValues}; use async_trait::async_trait; use std::collections::{BTreeMap, BTreeSet}; @@ -2650,7 +2654,7 @@ mod range_stitch_tests { fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 5, cols: 512 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -2775,7 +2779,7 @@ mod range_stitch_tests { .unwrap(); active.envelope.expiry_unix_ms = None; let active = crate::storage_engines::types::ActivePhysicalPlanHandle::new(active); - let hot = StreamingConfigHandle::from_active_physical_plan(active.clone()); + let hot = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let engine = ASAPQueryEngine::new(15).with_active_physical_plan(active); let error = engine .execute_metricsql_at(&identity, 1_000) diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index e82f54d1..c99856af 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -929,7 +929,7 @@ mod tests { const MATERIALIZATION: asap_types::PolicyFingerprint = asap_types::PolicyFingerprint(9001); let store = crate::storage_engines::sketch_db::index::SketchStore::new(); store.register(SummarySeriesMetadata { - sid: 41, + storage_handle: 41, metric_name: "http_requests_total".into(), group_by_keys: std::collections::BTreeSet::from(["job".into()]), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 4485cf07..ee6919ca 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -150,7 +150,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(901); idx.register(SummarySeriesMetadata { - sid: 9, + storage_handle: 9, metric_name: "bytes".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 658f9e9c..47f608e9 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -757,7 +757,7 @@ mod tests { let mut group_by_keys = std::collections::BTreeSet::new(); group_by_keys.insert("service".to_string()); idx.register(SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "unique_users".to_string(), group_by_keys, capability: Some(Capability::CardinalityApprox), @@ -799,7 +799,7 @@ mod tests { relative_accuracy: 0.01, }; idx.register(SummarySeriesMetadata { - sid: 1, + storage_handle: 1, metric_name: "latency_ms".to_string(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), @@ -1029,7 +1029,7 @@ mod tests { let idx = SketchStore::new(); idx.register( crate::storage_engines::sketch_db::index::SummarySeriesMetadata { - sid: 1, + storage_handle: 1, metric_name: "bytes_total".to_string(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1124,7 +1124,7 @@ mod tests { } let idx = SketchStore::new(); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1194,7 +1194,7 @@ mod tests { let policy = config.policy_fingerprint(); let idx = SketchStore::new(); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1268,7 +1268,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), @@ -1371,7 +1371,7 @@ mod tests { let idx = SketchStore::new(); let policy = asap_types::PolicyFingerprint(777); idx.register(SummarySeriesMetadata { - sid: 7, + storage_handle: 7, metric_name: "requests_total".into(), group_by_keys: std::collections::BTreeSet::new(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Rate)), diff --git a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs index 851e1c1c..e145db16 100644 --- a/data_plane/src/query_engines/asap_query_engine/summary_executor.rs +++ b/data_plane/src/query_engines/asap_query_engine/summary_executor.rs @@ -505,7 +505,7 @@ impl QueryExecutionContext<'_> { }; let mut sids = self .index - .series_ids_for_policy(binding.materialization.fingerprint()); + .storage_handles_for_output(binding.stored_output_reference); sids.sort_unstable(); sids.dedup(); let mut matched_metadata = 0usize; @@ -1637,7 +1637,7 @@ mod tests { fn kll_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() @@ -1660,7 +1660,7 @@ mod tests { fn hll_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::Hll { precision: 10 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::CardinalityApprox), @@ -1706,7 +1706,7 @@ mod tests { fn cms_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyEstimate(Some(SketchAlgorithm::Cms))), @@ -1779,7 +1779,7 @@ mod tests { fn cms_with_heap_meta(sid: u64, metric: &str) -> SummarySeriesMetadata { let cfg = SketchConfig::CountMin { rows: 4, cols: 256 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: BTreeSet::new(), capability: Some(Capability::FrequencyTopk(Some( @@ -1849,7 +1849,7 @@ mod tests { fn sum_exact_agg_meta(sid: u64, metric: &str, group_by: &[&str]) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys: group_by .iter() diff --git a/data_plane/src/query_engines/routing/backend_storage_routing.rs b/data_plane/src/query_engines/routing/backend_storage_routing.rs index bbae8d30..e3aa2b39 100644 --- a/data_plane/src/query_engines/routing/backend_storage_routing.rs +++ b/data_plane/src/query_engines/routing/backend_storage_routing.rs @@ -804,7 +804,7 @@ pub fn routing_table_hash(table: &BackendStorageRouting) -> String { // --------------------------------------------------------------------------- /// Per-tenant atomic-swap wrapper around `BackendStorageRouting`, -/// mirroring [`crate::storage_engines::types::StreamingConfigHandle`]. Lets the +/// mirroring [`crate::storage_engines::types::InstalledPrecomputePlanHandle`]. Lets the /// `POST /api/v1/storage_routing` HTTP handler swap one tenant's table /// at runtime without restarting the backend or touching any other /// tenant's table. Cloneable; clones share the underlying `ArcSwap` so diff --git a/data_plane/src/query_engines/routing/capability_matching.rs b/data_plane/src/query_engines/routing/capability_matching.rs index 379f3c3f..ca588a65 100644 --- a/data_plane/src/query_engines/routing/capability_matching.rs +++ b/data_plane/src/query_engines/routing/capability_matching.rs @@ -4,7 +4,7 @@ //! //! Split out of `asap_types`'s former `capability_matching` module (see //! `scratchpad/artifacts/enum-unification-plan.md`). `StorageBackend` and -//! the `StreamingConfig` wire format it's a field of both turned out to +//! the `InstalledPrecomputePlan` wire format it's a field of both turned out to //! have zero real `control_plane` dependency either — see //! [`crate::storage_engines::types::storage_backend`]'s module doc — so //! both moved into this crate; only the routing *decision* below was ever diff --git a/data_plane/src/query_engines/routing/mod.rs b/data_plane/src/query_engines/routing/mod.rs index 3bf7ad70..caa5f554 100644 --- a/data_plane/src/query_engines/routing/mod.rs +++ b/data_plane/src/query_engines/routing/mod.rs @@ -18,7 +18,7 @@ //! * [`capability_matching`] — the storage-backend routing policy itself //! (`AccuracyTarget`, `compatible_storage_backends`). Split out of //! `asap_types`'s former `capability_matching` module. `StorageBackend` -//! itself later moved into this crate too, alongside `StreamingConfig` +//! itself later moved into this crate too, alongside `InstalledPrecomputePlan` //! (see `crate::storage_engines::types::storage_backend`'s module doc) — //! `control_plane` turned out to have zero real dependency on either. //! diff --git a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs index ce8e476d..9dacddcf 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/mod.rs @@ -491,7 +491,7 @@ impl BackfillRegistry { /// world there is no `SchemaRegistry::get(agg_id)` to look /// it up from, and the caller (typically the HTTP handler /// or control plane) already has the wall-clock snapshot in - /// scope from its `StreamingConfig` reconcile event. + /// scope from its `InstalledPrecomputePlan` reconcile event. /// * **Within data retention** (if `data_retention_ms` is /// provided): `time_range.0 >= now - data_retention_ms`. /// Method B from the design discussion — fail fast instead diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index 30a7f838..c680649c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -21,7 +21,9 @@ use tracing::debug; use crate::drivers::ingest::population_attrs_fingerprint; use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::precompute_engine::worker::parse_labels_from_series_key; -use crate::storage_engines::types::{AggregateCore, KeyByLabelValues, StreamingConfigHandle}; +use crate::storage_engines::types::{ + AggregateCore, InstalledPrecomputePlanHandle, KeyByLabelValues, +}; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; @@ -87,22 +89,26 @@ fn resolve_backfill_bucket_sid( .map(|name| (name.as_str(), *labels.get(name.as_str()).unwrap_or(&""))) .collect(); let attrs_fp = population_attrs_fingerprint(config.population_key_encoding, &grouping_pairs)?; - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(config); - resolver.resolve_with_reactivation(&config.metric, &attrs_fp, &agg_kind_canonical, |sid| { - store.map_or(Ok(None), |store| { - store.validate_routed_catalog_generation(captured_generation)?; - let activation = - store.authorize_series_reactivation(sid, config.policy_fingerprint().into())?; - if activation - .as_deref() - .is_some_and(|generation| Some(generation) != captured_generation) - { - return Err("stale backfill job cannot reactivate series".into()); - } - Ok(activation) - }) - }) + if let Some(store) = store { + return store.resolve_output_storage_handle( + resolver, + config.policy_fingerprint().into(), + &attrs_fp, + captured_generation, + ); + } + #[cfg(test)] + { + let store = crate::storage_engines::sketch_db::index::SketchStore::new(); + store.resolve_output_storage_handle( + resolver, + config.policy_fingerprint().into(), + &attrs_fp, + captured_generation, + ) + } + #[cfg(not(test))] + Err("backfill output requires an installed SummaryStore".into()) } /// Fallback bucket id for the resolver-less code path (registry-only @@ -124,10 +130,10 @@ fn fallback_bucket_id(group_key: &str) -> u64 { /// without round-tripping through the worker. pub struct BackfillWindowProcessor { /// Live config source. The processor snapshots the latest - /// `StreamingConfig` at each window to find the + /// `InstalledPrecomputePlan` at each window to find the /// `PrecomputeMaterialization` for `agg_id`. The snapshot is cheap /// (Arc refcount bump) so we don't optimise further. - config: StreamingConfigHandle, + config: InstalledPrecomputePlanHandle, /// Destination for rebuilt windows. Tests may omit it to record registry /// provenance without storing payloads. summary_store: Option>, @@ -149,7 +155,7 @@ pub struct BackfillWindowProcessor { impl BackfillWindowProcessor { pub fn new( - config: StreamingConfigHandle, + config: InstalledPrecomputePlanHandle, registry: Arc, job_id: u64, ) -> Self { @@ -208,7 +214,7 @@ impl WindowProcessor for BackfillWindowProcessor { let config = snapshot .get_aggregation_config(agg_id) .cloned() - .ok_or_else(|| format!("agg_id {agg_id} not in current StreamingConfig"))?; + .ok_or_else(|| format!("agg_id {agg_id} not in current InstalledPrecomputePlan"))?; let program = snapshot.raw_programs.get(&agg_id).cloned(); #[cfg(not(test))] if program.is_none() { @@ -312,7 +318,9 @@ impl WindowProcessor for BackfillWindowProcessor { self.job_id, PolicyFingerprint::from_config(&config), ); - output.series_id = Some(sid); + output.storage_handle = Some(sid); + output.stored_output_reference = + snapshot.stored_output_reference(config.policy_fingerprint().into()); output.catalog_generation = self.catalog_generation.clone(); batch.push((sid, output, accumulator)); } @@ -329,13 +337,39 @@ impl WindowProcessor for BackfillWindowProcessor { // to the same sid (the resolver is idempotent), // but the round-trip is redundant now that we hold // the value. - for (sid, output, accumulator) in &batch { - idx.ingest_precompute_with_series_id( - *sid, - &config, - output, - accumulator.as_ref(), - ) + for (_handle, output, accumulator) in &batch { + let reference = output + .stored_output_reference + .ok_or("backfill has no selected stored output")?; + let (plan_id, plan_version) = self + .catalog_generation + .as_deref() + .map(|generation| (generation.plan_id, generation.plan_version)) + .unwrap_or((0, 0)); + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + output: reference, + population: config + .grouping_labels + .iter() + .cloned() + .zip(output.key.clone().unwrap_or_default().labels) + .collect(), + window: asap_types::sds::HalfOpenTimeRange { + start_ms: i64::try_from(output.start_timestamp)?, + end_ms: i64::try_from(output.end_timestamp)?, + }, + }; + idx.publish_unadmitted_summary_update(|writer| { + writer.write_stored_summary( + &address, + _resolver, + &config, + output, + accumulator.as_ref(), + ) + }) .ok_or("backfill summary state publication rejected")?; } } @@ -373,7 +407,7 @@ mod tests { }; use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::BackfillSource; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::enums::WindowKind; use asap_types::AggregationType; use asap_types::KeyByLabelNames; @@ -406,10 +440,10 @@ mod tests { ) } - fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { + fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { let mut map = std::collections::HashMap::new(); map.insert(config.policy_fp_u64(), config); - Arc::new(StreamingConfig::new(map)) + Arc::new(InstalledPrecomputePlan::new(map)) } #[tokio::test] @@ -417,7 +451,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -461,7 +495,7 @@ mod tests { async fn unknown_agg_id_fails_cleanly() { let cfg = sum_config(1, "m", vec![]); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( 999, @@ -470,12 +504,14 @@ mod tests { 1, ); let processor = BackfillWindowProcessor::new(hot, registry.clone(), job_id); - // agg_id=999 isn't in the StreamingConfig. + // agg_id=999 isn't in the InstalledPrecomputePlan. let err = processor .process_window(999, (0, 10), vec![]) .await .expect_err("unknown agg should fail"); - assert!(err.to_string().contains("not in current StreamingConfig")); + assert!(err + .to_string() + .contains("not in current InstalledPrecomputePlan")); assert!(registry.windows_written_by(job_id).is_empty()); } @@ -484,7 +520,7 @@ mod tests { let cfg = sum_config(1, "m", vec![]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -505,7 +541,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let job_id = registry.create( fp, @@ -797,7 +833,7 @@ mod tests { let cfg = sum_config(1, "latency", vec!["svc"]); let fp = cfg.policy_fp_u64(); let streaming = streaming_config_with(cfg.clone()); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let summary_store = Arc::new(SketchStore::new()); let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( @@ -863,10 +899,23 @@ mod tests { // `SeriesIdResolver::lookup` would return for the same // `(metric, grouping-values, agg_kind)` tuple — i.e. live // ingest and backfill share one sid namespace. - let sid_a = - resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"a\"}", None, None).unwrap(); - let sid_b = - resolve_backfill_bucket_sid(&resolver, &cfg, "latency{svc=\"b\"}", None, None).unwrap(); + let generation = summary_store.active_catalog_generation().unwrap(); + let sid_a = resolve_backfill_bucket_sid( + &resolver, + &cfg, + "latency{svc=\"a\"}", + Some(&summary_store), + Some(&generation), + ) + .unwrap(); + let sid_b = resolve_backfill_bucket_sid( + &resolver, + &cfg, + "latency{svc=\"b\"}", + Some(&summary_store), + Some(&generation), + ) + .unwrap(); assert_ne!(sid_a, sid_b, "distinct svc values mint distinct sids"); assert_eq!(summary_store.classify(sid_a), SeriesLookup::Hit); assert_eq!(summary_store.classify(sid_b), SeriesLookup::Hit); @@ -905,11 +954,10 @@ mod tests { ); let attrs = population_attrs_fingerprint(encoding, &[("svc", "a"), ("zone", "z0")]).unwrap(); - let expected_sid = resolver.resolve( - &cfg.metric, - &attrs, - &crate::storage_engines::sketch_db::data::materialization_kind_for_config(&cfg), - ); + let store = crate::storage_engines::sketch_db::index::SketchStore::new(); + let expected_sid = store + .resolve_output_storage_handle(&resolver, cfg.policy_fingerprint().into(), &attrs, None) + .unwrap(); if encoding.is_legacy() { assert_eq!(backfill_result.unwrap(), expected_sid); } else { @@ -935,7 +983,16 @@ mod tests { asap_physical_operators::accumulators::sum_accumulator::SumAccumulator::with_sum(1.0); let live_sid = store .ingest_precompute_for_agg_config( - |metric, attrs, kind| resolver.resolve(metric, attrs, kind), + |_metric, attrs, _kind| { + store + .resolve_output_storage_handle( + &resolver, + cfg.policy_fingerprint().into(), + attrs, + None, + ) + .ok() + }, &cfg, &output, &acc, diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index 86d64fac..35c9b135 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -58,7 +58,7 @@ use crate::storage_engines::sketch_db::backfill::worker::BackfillWorker; use crate::storage_engines::sketch_db::backfill::{ BackfillRegistry, BackfillSource, BackfillStatus, }; -use crate::storage_engines::types::StreamingConfigHandle; +use crate::storage_engines::types::InstalledPrecomputePlanHandle; /// Given a `BackfillSource`, return a reader that can read raw /// samples from it. Used by the service to pick a concrete reader @@ -103,7 +103,7 @@ pub struct BackfillService { /// Shared sid mint authority — wired alongside `sketch_index` so /// backfilled precompute sids share the namespace with live ingest. series_resolver: Option>, - config_source: StreamingConfigHandle, + config_source: InstalledPrecomputePlanHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, } @@ -111,7 +111,7 @@ pub struct BackfillService { impl BackfillService { pub fn new( registry: Arc, - config_source: StreamingConfigHandle, + config_source: InstalledPrecomputePlanHandle, reader_factory: ReaderFactory, service_config: BackfillServiceConfig, ) -> Self { @@ -341,7 +341,7 @@ mod tests { use crate::storage_engines::sketch_db::backfill::raw_sample_reader::{ MockRawSampleReader, RawSample, }; - use crate::storage_engines::types::StreamingConfig; + use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; use asap_types::AggregationType; @@ -369,10 +369,10 @@ mod tests { ) } - fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { + fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { let mut m = std::collections::HashMap::new(); m.insert(cfg.policy_fp_u64(), cfg); - Arc::new(StreamingConfig::new(m)) + Arc::new(InstalledPrecomputePlan::new(m)) } async fn wait_for_status( @@ -405,7 +405,7 @@ mod tests { let mut cfg = sum_config(1, "latency"); cfg.table_name = Some("expected_table".into()); let agg_fp = cfg.policy_fp_u64(); - let hot = StreamingConfigHandle::from_arc(streaming_with(cfg)); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming_with(cfg)); let registry = Arc::new(BackfillRegistry::new()); let calls = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let called = calls.clone(); @@ -441,7 +441,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory returns a fresh mock reader per call — seeded with a @@ -488,7 +488,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( @@ -522,7 +522,7 @@ mod tests { let cfg = sum_config(1, "latency"); let agg_fp = cfg.policy_fp_u64(); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); // Factory records the order in which it's invoked. @@ -584,7 +584,7 @@ mod tests { async fn service_shutdown_stops_the_loop() { let cfg = sum_config(1, "m"); let streaming = streaming_with(cfg); - let hot = StreamingConfigHandle::from_arc(streaming.clone()); + let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let service = BackfillService::new( diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index a4410abe..d9214899 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -7,7 +7,8 @@ use super::*; use crate::storage_engines::types::AggregateCore; pub(crate) struct FrozenExactWindows { - pub(crate) sid: u64, + pub(crate) stored_output_reference: asap_types::sds::StoredOutputReference, + pub(crate) storage_handle: u64, pub(crate) definition: SummaryDefinitionId, pub(crate) generation: Arc, pub(crate) group: BTreeMap, @@ -122,12 +123,20 @@ impl SketchStore { let writer = metadata .as_ref() .ok_or("immutable population proof requires durable metadata")?; + // Retained populations remain a completeness fence even when their + // generation is no longer readable. Losing their binding cannot certify + // that a newly observed subset is the complete source population. Ok(writer .load_strict() .map_err(|error| error.to_string())? .into_iter() - .filter(|record| !record.removed && record.summary_definition_id == Some(definition)) - .map(|record| record.sid) + .filter(|record| { + !record.removed + && record.summary_definition_id == Some(definition) + && record.stored_output_reference + == self.descriptors.stored_output_reference(definition) + }) + .map(|record| record.storage_handle) .collect()) } @@ -183,7 +192,12 @@ impl SketchStore { population_ids.extend( instances .iter() - .filter(|(_, binding)| binding.metadata.policy_fp == definition.fingerprint()) + .filter(|(_, binding)| { + binding.metadata.policy_fp == definition.fingerprint() + && binding.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && binding.catalog_generation.as_deref() == Some(generation) + }) .map(|(sid, _)| *sid), ); if !require_complete_population && population_ids.len() > 1 { @@ -216,6 +230,8 @@ impl SketchStore { let mut coordinates = BTreeMap::new(); for (sid, binding) in instances.iter() { if binding.metadata.policy_fp != definition.fingerprint() + || binding.stored_output_reference + != self.descriptors.stored_output_reference(definition) || binding.catalog_generation.as_deref() != Some(generation) || !binding.metadata.is_writable() { @@ -285,13 +301,15 @@ impl SketchStore { .admission .read() .map_err(|_| "admission registry poisoned")?; - let (keys, singleton_population_complete) = { + let (keys, singleton_population_complete, stored_output_reference) = { let bindings = self .instances .read() .map_err(|_| "instance registry poisoned")?; let binding = bindings.get(&sid).ok_or("immutable input SID is absent")?; if binding.metadata.policy_fp != definition.fingerprint() + || binding.stored_output_reference + != self.descriptors.stored_output_reference(definition) || binding.catalog_generation.as_deref() != Some(generation.as_ref()) || binding.metadata.status() == AggStatus::Expired { @@ -303,6 +321,9 @@ impl SketchStore { .iter() .filter(|(_, candidate)| { candidate.metadata.policy_fp == definition.fingerprint() + && candidate.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && candidate.catalog_generation.as_deref() == Some(generation.as_ref()) }) .map(|(sid, _)| *sid), ); @@ -312,7 +333,13 @@ impl SketchStore { asap_types::sds::DataSourceIdentity::Derived { .. } ) && population_ids == BTreeSet::from([sid]); - (binding.metadata.group_by_keys.clone(), singleton) + ( + binding.metadata.group_by_keys.clone(), + singleton, + binding + .stored_output_reference + .ok_or("immutable input has no stored-output binding")?, + ) }; drop(admission); if group.keys().cloned().collect::>() != keys { @@ -371,7 +398,8 @@ impl SketchStore { } self.validate_routed_catalog_generation(Some(generation.as_ref()))?; Ok(FrozenExactWindows { - sid, + stored_output_reference, + storage_handle: sid, definition, generation: Arc::clone(generation), group: group.clone(), @@ -406,16 +434,17 @@ impl SketchStore { let mut populations = BTreeSet::new(); for source in sources { if &source.generation != generation - || !populations.insert((source.definition, source.sid, &source.group)) + || !populations.insert((source.definition, source.storage_handle, &source.group)) { return Err( "immutable publication input cohort has mixed generations or duplicates".into(), ); } let binding = instances - .get(&source.sid) + .get(&source.storage_handle) .ok_or("immutable source was removed")?; - if binding.metadata.policy_fp != source.definition.fingerprint() + if binding.stored_output_reference != Some(source.stored_output_reference) + || binding.metadata.policy_fp != source.definition.fingerprint() || binding.catalog_generation.as_deref() != Some(generation.as_ref()) || !binding.metadata.is_writable() { @@ -440,14 +469,23 @@ impl SketchStore { supplied .entry(input.definition) .or_default() - .insert(input.sid); + .insert(input.storage_handle); } for (definition, supplied_sids) in supplied { let mut current = self.durable_maintenance_population_ids(definition)?; current.extend( instances .iter() - .filter(|(_, binding)| binding.metadata.policy_fp == definition.fingerprint()) + .filter(|(_, binding)| { + binding.metadata.policy_fp == definition.fingerprint() + && binding.stored_output_reference + == self.descriptors.stored_output_reference(definition) + && binding.catalog_generation.as_deref() + == cohort + .inputs() + .first() + .map(|input| input.generation.as_ref()) + }) .map(|(sid, _)| *sid), ); if current != supplied_sids { @@ -1042,7 +1080,7 @@ mod tests { .load_strict() .unwrap() .iter() - .any(|record| record.sid == 905)); + .any(|record| record.storage_handle == 905)); store.force_expire(901).unwrap(); assert!(store .publish_frozen_maintenance_output(903, target, &output, &sum, &cohort, [42; 32]) @@ -1061,7 +1099,7 @@ mod tests { .load_strict() .unwrap() .iter() - .any(|record| record.sid == 903)); + .any(|record| record.storage_handle == 903)); let mut next = catalog; next.plan_version += 1; store.install_summary_catalog(Arc::new(next)).unwrap(); @@ -1377,7 +1415,7 @@ mod tests { .load_strict() .unwrap() .into_iter() - .map(|record| (record.sid, serde_json::to_value(record).unwrap())) + .map(|record| (record.storage_handle, serde_json::to_value(record).unwrap())) .collect::>() }; let records_before = durable_records(); @@ -1468,7 +1506,10 @@ mod tests { .unwrap(); assert!(!store.completed_windows.read().unwrap().contains_key(&601)); let input = FrozenExactWindows { - sid: 600, + stored_output_reference: asap_types::sds::StoredOutputReference::for_definition( + source_id, + ), + storage_handle: 600, definition: source_id, generation, group: BTreeMap::new(), diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index 5dce3059..132c6f4b 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -19,6 +19,7 @@ //! See design doc §4.6 ("OTLP metadata model + backend store layout") at //! `docs/design_docs/series-identity.md`. +use asap_types::sds::StoredOutputReference; use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; use std::sync::{Arc, RwLock}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -192,14 +193,14 @@ fn build_attrs_fp_and_label_map( Ok((attrs_fp, label_values_map)) } -/// Metadata for one logical sketch instance, keyed by sid. Ingest registers it -/// on first sight; later writes append to the associated per-sid storage. +/// Physical metadata for one stored-output population. Its local handle locates +/// rows; the owning `SdsBinding` retains the selected output and plan generation. /// /// Lifecycle status is derived from retirement and expiry timestamps plus the /// wall clock. Only active instances accept writes. #[derive(Debug, Clone)] pub struct SummarySeriesMetadata { - pub sid: u64, + pub storage_handle: u64, pub metric_name: String, /// The group-by KEY set — `dp.attributes.keys()` after the agent's /// `AggregateBy` rollup folded other labels into the sketch state. @@ -231,7 +232,7 @@ pub struct SummarySeriesMetadata { /// `AggSchema::expires_at_ms`. pub expires_at_ms: Option, /// Content-addressed back-reference to the policy that minted this - /// sid. Together with [`SketchStore::policy_to_series_ids`] this gives + /// sid. Together with [`SketchStore::output_to_storage_handles`] this gives /// the query path a direct `policy_fp → [sid]` index without /// walking the metadata map. `PolicyFingerprint::UNSET` is reserved /// for the legacy registration path that doesn't carry a source @@ -621,9 +622,6 @@ pub struct SketchStore { instances: RwLock>, /// Interns immutable SDS descriptors across all Series IDs and panes. descriptors: SummaryDescriptorRegistry, - /// Previous payload generations admitted by an explicit definition - /// compatibility check during plan installation. - compatible_source_generations: RwLock>, /// sid → item_label (the data-point attribute NAME, e.g. "service" /// or "endpoint") for CountMin/CountSketch sids registered in /// per-item mode. Its presence is what makes a CMS sid answerable by @@ -655,32 +653,32 @@ pub struct SketchStore { /// /// ## Cross-index atomicity invariant (P2-1) /// - /// `instances`, `policy_to_series_ids`, and `metric_to_series_ids` form ONE + /// `instances`, `output_to_storage_handles`, and `metric_to_series_ids` form ONE /// logical index whose three maps must agree: every sid present in /// `instances` must also be present in `metric_to_series_ids` (keyed by /// its metric_name) and — when its `policy_fp` is non-UNSET — in - /// `policy_to_series_ids`. A concurrent reader must never observe a sid in + /// `output_to_storage_handles`. A concurrent reader must never observe a sid in /// `instances` that is missing from the secondary indexes (or vice /// versa). To preserve this, every writer ([`Self::register`], /// [`Self::remove_instance`]) acquires ALL THREE write guards - /// together in the fixed order `instances → policy_to_series_ids → + /// together in the fixed order `instances → output_to_storage_handles → /// metric_to_series_ids` BEFORE mutating any of them, so the update is /// atomic with respect to any reader that takes the `instances` lock. /// The fixed acquisition order is also the deadlock-avoidance order: /// no code path takes these locks in a different order. - policy_to_series_ids: RwLock>>, + output_to_storage_handles: RwLock>>, /// Secondary index: `metric_name → {sids}` (P2-2). Lets /// [`Self::instances_matching`] do a keyed lookup of the sids for a /// metric instead of an O(N) full scan of `instances`. Maintained in /// lock-step with `instances` under the same write-lock domain (see - /// the atomicity invariant on `policy_to_series_ids`). Holds every + /// the atomicity invariant on `output_to_storage_handles`). Holds every /// registered sid (UNSET-policy sids included), since the query path /// keys candidate selection on metric name, not policy. metric_to_series_ids: RwLock>>, - /// Pointer (as `usize`) of the `Arc` this store + /// Pointer (as `usize`) of the `Arc` this store /// last reconciled against. `reconcile_from_streaming_config` runs /// on every ingest batch, but the config is a lock-free - /// `Arc>` that only changes its `Arc` + /// `Arc>` that only changes its `Arc` /// identity on a control-plane swap (rare). Gating the full /// catalog scan on a cheap pointer compare against this field lets /// the steady-state ingest path skip reconcile entirely. @@ -697,7 +695,7 @@ pub struct SketchStore { /// has sealed epochs to persist) with retention-drop disabled (the /// flush-then-evict loop is the memory bound). persistence_read: RwLock>>, - persistence_metadata: RwLock>>, + persistence_metadata: RwLock>>, immutable_publisher: RwLock>, removed_sids: RwLock< BTreeMap< @@ -747,26 +745,72 @@ pub enum SeriesLookup { pub(crate) struct SummaryPublicationWriter<'a>(&'a SketchStore); impl SummaryPublicationWriter<'_> { - pub(crate) fn ingest_precompute_with_series_id( + pub(crate) fn write_stored_summary( &self, - sid: u64, + address: &asap_types::sds::StoredSummaryKey, + resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { + address.validate().ok()?; + if self + .0 + .descriptors + .stored_output_reference(address.output.definition_id) + != Some(address.output) + || address.output.definition_id.fingerprint() != output.policy_fp + || output + .stored_output_reference + .is_some_and(|reference| reference != address.output) + { + return None; + } + let (population_key, population) = build_attrs_fp_and_label_map(config, output).ok()?; + if population != address.population + || (output.start_timestamp, output.end_timestamp) + != ( + u64::try_from(address.window.start_ms).ok()?, + u64::try_from(address.window.end_ms).ok()?, + ) + { + return None; + } + let generation = output.catalog_generation.as_deref(); + if generation + .is_some_and(|g| (g.plan_id, g.plan_version) != (address.plan_id, address.plan_version)) + { + return None; + } + let handle = self + .0 + .resolve_output_storage_handle( + resolver, + address.output.definition_id, + &population_key, + generation, + ) + .ok()?; + if output + .storage_handle + .is_some_and(|captured| captured != handle) + { + return None; + } self.0 - .ingest_precompute_with_admission(sid, config, output, state) + .ingest_precompute_with_admission(handle, config, output, state) } - pub(crate) fn ingest_precompute_for_agg_config>>( + #[cfg(test)] + pub(crate) fn ingest_precompute_with_series_id( &self, - mint: impl FnOnce(&str, &str, &str) -> R, + sid: u64, config: &asap_types::PrecomputeMaterialization, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { self.0 - .ingest_precompute_config_with_admission(mint, config, output, state) + .ingest_precompute_with_admission(sid, config, output, state) } #[cfg(test)] @@ -788,6 +832,34 @@ impl SketchStore { Self::default() } + /// Resolve a local row handle from the installed DAG output and population. + /// The durable identity includes the plan version and exact output reference. + pub fn resolve_output_storage_handle( + &self, + resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, + definition: SummaryDefinitionId, + population_key: &str, + generation: Option<&CatalogGeneration>, + ) -> Result { + self.validate_routed_catalog_generation(generation)?; + let output = self + .descriptors + .stored_output_reference(definition) + .ok_or("definition has no installed stored output")?; + let (plan_id, plan_version) = match generation { + Some(generation) => (generation.plan_id, generation.plan_version), + #[cfg(test)] + None => (0, 0), + #[cfg(not(test))] + None => return Err("stored output requires an installed plan generation".into()), + }; + let identity = + serde_json::to_string(&(plan_id, plan_version, output)).map_err(|e| e.to_string())?; + resolver + .try_resolve("stored-output", population_key, &identity) + .map_err(|e| e.to_string()) + } + /// Classify a sid for query routing. See `SeriesLookup` for semantics. pub fn classify(&self, sid: u64) -> SeriesLookup { let known = self.instances.read().unwrap().contains_key(&sid); @@ -815,11 +887,11 @@ impl SketchStore { /// ## Atomicity (P2-1) /// /// All three index write guards are acquired together, in the fixed - /// order `instances → policy_to_series_ids → metric_to_series_ids`, BEFORE any + /// order `instances → output_to_storage_handles → metric_to_series_ids`, BEFORE any /// map is mutated. This makes the three updates atomic with respect /// to a concurrent reader that takes the `instances` lock: such a /// reader can never see the sid in `instances` while it is still - /// absent from `policy_to_series_ids` / `metric_to_series_ids` (the pre-fix race + /// absent from `output_to_storage_handles` / `metric_to_series_ids` (the pre-fix race /// where the two indexes were written under separate sequential /// locks). See the index-field doc comments for the full invariant. pub fn register(&self, meta: SummarySeriesMetadata) { @@ -832,8 +904,7 @@ impl SketchStore { meta: SummarySeriesMetadata, instances: &mut HashMap, ) -> bool { - let sid = meta.sid; - let policy_fp = meta.policy_fp; + let sid = meta.storage_handle; // A non-legacy materialization must resolve through the installed // authoritative catalog. Unknown identities fail closed and never // become queryable SummaryStore entries. @@ -848,17 +919,17 @@ impl SketchStore { .data_descriptor .time_series_metric() .map(str::to_owned); - // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. + // Fixed lock order: instances → output_to_storage_handles → metric_to_series_ids. if self.removed_sids.read().unwrap().contains_key(&sid) { tracing::warn!(sid, "rejecting reuse of a removed summary instance ID"); return false; } - let mut policy_idx = self.policy_to_series_ids.write().unwrap(); + let mut policy_idx = self.output_to_storage_handles.write().unwrap(); let mut metric_idx = self.metric_to_series_ids.write().unwrap(); - instances.insert(sid, instance); - if !policy_fp.is_unset() { - policy_idx.entry(policy_fp).or_default().insert(sid); + if let Some(output) = instance.stored_output_reference { + policy_idx.entry(output).or_default().insert(sid); } + instances.insert(sid, instance); if let Some(metric_name) = metric_name { metric_idx.entry(metric_name).or_default().insert(sid); } @@ -867,10 +938,59 @@ impl SketchStore { /// Install one authoritative catalog snapshot for future registrations. /// Existing Series IDs retain their generation's descriptor Arcs while draining. + #[cfg(test)] pub fn install_summary_catalog( &self, catalog: Arc, ) -> Result<(), String> { + let outputs = catalog + .definitions + .keys() + .map(|id| (*id, StoredOutputReference::for_definition(*id))) + .collect(); + self.install_catalog_outputs(catalog, outputs) + } + + /// Install the exact selected writer references together with their catalog. + pub fn install_precompute_plan( + &self, + catalog: Arc, + plan: &asap_types::precompute_plan::PrecomputePlan, + ) -> Result<(), String> { + plan.validate_against_catalog(&catalog) + .map_err(|e| e.to_string())?; + let outputs = plan + .schemas + .iter() + .map(|schema| (schema.materialization, schema.stored_output_reference)) + .collect::>(); + if outputs.len() != plan.schemas.len() { + return Err("stored output bindings must be unique per definition".into()); + } + self.install_catalog_outputs(catalog, outputs) + } + + fn install_catalog_outputs( + &self, + catalog: Arc, + outputs: BTreeMap, + ) -> Result<(), String> { + for (definition, output) in &outputs { + output.validate().map_err(|e| e.to_string())?; + if output.definition_id != *definition || !catalog.definitions.contains_key(definition) + { + return Err("stored output does not match the installed catalog".into()); + } + } + if outputs + .values() + .map(|output| output.stored_output_id) + .collect::>() + .len() + != outputs.len() + { + return Err("a stored output ID cannot name different definitions".into()); + } let reference = catalog.reference().map_err(|error| error.to_string())?; let mut inventory = self.admission.write().unwrap(); let generation = CatalogGeneration { @@ -879,26 +999,6 @@ impl SketchStore { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }; - let previous = self.descriptors.authoritative_snapshot(); - let previous_sources = self.compatible_source_generations.read().unwrap().clone(); - let mut compatible_sources = BTreeMap::new(); - if let Some((old_catalog, old_generation)) = &previous { - if old_catalog.plan_id == catalog.plan_id - && old_catalog.plan_version < catalog.plan_version - { - for (id, definition) in &catalog.definitions { - if old_catalog.definitions.get(id) == Some(definition) { - compatible_sources.insert( - *id, - previous_sources - .get(id) - .cloned() - .unwrap_or_else(|| (**old_generation).clone()), - ); - } - } - } - } let closed = self .persistence_metadata .read() @@ -908,11 +1008,8 @@ impl SketchStore { .transpose() .map_err(|error| error.to_string())? .flatten(); - // Publish the compatibility decision first so a reader that observes - // the successor catalog can also resolve its admitted source payload. - *self.compatible_source_generations.write().unwrap() = compatible_sources; self.descriptors - .install_catalog(Arc::clone(&catalog)) + .install_catalog_with_outputs(Arc::clone(&catalog), outputs) .map_err(|error| error.to_string())?; inventory.install(generation.clone()); if closed.as_ref() == Some(&generation) { @@ -1149,25 +1246,57 @@ impl SketchStore { if policy_fp.is_unset() { return Vec::new(); } + let Some(output) = self.descriptors.stored_output_reference(policy_fp.into()) else { + return Vec::new(); + }; + self.storage_handles_for_output(output) + } + + pub(crate) fn is_current_storage_handle(&self, handle: u64) -> bool { + let generation = self.active_catalog_generation(); + self.instances + .read() + .unwrap() + .get(&handle) + .is_some_and(|binding| { + binding.catalog_generation == generation + && binding.stored_output_reference + == self + .descriptors + .stored_output_reference(binding.policy_fp.into()) + }) + } + + pub fn stored_output_for_handle(&self, handle: u64) -> Option { + self.instances + .read() + .unwrap() + .get(&handle) + .and_then(|binding| binding.stored_output_reference) + } + + pub fn storage_handles_for_output(&self, output: StoredOutputReference) -> Vec { + if self + .descriptors + .stored_output_reference(output.definition_id) + != Some(output) + { + return Vec::new(); + } let candidates: Vec<_> = self - .policy_to_series_ids + .output_to_storage_handles .read() .unwrap() - .get(&policy_fp) + .get(&output) .map(|set| set.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidates .into_iter() .filter(|sid| { instances.get(sid).is_some_and(|binding| { - Self::instance_visible_for_read( - binding, - generation.as_deref(), - &compatible_sources, - ) + Self::instance_visible_in_generation(binding, generation.as_deref()) }) }) .collect() @@ -1186,31 +1315,12 @@ impl SketchStore { } } - fn instance_visible_for_read( - binding: &SdsBinding, - generation: Option<&CatalogGeneration>, - compatible_sources: &BTreeMap, - ) -> bool { - if Self::instance_visible_in_generation(binding, generation) { - return true; - } - let Some(generation) = generation else { - return false; - }; - let definition = SummaryDefinitionId::from(binding.metadata.policy_fp); - !matches!( - binding.data_descriptor.source, - asap_types::sds::DataSourceIdentity::Derived { .. } - ) && compatible_sources.get(&definition) == binding.catalog_generation.as_deref() - && binding.catalog_generation.as_deref() != Some(generation) - } - #[cfg(test)] /// Live policy count — number of distinct fingerprints with at /// least one sid. Useful for telemetry / `/runtime` introspection /// (mirrors the legacy "active aggregation count" metric). pub fn policy_count(&self) -> usize { - self.policy_to_series_ids.read().unwrap().len() + self.output_to_storage_handles.read().unwrap().len() } /// Look up the metadata for a sid (cloned because callers usually @@ -1267,11 +1377,10 @@ impl SketchStore { snapshot_sha256: reference.snapshot_sha256, }; let instances = self.instances.read().unwrap(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let durable = self.persistence_read.read().unwrap().clone(); let mut reported = BTreeMap::new(); for (series_id, binding) in instances.iter() { - if !Self::instance_visible_for_read(binding, Some(&generation), &compatible_sources) { + if !Self::instance_visible_in_generation(binding, Some(&generation)) { continue; } let reused_from_generation = (binding.catalog_generation.as_deref() @@ -1312,17 +1421,29 @@ impl SketchStore { .map_err(|_| "summary instance start exceeds signed timestamp range")?; let end_ms = i64::try_from(window.1) .map_err(|_| "summary instance end exceeds signed timestamp range")?; - let group_fingerprint = xxhash_rust::xxh64::xxh64( - &serde_json::to_vec(&group_values).map_err(|error| error.to_string())?, - 0, - ); - let instance_id = asap_types::sds::SummaryInstanceCoordinates { - summary_definition_id, - time_range: asap_types::sds::HalfOpenTimeRange { start_ms, end_ms }, - group_values: group_values.clone(), + let output = StoredOutputReference { + stored_output_id: *stored_output_id, + definition_id: summary_definition_id, + }; + if binding.stored_output_reference != Some(output) { + return Err("inventory output differs from stored payload binding".into()); + } + let address = asap_types::sds::StoredSummaryKey { + plan_id: generation.plan_id, + plan_version: generation.plan_version, + output, + population: group_values.clone(), + window: HalfOpenTimeRange { start_ms, end_ms }, + }; + let instance_id = asap_types::sds::SummaryInstanceId::new( + address.storage_key().map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let mut payload_address = address; + if let Some(source) = &reused_from_generation { + payload_address.plan_id = source.plan_id; + payload_address.plan_version = source.plan_version; } - .instance_id() - .map_err(|error| error.to_string())?; let instance = SummaryInstance { instance_id: instance_id.clone(), stored_output_id: *stored_output_id, @@ -1339,10 +1460,7 @@ impl SketchStore { }, state_reference: SummaryStateReference { store: "summary-store".into(), - key: format!( - "series:{series_id}:pane:{}-{}:group:{group_fingerprint:016x}", - window.0, window.1 - ), + key: payload_address.storage_key().map_err(|e| e.to_string())?, state_schema_version: binding.summary_descriptor.state_schema_version, generation: reused_from_generation .as_ref() @@ -2472,7 +2590,6 @@ impl SketchStore { .map(|sids| sids.iter().copied().collect()) .unwrap_or_default(); let generation = self.active_catalog_generation(); - let compatible_sources = self.compatible_source_generations.read().unwrap(); let instances = self.instances.read().unwrap(); candidate_sids .iter() @@ -2481,11 +2598,7 @@ impl SketchStore { .get(sid) .map(|m| { required_keys.is_subset(&m.group_by_keys) - && Self::instance_visible_for_read( - m, - generation.as_deref(), - &compatible_sources, - ) + && Self::instance_visible_in_generation(m, generation.as_deref()) }) .unwrap_or(false) }) @@ -2687,7 +2800,7 @@ impl SketchStore { } /// Record that the store has reconciled against the - /// `Arc` identified by `config_ptr` (the value of + /// `Arc` identified by `config_ptr` (the value of /// `Arc::as_ptr(..) as usize`), returning `true` if this is a *new* /// config pointer (i.e. the caller should run a full reconcile) or /// `false` if the store already reconciled against this exact @@ -2738,25 +2851,34 @@ impl SketchStore { .collect() } - fn metadata_record(&self, m: &SdsBinding) -> Option { + fn metadata_record( + &self, + m: &SdsBinding, + ) -> Option { let mut record = self.metadata_record_without_completion(m)?; - record.completed_through_ms = self.completed_windows.read().unwrap().get(&m.sid).copied(); + record.completed_through_ms = self + .completed_windows + .read() + .unwrap() + .get(&m.storage_handle) + .copied(); Some(record) } fn metadata_record_without_completion( &self, m: &SdsBinding, - ) -> Option { + ) -> Option { let mut record = - crate::storage_engines::sketch_db::index::persistence::metadata::SidMetaRecord::new( - m.sid, + crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataRecord::new( + m.storage_handle, m.metric_name.clone(), m.group_by_keys.iter().cloned().collect(), &m.agg_kind, m.first_seen_unix_ms, ); if !m.policy_fp.is_unset() { + record.stored_output_reference = m.stored_output_reference; record.summary_definition_id = Some(SummaryDefinitionId::from(m.policy_fp)); record.catalog_generation = Some(Arc::clone(m.catalog_generation.as_ref()?)); } @@ -2778,14 +2900,14 @@ impl SketchStore { .load() .map_err(|error| error.to_string())? .into_iter() - .find(|record| record.sid == instance.sid) + .find(|record| record.storage_handle == instance.storage_handle) .ok_or("cannot persist lifecycle without catalog identity")?, }; record.retired_at_ms = instance.retired_at_ms; record.expires_at_ms = instance.expires_at_ms; record.removed = removed; writer.upsert_all(&[record]).map_err(|error| { - tracing::error!(sid = instance.sid, %error, "durable summary lifecycle publication failed"); + tracing::error!(sid = instance.storage_handle, %error, "durable summary lifecycle publication failed"); error.to_string() }) } @@ -2910,7 +3032,7 @@ impl SketchStore { } /// Drop a sid's metadata + its series state + both secondary-index - /// entries (`policy_to_series_ids` and `metric_to_series_ids`). Mirrors + /// entries (`output_to_storage_handles` and `metric_to_series_ids`). Mirrors /// `SchemaRegistry::remove_schema` for the eviction path's /// post-data-drop cleanup. Returns the removed metadata, or `None` /// if the sid was absent. @@ -2918,7 +3040,7 @@ impl SketchStore { /// ## Atomicity (P2-1) /// /// Takes all three index write guards together in the fixed order - /// `instances → policy_to_series_ids → metric_to_series_ids` so the removal is + /// `instances → output_to_storage_handles → metric_to_series_ids` so the removal is /// atomic with respect to a concurrent reader — the sid never /// lingers in a secondary index after it has left `instances`. The /// `series` DashMap is touched after the index guards are released @@ -2927,7 +3049,7 @@ impl SketchStore { pub fn remove_instance(&self, sid: u64) -> Option> { let _mutation = self.begin_state_mutation(); let removed = { - // Fixed lock order: instances → policy_to_series_ids → metric_to_series_ids. + // Fixed lock order: instances → output_to_storage_handles → metric_to_series_ids. let mut instances = self.instances.write().ok()?; if let Some(instance) = instances.get(&sid) { self.persist_lifecycle(instance, true).ok()?; @@ -2942,15 +3064,15 @@ impl SketchStore { ), ); } - let mut policy_idx = self.policy_to_series_ids.write().unwrap(); + let mut policy_idx = self.output_to_storage_handles.write().unwrap(); let mut metric_idx = self.metric_to_series_ids.write().unwrap(); let removed = instances.remove(&sid); if let Some(meta) = &removed { - if !meta.policy_fp.is_unset() { - if let Some(set) = policy_idx.get_mut(&meta.policy_fp) { + if let Some(output) = meta.stored_output_reference { + if let Some(set) = policy_idx.get_mut(&output) { set.remove(&sid); if set.is_empty() { - policy_idx.remove(&meta.policy_fp); + policy_idx.remove(&output); } } } @@ -3025,6 +3147,7 @@ impl SketchStore { /// once a sid is retired by [`crate::storage_engines::sketch_db::lifecycle::reconcile_from_streaming_config`] /// further writes are rejected here so the eviction sweep can /// drop residual state cleanly. + #[cfg(test)] pub fn ingest_precompute_for_agg_config>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, @@ -3039,6 +3162,7 @@ impl SketchStore { self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, output, accumulator) } + #[cfg(test)] fn ingest_precompute_config_with_admission>>( &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, @@ -3113,7 +3237,7 @@ impl SketchStore { // query path; capability-matching couldn't see it. if !self.register_with_instances( SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: agg_cfg.metric.clone(), group_by_keys, capability: Some(capability), @@ -3157,6 +3281,7 @@ impl SketchStore { /// The §6.3 ingest barrier (`Retired` / `Expired` sids reject /// writes) and first-sight metadata registration are identical to /// the mint-driven path. + #[cfg(test)] pub fn ingest_precompute_with_series_id( &self, sid: u64, @@ -3305,8 +3430,9 @@ impl SketchStore { // Publish the writer before recovery or any background work so a // concurrent lifecycle operation cannot succeed without persistence. - let metadata_writer = - Arc::new(persistence::metadata::SidMetadataStore::new(&cfg.disk_path)); + let metadata_writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( + &cfg.disk_path, + )); // Restore the generation-wide raw admission barrier before exposing // recovered state or accepting another producer after restart. if let Some(closed) = metadata_writer.load_finite_closure()? { @@ -3329,7 +3455,7 @@ impl SketchStore { .filter(|record| record.removed) .map(|record| { ( - record.sid, + record.storage_handle, (record.catalog_generation, record.summary_definition_id), ) }), @@ -3407,9 +3533,9 @@ impl SketchStore { /// fresh dir) — those sids stay invisible until a live DataPoint /// re-registers them, the same as pre-fix behavior. pub fn register_recovered_disk_series(&self, disk_path: &std::path::Path) -> usize { - use crate::storage_engines::sketch_db::index::persistence::metadata::SidMetadataStore; + use crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataFile; - let store = SidMetadataStore::new(disk_path); + let store = StoredOutputMetadataFile::new(disk_path); let records = match store.load() { Ok(r) => r, Err(e) => { @@ -3424,7 +3550,7 @@ impl SketchStore { self.completed_windows .write() .unwrap() - .entry(rec.sid) + .entry(rec.storage_handle) .and_modify(|current| *current = (*current).max(end)) .or_insert(end); } @@ -3432,7 +3558,7 @@ impl SketchStore { continue; } // Don't clobber a live-registered instance. - if self.instance(rec.sid).is_some() { + if self.instance(rec.storage_handle).is_some() { continue; } let catalog = self.descriptors.authoritative_snapshot(); @@ -3444,9 +3570,11 @@ impl SketchStore { (Some(definition), Some(generation), Some((catalog, installed_generation))) => { if generation != installed_generation || !catalog.definitions.contains_key(definition) + || rec.stored_output_reference + != self.descriptors.stored_output_reference(*definition) { tracing::warn!( - sid = rec.sid, + sid = rec.storage_handle, "persisted summary catalog provenance differs; leaving state unbound" ); continue; @@ -3457,13 +3585,13 @@ impl SketchStore { // legacy path. Never promote such state into a catalog binding. (None, None, None) => PolicyFingerprint::UNSET, _ => { - tracing::warn!(sid = rec.sid, "persisted summary has no matching authoritative identity; leaving state unbound"); + tracing::warn!(sid = rec.storage_handle, "persisted summary has no matching authoritative identity; leaving state unbound"); continue; } }; let Some(agg_kind) = rec.agg_kind() else { tracing::warn!( - sid = rec.sid, + sid = rec.storage_handle, "skipping recovered sid: unrecognized agg_kind in sidecar" ); continue; @@ -3471,7 +3599,7 @@ impl SketchStore { let capability = rec.capability(); let accuracy = rec.accuracy(); self.register(SummarySeriesMetadata { - sid: rec.sid, + storage_handle: rec.storage_handle, metric_name: rec.metric_name, group_by_keys: rec.group_by_keys.into_iter().collect(), capability, @@ -3482,7 +3610,7 @@ impl SketchStore { expires_at_ms: rec.expires_at_ms, policy_fp, }); - if self.instance(rec.sid).is_some() { + if self.instance(rec.storage_handle).is_some() { registered += 1; } } @@ -3556,8 +3684,9 @@ impl crate::storage_engines::sketch_db::index::persistence::EpochSource for Sket fn instance_metadata_for_persist( &self, sid: u64, - ) -> Option - { + ) -> Option< + crate::storage_engines::sketch_db::index::persistence::metadata::StoredOutputMetadataRecord, + > { let g = self.instances.read().ok()?; let m = g.get(&sid)?; self.metadata_record(m) @@ -3753,7 +3882,7 @@ mod tests { relative_accuracy: 0.01, }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "m".into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), @@ -4325,11 +4454,11 @@ mod tests { let retired = idx.list_by_status(AggStatus::Retired); let expired = idx.list_by_status(AggStatus::Expired); assert_eq!(active.len(), 1); - assert_eq!(active[0].sid, 1); + assert_eq!(active[0].storage_handle, 1); assert_eq!(retired.len(), 1); - assert_eq!(retired[0].sid, 2); + assert_eq!(retired[0].storage_handle, 2); assert_eq!(expired.len(), 1); - assert_eq!(expired[0].sid, 3); + assert_eq!(expired[0].storage_handle, 3); } #[test] @@ -4339,7 +4468,7 @@ mod tests { idx.append_sample(1, BTreeMap::new(), (0, 10), sample(1)); assert_eq!(idx.classify(1), SeriesLookup::Hit); let removed = idx.remove_instance(1).expect("sid known"); - assert_eq!(removed.sid, 1); + assert_eq!(removed.storage_handle, 1); assert_eq!(idx.classify(1), SeriesLookup::Unknown); } @@ -5066,7 +5195,7 @@ mod tests { #[test] fn catalog_recovery_keeps_legacy_and_foreign_generation_state_unbound() { use crate::storage_engines::sketch_db::index::persistence::metadata::{ - SidMetaRecord, SidMetadataStore, + StoredOutputMetadataFile, StoredOutputMetadataRecord, }; let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( @@ -5078,15 +5207,15 @@ mod tests { .unwrap(); let fingerprint = plan.precompute_plan.materializations[0].policy_fingerprint(); let metadata = meta_with_policy(507, fingerprint); - let record = SidMetaRecord::new( - metadata.sid, + let record = StoredOutputMetadataRecord::new( + metadata.storage_handle, metadata.metric_name.clone(), metadata.group_by_keys.iter().cloned().collect(), &metadata.agg_kind, 0, ); let tmp = tempfile::tempdir().unwrap(); - let sidecar = SidMetadataStore::new(tmp.path()); + let sidecar = StoredOutputMetadataFile::new(tmp.path()); sidecar.upsert_all(&[record.clone()]).unwrap(); let store = SketchStore::new(); store @@ -5109,7 +5238,7 @@ mod tests { } #[test] - fn unchanged_definition_explicitly_reuses_a_committed_previous_generation_payload() { + fn unchanged_definition_does_not_rebind_previous_generation_payload() { let snapshot: control_plane::physical::compiler::BackendLocalPlanningInput = serde_json::from_str(include_str!( "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" @@ -5128,27 +5257,7 @@ mod tests { let mut next = plan.summary_catalog; next.plan_version += 1; store.install_summary_catalog(Arc::new(next)).unwrap(); - assert_eq!(store.series_ids_for_policy(fingerprint), vec![509]); - let output = asap_types::sds::StoredOutputReference::for_definition(fingerprint.into()) - .stored_output_id; - let inventory = store - .observed_summary_inventory( - "backend-a", - "store-a", - &BTreeMap::from([( - SummaryDefinitionId::from(fingerprint), - (output, "producer-a".into()), - )]), - 1, - 100, - ) - .unwrap(); - let reused = inventory.instances.values().next().unwrap(); - assert_eq!(reused.stored_output_id, output); - assert_eq!( - reused.reused_from_generation.as_ref().unwrap().plan_version + 1, - reused.catalog_generation.plan_version - ); + assert!(store.series_ids_for_policy(fingerprint).is_empty()); let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( plan.precompute_plan.envelope.plan_id, plan.precompute_plan.envelope.plan_version + 2, @@ -5293,7 +5402,7 @@ mod tests { .unwrap(); store.register(meta_with_policy(850, fingerprint)); let generation = store.active_catalog_generation().unwrap(); - let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), )); *store.persistence_metadata.write().unwrap() = Some(writer.clone()); @@ -5466,7 +5575,7 @@ mod tests { let store = SketchStore::new(); store.register(meta(800)); let directory = tempfile::tempdir().unwrap(); - let writer = Arc::new(persistence::metadata::SidMetadataStore::new( + let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), )); // A directory in place of the sidecar causes the real writer to fail. @@ -5626,7 +5735,14 @@ mod tests { assert!(!inventory.instances.is_empty()); assert!(inventory.instances.values().all(|instance| { instance.group_values.get("job").map(String::as_str) == Some("api") - && instance.state_reference.key.starts_with("series:506:pane:") + && serde_json::from_str::( + &instance.state_reference.key, + ) + .is_ok_and(|key| { + key.output.definition_id.fingerprint() == fingerprint + && key.population == instance.group_values + && key.window == instance.time_range + }) })); } @@ -5656,7 +5772,7 @@ mod tests { fn meta_kll_host(sid: u64) -> SummarySeriesMetadata { let cfg = SketchConfig::Kll { k: 200 }; SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: "http_latency".into(), group_by_keys: ["host".to_string()].into_iter().collect(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::Kll))), @@ -6339,4 +6455,165 @@ mod tests { } persistence.shutdown(); } + // Exact selected-output bindings survive disk recovery and cannot be + // rebound to a different output merely because its definition matches. + #[test] + fn stored_output_reference_isolates_identity_and_survives_restart() { + use crate::drivers::ingest::series_resolver::SeriesIdResolver; + let snapshot = serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let mut plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let schema = &mut plan.precompute_plan.schemas[0]; + schema.stored_output_reference.stored_output_id = asap_types::sds::StoredOutputId(101); + let output = schema.stored_output_reference; + let fingerprint = output.definition_id.fingerprint(); + let directory = tempfile::tempdir().unwrap(); + let disk = directory.path().join("state"); + let wal = directory.path().join("outputs.wal"); + let handle; + { + let store = Arc::new(SketchStore::new()); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + let resolver = SeriesIdResolver::open(wal.clone()).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + handle = store + .resolve_output_storage_handle( + &resolver, + output.definition_id, + "population-a", + Some(&generation), + ) + .unwrap(); + let other_population = store + .resolve_output_storage_handle( + &resolver, + output.definition_id, + "population-b", + Some(&generation), + ) + .unwrap(); + assert_ne!(handle, other_population); + store.register(meta_with_policy(handle, fingerprint)); + let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); + for pane in 0..4 { + store.append_sample( + handle, + BTreeMap::new(), + (pane * 30_000, (pane + 1) * 30_000), + sample(1), + ); + } + assert!(wait_until( + || !persistence.manifest.live_parts().is_empty() + && store.list_sealed_epochs_len() == 0, + Duration::from_secs(5) + )); + assert_eq!(store.storage_handles_for_output(output), vec![handle]); + let wrong = StoredOutputReference { + stored_output_id: asap_types::sds::StoredOutputId(102), + ..output + }; + assert!(store.storage_handles_for_output(wrong).is_empty()); + persistence.shutdown(); + } + { + let store = Arc::new(SketchStore::new()); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + let resolver = SeriesIdResolver::open(wal.clone()).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + assert_eq!( + store + .resolve_output_storage_handle( + &resolver, + output.definition_id, + "population-a", + Some(&generation) + ) + .unwrap(), + handle + ); + let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); + assert_eq!(store.storage_handles_for_output(output), vec![handle]); + assert!(!store.query_range(handle, 0, 120_000).is_empty()); + persistence.shutdown(); + } + plan.precompute_plan.schemas[0] + .stored_output_reference + .stored_output_id = asap_types::sds::StoredOutputId(102); + let store = Arc::new(SketchStore::new()); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + let resolver = SeriesIdResolver::open(wal).unwrap(); + let generation = store.active_catalog_generation().unwrap(); + assert_ne!( + store + .resolve_output_storage_handle( + &resolver, + output.definition_id, + "population-a", + Some(&generation) + ) + .unwrap(), + handle + ); + let mut persistence = store.start_persistence(durable_cfg(disk)).unwrap(); + assert!(store + .storage_handles_for_output(plan.precompute_plan.schemas[0].stored_output_reference) + .is_empty()); + assert!(store.instance(handle).is_none()); + persistence.shutdown(); + } + // A versioned output address cannot silently resolve the previous version. + #[test] + fn stored_output_reference_does_not_alias_previous_plan_version() { + let snapshot = serde_json::from_str(include_str!( + "../../../../../docs/examples/asapquery-compatibility-demo-snapshot.json" + )) + .unwrap(); + let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) + .compile_promql() + .unwrap(); + let output = plan.precompute_plan.schemas[0].stored_output_reference; + let store = SketchStore::new(); + store + .install_precompute_plan( + Arc::new(plan.summary_catalog.clone()), + &plan.precompute_plan, + ) + .unwrap(); + store.register(meta_with_policy(100, output.definition_id.fingerprint())); + assert_eq!(store.storage_handles_for_output(output), vec![100]); + let mut next = plan.summary_catalog; + next.plan_version += 1; + store + .install_catalog_outputs( + Arc::new(next), + plan.precompute_plan + .schemas + .iter() + .map(|schema| (schema.materialization, schema.stored_output_reference)) + .collect(), + ) + .unwrap(); + assert!(store.storage_handles_for_output(output).is_empty()); + assert!(!store.is_current_storage_handle(100)); + } } diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 2d16af78..ee3e7470 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -115,7 +115,7 @@ impl SchemaEvictionService { let _ = &self.backfill; for meta in expired { - let sid = meta.sid; + let sid = meta.storage_handle; let metric = meta.metric_name.clone(); if self.config.dry_run { @@ -193,7 +193,7 @@ pub fn warn_if_retention_inverted( #[cfg(test)] mod tests { use super::*; - use crate::storage_engines::types::{AggregationType, StreamingConfig}; + use crate::storage_engines::types::{AggregationType, InstalledPrecomputePlan}; use asap_physical_operators::accumulators::SumAccumulator; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::enums::WindowKind; @@ -233,7 +233,7 @@ mod tests { /// derived from each dummy_agg. Returns the config plus the /// marker_id → fingerprint mapping so callers can look up the /// right key. - fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { + fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { let mut map = HashMap::new(); let mut id_to_fp = HashMap::new(); for &id in ids { @@ -242,12 +242,12 @@ mod tests { id_to_fp.insert(id, fp); map.insert(fp, cfg); } - (Arc::new(StreamingConfig::new(map)), id_to_fp) + (Arc::new(InstalledPrecomputePlan::new(map)), id_to_fp) } fn write_one( summary_store: &SketchStore, - streaming_config: &StreamingConfig, + installed_precompute_plan: &InstalledPrecomputePlan, agg_id: u64, ts: u64, ) -> u64 { @@ -260,7 +260,9 @@ mod tests { None, asap_types::PolicyFingerprint(agg_id), ); - let agg_cfg = streaming_config.get_aggregation_config(agg_id).unwrap(); + let agg_cfg = installed_precompute_plan + .get_aggregation_config(agg_id) + .unwrap(); // Test-scoped resolver — each call mints fresh. Production // shares one resolver across all sinks; tests don't need that // because each fixture is isolated. Static-lifetime so multiple diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index 0b712f45..c5bd2d50 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -10,7 +10,7 @@ use std::collections::{BTreeSet, HashSet}; use std::sync::Arc; use std::time::Duration; -use crate::storage_engines::types::StreamingConfig; +use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; @@ -22,7 +22,7 @@ use crate::storage_engines::sketch_db::lifecycle::AggStatus; pub struct SidReconcileSummary { /// Sids that transitioned `Active → Retired` because their /// content signature is no longer present in the new - /// `StreamingConfig`. Already-Retired or already-Expired sids + /// `InstalledPrecomputePlan`. Already-Retired or already-Expired sids /// are not re-touched. pub retired: Vec, } @@ -35,14 +35,14 @@ pub struct SidReconcileSummary { /// behavior. /// Ingest-path entry point: reconcile only when `config` is a config /// the store has not yet reconciled against. The streaming config is a -/// lock-free `Arc>` whose `Arc` identity only +/// lock-free `Arc>` whose `Arc` identity only /// changes on a (rare) control-plane swap, so in steady state every /// ingest batch hands us the *same* `Arc`. Gating on the `Arc` data /// pointer collapses the per-batch reconcile to a single relaxed atomic /// load in that common case, skipping the full catalog scan + any /// signature derivation. /// -/// Pass the same `Arc` the ingest batch snapshotted so +/// Pass the same `Arc` the ingest batch snapshotted so /// the pointer is stable; the snapshot is held alive for the call's /// duration, so the pointer cannot be reused by a concurrently-dropped /// config (no ABA hazard within a batch). @@ -51,7 +51,7 @@ pub struct SidReconcileSummary { /// `Some(summary)` with the retired sids when a reconcile actually ran. pub fn reconcile_if_config_changed( store: &SketchStore, - config: &Arc, + config: &Arc, retention: Duration, ) -> Option { let config_ptr = Arc::as_ptr(config) as usize; @@ -63,7 +63,7 @@ pub fn reconcile_if_config_changed( pub fn reconcile_from_streaming_config( store: &SketchStore, - config: &StreamingConfig, + config: &InstalledPrecomputePlan, retention: Duration, ) -> SidReconcileSummary { let live_signatures = build_live_signature_set(config); @@ -176,7 +176,7 @@ fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { signature_bytes(&cfg.metric, &agg_kind, &group_by_keys) } -fn build_live_signature_set(config: &StreamingConfig) -> HashSet> { +fn build_live_signature_set(config: &InstalledPrecomputePlan) -> HashSet> { config .materializations() .values() @@ -304,7 +304,7 @@ mod tests { ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, @@ -321,12 +321,12 @@ mod tests { } } - fn streaming(configs: Vec) -> StreamingConfig { + fn streaming(configs: Vec) -> InstalledPrecomputePlan { let mut map = HashMap::new(); for (i, c) in configs.into_iter().enumerate() { map.insert(i as u64 + 1, c); } - StreamingConfig::new(map) + InstalledPrecomputePlan::new(map) } #[test] @@ -410,7 +410,7 @@ mod tests { ) -> SummarySeriesMetadata { let group_by_keys: BTreeSet = group_by.into_iter().map(|s| s.to_string()).collect(); SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.to_string(), group_by_keys, capability: None, diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs index 78f5e8df..f73f1a69 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/status.rs @@ -13,11 +13,11 @@ use std::time::Duration; /// clock crosses `expires_at_ms`, without any state mutation). #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum AggStatus { - /// Sid is reachable from the current `StreamingConfig`. Writes + /// Sid is reachable from the current `InstalledPrecomputePlan`. Writes /// accepted; queries see live data. Active, /// Sid's signature no longer appears in the current - /// `StreamingConfig` but its data is still within retention. + /// `InstalledPrecomputePlan` but its data is still within retention. /// Writes rejected by the §6.3 sid-level ingest barrier /// (`SketchStore::ingest_precompute_for_agg_config` returns /// `None`); reads allowed for queries that reference the diff --git a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs index 6018bdeb..dc9e8cb6 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/flusher.rs @@ -42,7 +42,7 @@ pub(crate) struct FlusherShared { /// Per-sid metadata sidecar — upserted on every flush so recovery can /// re-register disk-resident sids as queryable instances. See /// [`super::metadata`]. - pub sid_metadata: Arc, + pub sid_metadata: Arc, pub next_part_id: AtomicU64, pub shutdown: AtomicBool, /// Woken by the insert path when it hits `hard_cap_bytes` and by @@ -77,7 +77,9 @@ impl FlusherHandle { where S: EpochSource + 'static, { - let metadata = Arc::new(super::metadata::SidMetadataStore::new(&cfg.disk_path)); + let metadata = Arc::new(super::metadata::StoredOutputMetadataFile::new( + &cfg.disk_path, + )); Self::start_with_metadata(cfg, manifest, source, metadata) } @@ -85,7 +87,7 @@ impl FlusherHandle { cfg: SketchStorePersistenceConfig, manifest: Arc, source: Arc, - sid_metadata: Arc, + sid_metadata: Arc, ) -> PersistResult where S: EpochSource + 'static, @@ -134,7 +136,7 @@ impl FlusherHandle { }) } - pub(crate) fn metadata_store(&self) -> Arc { + pub(crate) fn metadata_store(&self) -> Arc { Arc::clone(&self.inner.sid_metadata) } @@ -345,7 +347,7 @@ fn run_tick(shared: &Arc, source: &S) -> PersistR // is durable whenever the part it describes is. A sidecar write // failure must NOT abort the flush (the part is already durable) // — log and continue; recovery degrades to live-ingest re-register. - let sid_meta: Vec = { + let sid_meta: Vec = { let mut seen = std::collections::HashSet::new(); snapshots .iter() diff --git a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs index dcce818b..76025600 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/immutable_output.rs @@ -5,7 +5,7 @@ use sha2::{Digest, Sha256}; use super::flusher::{FlusherHandle, FlusherShared}; use super::manifest::PartEntry; -use super::metadata::SidMetaRecord; +use super::metadata::StoredOutputMetadataRecord; use super::part::{part_dir_path, PartReader, PartWriter}; use super::source::{EpochSnapshot, EpochSnapshotEntry}; use super::{PersistError, PersistResult}; @@ -29,11 +29,15 @@ fn invalid(message: &str) -> PersistError { PersistError::Format(message.into()) } -fn validate_identity(current: &SidMetaRecord, record: &SidMetaRecord) -> PersistResult<()> { - if current.sid != record.sid +fn validate_identity( + current: &StoredOutputMetadataRecord, + record: &StoredOutputMetadataRecord, +) -> PersistResult<()> { + if current.storage_handle != record.storage_handle || current.removed || current.retired_at_ms.is_some() || current.expires_at_ms.is_some() + || current.stored_output_reference != record.stored_output_reference || current.summary_definition_id != record.summary_definition_id || current.catalog_generation != record.catalog_generation || current.metric_name != record.metric_name @@ -100,7 +104,7 @@ impl FlusherHandle { } pub fn publish_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], snapshot: &EpochSnapshot, ) -> PersistResult { @@ -120,13 +124,13 @@ impl FlusherShared { /// randomized sketch again. This never creates or replaces a payload. pub fn lookup_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], start_ms: u64, end_ms: u64, ) -> PersistResult> { self.sid_metadata.transaction(|records, _| { - let Some(current) = records.get(&record.sid.to_string()) else { + let Some(current) = records.get(&record.storage_handle.to_string()) else { return Ok(None); }; validate_identity(current, record)?; @@ -139,7 +143,7 @@ impl FlusherShared { if previous.input_digest != input_digest { return Err(invalid("immutable lookup input lineage differs")); } - self.validate_reserved_part(record.sid, previous)?; + self.validate_reserved_part(record.storage_handle, previous)?; if !self .manifest .live_parts() @@ -159,16 +163,16 @@ impl FlusherShared { /// input and payload. This does not guarantee source availability after GC. pub fn publish_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], snapshot: &EpochSnapshot, ) -> PersistResult { - if snapshot.agg_id != record.sid || record.removed { + if snapshot.agg_id != record.storage_handle || record.removed { return Err(invalid("immutable publication SID is invalid or removed")); } let payload_digest = fingerprint(snapshot)?; self.sid_metadata.transaction(|records, metadata| { - let key = record.sid.to_string(); + let key = record.storage_handle.to_string(); let mut current = records.get(&key).cloned().unwrap_or_else(|| record.clone()); validate_identity(¤t, record)?; // Completion fences window ends. An advancing overlapping full @@ -191,7 +195,7 @@ impl FlusherShared { "completed immutable retry differs from latest publication", )); } - self.validate_reserved_part(record.sid, previous)?; + self.validate_reserved_part(record.storage_handle, previous)?; if !self .manifest .live_parts() @@ -236,7 +240,7 @@ impl FlusherShared { if path.exists() { // Never overwrite a reserved part silently: callers can recover a // durable part without sources; damaged/partial parts fail closed. - self.validate_reserved_part(record.sid, &reservation)?; + self.validate_reserved_part(record.storage_handle, &reservation)?; } else { PartWriter::write_part(&path, reservation.part_id, std::slice::from_ref(snapshot))?; std::fs::File::open( @@ -244,7 +248,7 @@ impl FlusherShared { .ok_or_else(|| invalid("missing parts parent"))?, )? .sync_all()?; - self.validate_reserved_part(record.sid, &reservation)?; + self.validate_reserved_part(record.storage_handle, &reservation)?; } self.finish_reserved_part(&mut current, &reservation)?; records.insert(key, current); @@ -260,12 +264,15 @@ impl FlusherShared { /// validation. A different pending input cannot be acknowledged by this call. pub fn resume_matching_immutable_window( &self, - record: &SidMetaRecord, + record: &StoredOutputMetadataRecord, input_digest: [u8; 32], start_ms: u64, end_ms: u64, ) -> PersistResult> { - self.resume_immutable_window(record.sid, Some((record, input_digest, start_ms, end_ms))) + self.resume_immutable_window( + record.storage_handle, + Some((record, input_digest, start_ms, end_ms)), + ) } /// Complete a pending durable part without reconstructing its source input. @@ -281,7 +288,7 @@ impl FlusherShared { fn resume_immutable_window( &self, sid: u64, - expected: Option<(&SidMetaRecord, [u8; 32], u64, u64)>, + expected: Option<(&StoredOutputMetadataRecord, [u8; 32], u64, u64)>, ) -> PersistResult> { self.sid_metadata.transaction(|records, metadata| { let key = sid.to_string(); @@ -358,7 +365,7 @@ impl FlusherShared { fn finish_reserved_part( &self, - current: &mut SidMetaRecord, + current: &mut StoredOutputMetadataRecord, pending: &ImmutableOutputReservation, ) -> PersistResult<()> { let path = part_dir_path(&self.cfg.disk_path.join("parts"), pending.part_id); @@ -434,8 +441,8 @@ mod tests { ) .unwrap() } - fn record() -> SidMetaRecord { - SidMetaRecord::new( + fn record() -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( 1, "derived".into(), vec![], diff --git a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs index 552883c1..21f752e8 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/metadata.rs @@ -35,7 +35,7 @@ //! //! ## Format //! -//! A single JSON object `{ "": SidMetaRecord, ... }` written +//! A single JSON object `{ "": StoredOutputMetadataRecord, ... }` written //! atomically (tmp + rename) on every upsert. JSON (not the custom //! binary part format) because the record count equals live sid //! cardinality (small) and the schema is human-inspectable for @@ -300,8 +300,10 @@ impl AggKindRec { /// `capability` and `accuracy` are DERIVED from `agg_kind` on load, the /// same way the ingest path derives them, so the record stays minimal. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -pub struct SidMetaRecord { - pub sid: u64, +pub struct StoredOutputMetadataRecord { + #[serde(default)] + pub stored_output_reference: Option, + pub storage_handle: u64, /// Authoritative identity and provenance, absent on legacy sidecars. #[serde(default)] pub summary_definition_id: Option, @@ -328,7 +330,7 @@ pub struct SidMetaRecord { pub last_immutable: Option, } -impl SidMetaRecord { +impl StoredOutputMetadataRecord { /// Build a record from the live store-side fields. `agg_kind` is the /// structured `AggKind`; `capability`/`accuracy` are intentionally /// NOT stored (re-derived on load). @@ -340,7 +342,8 @@ impl SidMetaRecord { first_seen_unix_ms: i64, ) -> Self { Self { - sid, + storage_handle: sid, + stored_output_reference: None, summary_definition_id: None, catalog_generation: None, metric_name, @@ -409,8 +412,10 @@ struct DataDescriptorRec { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] -struct SidBindingRec { - sid: u64, +struct StoredOutputBindingRecord { + #[serde(default)] + stored_output_reference: Option, + storage_handle: u64, #[serde(default)] summary_definition_id: Option, #[serde(default)] @@ -441,15 +446,15 @@ struct SdsSidecar { catalog_generations: HashMap>, summary_descriptors: HashMap, data_descriptors: HashMap, - bindings: HashMap, + bindings: HashMap, } impl SdsSidecar { - fn from_records(records: impl IntoIterator) -> Self { + fn from_records(records: impl IntoIterator) -> Self { use crate::storage_engines::sketch_db::sds::{data_descriptor_id, summary_descriptor_id}; let mut sidecar = Self { - schema_version: 3, + schema_version: 4, catalog_generations: HashMap::new(), summary_descriptors: HashMap::new(), data_descriptors: HashMap::new(), @@ -489,9 +494,10 @@ impl SdsSidecar { digest }); sidecar.bindings.insert( - record.sid.to_string(), - SidBindingRec { - sid: record.sid, + record.storage_handle.to_string(), + StoredOutputBindingRecord { + storage_handle: record.storage_handle, + stored_output_reference: record.stored_output_reference, summary_definition_id: record.summary_definition_id, catalog_generation_sha256: generation_sha256, summary_descriptor_id: summary_id, @@ -509,7 +515,7 @@ impl SdsSidecar { sidecar } - fn into_records(self) -> PersistResult> { + fn into_records(self) -> PersistResult> { self.bindings .into_values() .map(|binding| { @@ -519,7 +525,7 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing summary descriptor {}", - binding.sid, binding.summary_descriptor_id + binding.storage_handle, binding.summary_descriptor_id )) })?; let data = self @@ -528,11 +534,12 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing data descriptor {}", - binding.sid, binding.data_descriptor_id + binding.storage_handle, binding.data_descriptor_id )) })?; - Ok(SidMetaRecord { - sid: binding.sid, + Ok(StoredOutputMetadataRecord { + storage_handle: binding.storage_handle, + stored_output_reference: binding.stored_output_reference, summary_definition_id: binding.summary_definition_id, catalog_generation: binding .catalog_generation_sha256 @@ -544,7 +551,7 @@ impl SdsSidecar { .ok_or_else(|| { PersistError::Format(format!( "SeriesId {} references missing catalog generation", - binding.sid + binding.storage_handle )) }) }) @@ -570,12 +577,12 @@ impl SdsSidecar { /// rewrite per flush tick is cheap and keeps the on-disk file always /// consistent with no log-replay machinery. #[derive(Debug)] -pub struct SidMetadataStore { +pub struct StoredOutputMetadataFile { path: PathBuf, writer: std::sync::Mutex<()>, } -impl SidMetadataStore { +impl StoredOutputMetadataFile { /// Open (or lazily create on first write) the sidecar at /// `/sid_metadata.json`. pub fn new(disk_path: &Path) -> Self { @@ -623,7 +630,7 @@ impl SidMetadataStore { /// doesn't exist yet (fresh dir, or parts written before this feature /// landed) or when it is unparsable (treated as "no recoverable /// metadata" — the live ingest path still re-registers on first DP). - pub fn load(&self) -> PersistResult> { + pub fn load(&self) -> PersistResult> { let mut f = match File::open(&self.path) { Ok(f) => f, Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -647,7 +654,7 @@ impl SidMetadataStore { }; if matches!( value.get("schema_version").and_then(|v| v.as_u64()), - Some(2 | 3) + Some(4) ) { let sidecar: SdsSidecar = match serde_json::from_value(value) { Ok(sidecar) => sidecar, @@ -672,35 +679,29 @@ impl SidMetadataStore { } }; } - // Version 1 was a flat SeriesId map. Read it and normalize on the next write. - let map: HashMap = match serde_json::from_value(value) { - Ok(map) => map, - Err(error) => { - tracing::warn!(path = %self.path.display(), %error, "legacy sid metadata is invalid; ignoring"); - return Ok(Vec::new()); - } - }; - Ok(map.into_values().collect()) + Err(PersistError::Format( + "stored-output metadata requires schema version 4".into(), + )) } /// Upsert a batch of records, merging with whatever is already on /// disk (last write wins per sid). Atomic via tmp + rename + dir /// fsync, matching the manifest's durability discipline. - pub fn upsert_all(&self, records: &[SidMetaRecord]) -> PersistResult<()> { + pub fn upsert_all(&self, records: &[StoredOutputMetadataRecord]) -> PersistResult<()> { let _writer = self.writer.lock().map_err(|_| { PersistError::Io(std::io::Error::other("summary metadata writer poisoned")) })?; if records.is_empty() { return Ok(()); } - let mut map: HashMap = self + let mut map: HashMap = self .load_strict()? .into_iter() - .map(|r| (r.sid.to_string(), r)) + .map(|r| (r.storage_handle.to_string(), r)) .collect(); let mut changed = false; for r in records { - let key = r.sid.to_string(); + let key = r.storage_handle.to_string(); let mut next = r.clone(); if let Some(existing) = map.get(&key) { // Lifecycle is monotone for a SeriesId. An older flush snapshot @@ -730,7 +731,7 @@ impl SidMetadataStore { self.write_atomic(json.as_bytes()) } - pub fn load_strict(&self) -> PersistResult> { + pub fn load_strict(&self) -> PersistResult> { let bytes = match fs::read(&self.path) { Ok(bytes) => bytes, Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), @@ -739,7 +740,7 @@ impl SidMetadataStore { let value: serde_json::Value = serde_json::from_slice(&bytes) .map_err(|error| PersistError::Format(format!("invalid SID metadata: {error}")))?; if let Some(version) = value.get("schema_version") { - if !matches!(version.as_u64(), Some(2 | 3)) { + if !matches!(version.as_u64(), Some(4)) { return Err(PersistError::Format( "unsupported SID metadata version".into(), )); @@ -748,15 +749,18 @@ impl SidMetadataStore { .map_err(|error| PersistError::Format(error.to_string()))?; sidecar.into_records() } else { - let records: HashMap = serde_json::from_value(value) - .map_err(|error| PersistError::Format(error.to_string()))?; - Ok(records.into_values().collect()) + Err(PersistError::Format( + "stored-output metadata requires schema version 4".into(), + )) } } pub(super) fn transaction( &self, - operation: impl FnOnce(&mut HashMap, &Self) -> PersistResult, + operation: impl FnOnce( + &mut HashMap, + &Self, + ) -> PersistResult, ) -> PersistResult { let _writer = self .writer @@ -765,13 +769,13 @@ impl SidMetadataStore { let mut records = self .load_strict()? .into_iter() - .map(|r| (r.sid.to_string(), r)) + .map(|r| (r.storage_handle.to_string(), r)) .collect(); operation(&mut records, self) } pub(super) fn write_records( &self, - records: &HashMap, + records: &HashMap, ) -> PersistResult<()> { let sidecar = SdsSidecar::from_records(records.values().cloned()); let bytes = @@ -810,8 +814,8 @@ mod tests { use super::*; use tempfile::TempDir; - fn sketch_meta(sid: u64) -> SidMetaRecord { - SidMetaRecord::new( + fn sketch_meta(sid: u64) -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( sid, "http_latency".into(), vec!["host".into(), "zone".into()], @@ -824,8 +828,8 @@ mod tests { ) } - fn exact_meta(sid: u64) -> SidMetaRecord { - SidMetaRecord::new( + fn exact_meta(sid: u64) -> StoredOutputMetadataRecord { + StoredOutputMetadataRecord::new( sid, "http_requests_total".into(), vec!["zone".into()], @@ -841,14 +845,14 @@ mod tests { #[test] fn load_on_missing_file_is_empty() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); assert!(s.load().unwrap().is_empty()); } #[test] fn authoritative_bindings_share_one_persisted_catalog_generation() { let directory = tempfile::tempdir().unwrap(); - let store = SidMetadataStore::new(directory.path()); + let store = StoredOutputMetadataFile::new(directory.path()); let generation = std::sync::Arc::new(asap_types::sds::CatalogGeneration { schema_version: 1, plan_id: 7, @@ -859,7 +863,7 @@ mod tests { first.summary_definition_id = Some(asap_types::PolicyFingerprint(7).into()); first.catalog_generation = Some(std::sync::Arc::clone(&generation)); let mut second = first.clone(); - second.sid = 2; + second.storage_handle = 2; store.upsert_all(&[first, second]).unwrap(); let json: serde_json::Value = serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); @@ -879,18 +883,18 @@ mod tests { #[test] fn upsert_then_load_round_trips() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); s.upsert_all(&[sketch_meta(1), exact_meta(2)]).unwrap(); let mut got = s.load().unwrap(); - got.sort_by_key(|r| r.sid); + got.sort_by_key(|r| r.storage_handle); assert_eq!(got.len(), 2); assert_eq!(got[0], sketch_meta(1)); assert_eq!(got[1], exact_meta(2)); let persisted: serde_json::Value = serde_json::from_slice(&std::fs::read(s.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); + assert_eq!(persisted["schema_version"], 4); assert_eq!( persisted["summary_descriptors"].as_object().unwrap().len(), 2 @@ -902,7 +906,7 @@ mod tests { #[test] fn equivalent_sids_persist_one_copy_of_each_descriptor() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); + let store = StoredOutputMetadataFile::new(tmp.path()); let mut second = sketch_meta(2); second.first_seen_unix_ms = 9999; store.upsert_all(&[sketch_meta(1), second]).unwrap(); @@ -921,7 +925,7 @@ mod tests { #[test] fn broken_descriptor_reference_does_not_partially_recover() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); + let store = StoredOutputMetadataFile::new(tmp.path()); store.upsert_all(&[sketch_meta(1), exact_meta(2)]).unwrap(); let mut persisted: serde_json::Value = @@ -940,24 +944,21 @@ mod tests { } #[test] - fn legacy_flat_sidecar_is_read_and_migrated_on_write() { + fn legacy_flat_sidecar_is_rejected_without_rewriting_it() { let tmp = TempDir::new().unwrap(); - let store = SidMetadataStore::new(tmp.path()); - let legacy = HashMap::from([("1".to_string(), sketch_meta(1))]); - std::fs::write(store.path(), serde_json::to_vec(&legacy).unwrap()).unwrap(); - - assert_eq!(store.load().unwrap(), vec![sketch_meta(1)]); - store.upsert_all(&[exact_meta(2)]).unwrap(); - let persisted: serde_json::Value = - serde_json::from_slice(&std::fs::read(store.path()).unwrap()).unwrap(); - assert_eq!(persisted["schema_version"], 3); - assert_eq!(store.load().unwrap().len(), 2); + let store = StoredOutputMetadataFile::new(tmp.path()); + let legacy = + serde_json::to_vec(&HashMap::from([("1".to_string(), sketch_meta(1))])).unwrap(); + std::fs::write(store.path(), &legacy).unwrap(); + assert!(store.load().is_err()); + assert!(store.upsert_all(&[exact_meta(2)]).is_err()); + assert_eq!(std::fs::read(store.path()).unwrap(), legacy); } #[test] fn upsert_merges_and_overwrites_per_sid() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); s.upsert_all(&[sketch_meta(1)]).unwrap(); // New sid + updated metric for sid 1. let mut updated = sketch_meta(1); @@ -965,7 +966,7 @@ mod tests { s.upsert_all(&[updated.clone(), exact_meta(2)]).unwrap(); let mut got = s.load().unwrap(); - got.sort_by_key(|r| r.sid); + got.sort_by_key(|r| r.storage_handle); assert_eq!(got.len(), 2); assert_eq!(got[0].metric_name, "http_latency_v2"); assert_eq!(got[1], exact_meta(2)); @@ -991,7 +992,7 @@ mod tests { #[test] fn unparsable_file_loads_as_empty() { let tmp = TempDir::new().unwrap(); - let s = SidMetadataStore::new(tmp.path()); + let s = StoredOutputMetadataFile::new(tmp.path()); std::fs::write(s.path(), b"{not json").unwrap(); assert!(s.load().unwrap().is_empty()); } diff --git a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs index 552fe4e0..9627f08d 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/mod.rs @@ -32,7 +32,7 @@ pub mod recovery; pub use config::SketchStorePersistenceConfig; pub use manifest::{Manifest, PartEntry}; -pub use metadata::{SidMetaRecord, SidMetadataStore}; +pub use metadata::{StoredOutputMetadataFile, StoredOutputMetadataRecord}; pub use part::{PartId, PartReader, PartWriter, SnapshotEntry}; pub use source::{EpochSource, SealedEpochRef}; diff --git a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs index a2140260..ebf09525 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/recovery.rs @@ -34,7 +34,7 @@ pub fn recover(disk_path: &Path) -> PersistResult<(Manifest, RecoveryReport)> { // Validate every live part by reading its meta.bin header. let mut to_drop: Vec = Vec::new(); - let pending: HashSet = super::metadata::SidMetadataStore::new(disk_path) + let pending: HashSet = super::metadata::StoredOutputMetadataFile::new(disk_path) .load_strict()? .into_iter() .filter_map(|record| record.pending_immutable.map(|pending| pending.part_id)) diff --git a/data_plane/src/storage_engines/sketch_db/persistence/source.rs b/data_plane/src/storage_engines/sketch_db/persistence/source.rs index 661b6a92..6a384fb0 100644 --- a/data_plane/src/storage_engines/sketch_db/persistence/source.rs +++ b/data_plane/src/storage_engines/sketch_db/persistence/source.rs @@ -142,7 +142,10 @@ pub trait EpochSource: Send + Sync { /// Returns `None` when the sid is unknown to the source (e.g. a test /// fake, or a sid whose instance metadata was evicted). Default impl /// returns `None` so existing/test sources need not implement it. - fn instance_metadata_for_persist(&self, _sid: u64) -> Option { + fn instance_metadata_for_persist( + &self, + _sid: u64, + ) -> Option { None } } diff --git a/data_plane/src/storage_engines/sketch_db/query/timeline.rs b/data_plane/src/storage_engines/sketch_db/query/timeline.rs index 92a64b0b..0c8c4d03 100644 --- a/data_plane/src/storage_engines/sketch_db/query/timeline.rs +++ b/data_plane/src/storage_engines/sketch_db/query/timeline.rs @@ -299,7 +299,7 @@ mod tests { expires: Option, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.into(), group_by_keys: BTreeSet::new(), capability: Some(Capability::QuantileApprox(Some(SketchAlgorithm::DDSketch))), diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index ffefbd25..99f023a7 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -75,12 +75,12 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { }) } -/// Runtime foreign-key binding from one SeriesId to shared descriptors. Every pane -/// row stored under the SeriesId is a Summary Instance: `(binding, interval, -/// group-values, state)`. Descriptor references are normalized here instead of -/// copied into every pane row. +/// Normalized stored-output binding shared by its population/window records. +/// The selected output and catalog generation are durable identity; metadata's +/// numeric handle only locates the physical rows in this store. #[derive(Debug, Clone)] pub struct SdsBinding { + pub stored_output_reference: Option, pub metadata: Arc, pub summary_descriptor: Arc, pub data_descriptor: Arc, @@ -108,14 +108,40 @@ pub struct SummaryDescriptorRegistry { Option<( Arc, Arc, + std::collections::BTreeMap< + asap_types::sds::SummaryDefinitionId, + asap_types::sds::StoredOutputReference, + >, )>, >, } impl SummaryDescriptorRegistry { + #[cfg(test)] pub fn install_catalog( &self, catalog: Arc, + ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { + let outputs = catalog + .definitions + .keys() + .map(|id| { + ( + *id, + asap_types::sds::StoredOutputReference::for_definition(*id), + ) + }) + .collect(); + self.install_catalog_with_outputs(catalog, outputs) + } + + pub fn install_catalog_with_outputs( + &self, + catalog: Arc, + outputs: std::collections::BTreeMap< + asap_types::sds::SummaryDefinitionId, + asap_types::sds::StoredOutputReference, + >, ) -> Result<(), asap_types::summary_catalog::SummaryCatalogError> { catalog.validate()?; let reference = catalog.reference()?; @@ -125,7 +151,7 @@ impl SummaryDescriptorRegistry { plan_version: reference.plan_version, snapshot_sha256: reference.snapshot_sha256, }); - *self.authoritative_catalog.write().unwrap() = Some((catalog, generation)); + *self.authoritative_catalog.write().unwrap() = Some((catalog, generation, outputs)); Ok(()) } @@ -136,7 +162,7 @@ impl SummaryDescriptorRegistry { .read() .unwrap() .as_ref() - .map(|(catalog, _)| Arc::clone(catalog)) + .map(|(catalog, _, _)| Arc::clone(catalog)) } pub fn authoritative_snapshot( @@ -145,12 +171,38 @@ impl SummaryDescriptorRegistry { Arc, Arc, )> { - self.authoritative_catalog.read().unwrap().clone() + self.authoritative_catalog + .read() + .unwrap() + .as_ref() + .map(|(catalog, generation, _)| (Arc::clone(catalog), Arc::clone(generation))) + } + + pub fn stored_output_reference( + &self, + definition: asap_types::sds::SummaryDefinitionId, + ) -> Option { + let installed = self.authoritative_catalog.read().unwrap(); + match installed.as_ref() { + Some((_, _, outputs)) => outputs.get(&definition).copied(), + None => (!definition.fingerprint().is_unset()) + .then(|| asap_types::sds::StoredOutputReference::for_definition(definition)), + } } pub fn bind(&self, metadata: SummarySeriesMetadata) -> Result { - let authoritative = self.authoritative_snapshot(); - let configured = if let Some((catalog, _)) = authoritative.as_ref() { + let authoritative = self.authoritative_catalog.read().unwrap().clone(); + let stored_output_reference = match &authoritative { + Some((_, _, outputs)) => Some( + *outputs + .get(&metadata.policy_fp.into()) + .ok_or("definition has no selected stored output")?, + ), + None => (!metadata.policy_fp.is_unset()).then(|| { + asap_types::sds::StoredOutputReference::for_definition(metadata.policy_fp.into()) + }), + }; + let configured = if let Some((catalog, _, _)) = authoritative.as_ref() { if metadata.policy_fp.is_unset() { return Err( "materialization identity is required by the installed SummaryCatalog".into(), @@ -206,10 +258,11 @@ impl SummaryDescriptorRegistry { }; Ok(SdsBinding { + stored_output_reference, metadata: Arc::new(metadata), summary_descriptor, data_descriptor, - catalog_generation: authoritative.map(|(_, generation)| generation), + catalog_generation: authoritative.map(|(_, generation, _)| generation), }) } @@ -300,7 +353,7 @@ mod tests { policy: u64, ) -> SummarySeriesMetadata { SummarySeriesMetadata { - sid, + storage_handle: sid, metric_name: metric.into(), group_by_keys: BTreeSet::from(["job".into()]), capability: None, diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index b81cc6a9..87a4cd2f 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -1,86 +1,15 @@ -//! Hot-reloadable `StreamingConfig` state. +//! Generation-consistent physical plan installation and execution views. //! -//! Wraps a shared `StreamingConfig` in `arc_swap::ArcSwap` so an -//! external control plane can push a new config at runtime via -//! `POST /api/v1/streaming-config` without restarting the query -//! engine binary. -//! -//! ## How the pieces see the swap -//! -//! Runtime bootstrap readers share clones of the same `StreamingConfigHandle` -//! handle (internally `Arc>`), so they -//! observe the swap at the same instant: -//! -//! * **Writes** — atomic via `ArcSwap::store`. Lock-free; readers that -//! hold a stale snapshot finish their work with the old config and -//! drop it when the last reference goes out of scope. -//! * **IngestState** — re-snapshots per ingest batch -//! (`config_snapshot()`). New aggregations start receiving data on -//! the next batch. -//! * **Precompute workers** — read the handle directly in -//! `get_or_create_group_state()`. No message passing, no polling; -//! new agg_ids are visible the moment a worker tries to create a -//! `GroupState` for them. -//! -//! Query execution obtains its generation-consistent runtime configuration, -//! query plan, and catalog from `RuntimePhysicalPlan` instead of this handle. -//! -//! ## Config-upgrade contract for the control plane -//! -//! The recommended way for a control plane to upgrade a metric's sketch -//! parameters (or aggregation type) is **monotonic, non-reused -//! `aggregation_id`s plus time-based retention**: -//! -//! 1. Control plane decides to upgrade, e.g. `CMS(width=256)` → -//! `CMS(width=1024)` for `test_metric`. -//! 2. Control plane allocates a **new** `aggregation_id` (never reused), -//! e.g. the old id was 1, the new id is 17. -//! 3. Control plane POSTs a new `StreamingConfig` where the old id is -//! **removed** and the new id is **added**: -//! - before: `{1: CMS(width=256)}` -//! - after: `{17: CMS(width=1024)}` -//! 4. What happens on the backend, with zero additional code: -//! - `IngestState` stops routing data to agg_id 1 and starts -//! routing to agg_id 17 (metric-name match unchanged). -//! - Workers evict the now-orphaned `GroupState` entries for -//! agg_id 1 after their last windows drain -//! (`evict_orphaned_groups`). -//! - New `GroupState` entries for agg_id 17 are created on -//! demand, with the new `CMS(width=1024)` parameters. -//! - Store entries under agg_id 1 are **not deleted** on the -//! config swap; they persist until `persistence_delete_older_than_secs` -//! retention elapses, at which point the persistence layer's -//! time-based TTL sweep drops the corresponding parts. -//! 5. Query semantics during the transition: -//! - Before the swap: `ASAPQueryEngine` matches against agg_id 1. -//! - After the swap: `ASAPQueryEngine` matches against agg_id 17. -//! Historical data in the store under agg_id 1 is not joined -//! into the answer; the new sketch warms up from zero. -//! - Callers that need query continuity across parameter changes -//! should implement an overlap period at the control plane (keep -//! both ids in the config long enough for the new id to accrue -//! enough history) — this is a control-plane-side concern, not a -//! backend one. -//! -//! ## What the contract requires from the control plane -//! -//! * Assign `aggregation_id`s from a monotonically-increasing counter. -//! * Never reuse an `aggregation_id` after it has been removed from -//! a `StreamingConfig` push. -//! * Rely on the backend's `persistence_delete_older_than_secs` for -//! store cleanup — do not try to explicitly delete old agg_id data. -//! -//! Violating "never reuse" is safe in terms of correctness (the -//! backend creates a fresh `GroupState` either way), but it can -//! produce confusing store states where data under the same agg_id -//! spans multiple parameter generations. +//! Production consumers obtain the installed precompute program through the +//! active physical plan. Plan publication installs query and precompute bindings +//! together; a runtime lookup view cannot independently redefine computation. use std::collections::BTreeMap; use std::sync::Arc; use arc_swap::ArcSwap; -use crate::storage_engines::types::StreamingConfig; +use crate::storage_engines::types::InstalledPrecomputePlan; /// One immutable, generation-consistent runtime snapshot. Every execution /// subsystem must project its view from the same `Arc`. @@ -93,7 +22,7 @@ pub struct RuntimePhysicalPlan { pub summary_catalog: Option>, pub precompute_plan: asap_types::precompute_plan::PrecomputePlan, pub transmission_plan: asap_types::producer_plan::TransmissionPlan, - pub streaming_config: Arc, + pub installed_precompute_plan: Arc, pub query_plan: Arc, pub storage_routing: Arc, } @@ -568,74 +497,69 @@ impl std::fmt::Debug for ActivePhysicalPlanHandle { } } -/// Streaming materialization view with two compatibility modes. Legacy mode -/// owns a swappable config; active-plan mode reads the config from the current -/// immutable runtime plan. In active-plan mode, `swap` only updates the legacy -/// backing slot and does not publish a new plan. Use plan activation to change -/// the authoritative configuration. +/// Precompute projection of the authoritative active physical plan. #[derive(Clone)] -pub struct StreamingConfigHandle { - inner: Arc>, - active: Option, +pub struct InstalledPrecomputePlanHandle { + source: InstalledPrecomputePlanSource, } -impl StreamingConfigHandle { - /// Construct with an initial `StreamingConfig`. Takes ownership — - /// callers who need to keep their own handle should `.clone()` the - /// `StreamingConfig` before calling `new`. - pub fn new(initial: StreamingConfig) -> Self { - Self { - inner: Arc::new(ArcSwap::new(Arc::new(initial))), - active: None, - } +#[derive(Clone)] +enum InstalledPrecomputePlanSource { + Active(ActivePhysicalPlanHandle), + #[cfg(test)] + Fixture(Arc>), +} + +impl InstalledPrecomputePlanHandle { + #[cfg(test)] + pub fn new(initial: InstalledPrecomputePlan) -> Self { + Self::from_arc(Arc::new(initial)) } - /// Construct from a pre-built `Arc` — useful - /// when the caller already has the config behind an `Arc` and - /// wants to avoid a redundant clone. - pub fn from_arc(initial: Arc) -> Self { + #[cfg(test)] + pub fn from_arc(initial: Arc) -> Self { Self { - inner: Arc::new(ArcSwap::new(initial)), - active: None, + source: InstalledPrecomputePlanSource::Fixture(Arc::new(ArcSwap::new(initial))), } } pub fn from_active_physical_plan(active: ActivePhysicalPlanHandle) -> Self { - let initial = active.active_snapshot().streaming_config.clone(); Self { - inner: Arc::new(ArcSwap::new(initial)), - active: Some(active), + source: InstalledPrecomputePlanSource::Active(active), } } - /// Return a cheap, cloneable snapshot of the current config. The - /// returned `Arc` is stable for the caller's lifetime — a - /// concurrent swap produces a new `Arc` and leaves this one alone. - pub fn snapshot(&self) -> Arc { - self.active - .as_ref() - .map(|a| a.active_snapshot().streaming_config.clone()) - .unwrap_or_else(|| self.inner.load_full()) + pub fn snapshot(&self) -> Arc { + match &self.source { + InstalledPrecomputePlanSource::Active(active) => { + active.active_snapshot().installed_precompute_plan.clone() + } + #[cfg(test)] + InstalledPrecomputePlanSource::Fixture(view) => view.load_full(), + } } pub fn active_physical_plan_snapshot(&self) -> Option> { - self.active.as_ref().map(|active| active.active_snapshot()) + match &self.source { + InstalledPrecomputePlanSource::Active(active) => Some(active.active_snapshot()), + #[cfg(test)] + InstalledPrecomputePlanSource::Fixture(_) => None, + } } - /// Atomically replace the current config. The previous `Arc` is - /// dropped when the last reader holding it goes out of scope. - /// Returns the `Arc` that was just replaced, for callers that - /// want to diff old vs new (e.g. to log agg_ids that were added - /// or removed). - pub fn swap(&self, new: StreamingConfig) -> Arc { - self.inner.swap(Arc::new(new)) + #[cfg(test)] + pub fn swap(&self, new: InstalledPrecomputePlan) -> Arc { + match &self.source { + InstalledPrecomputePlanSource::Fixture(view) => view.swap(Arc::new(new)), + InstalledPrecomputePlanSource::Active(_) => panic!("activate a complete physical plan"), + } } } -impl std::fmt::Debug for StreamingConfigHandle { +impl std::fmt::Debug for InstalledPrecomputePlanHandle { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let snap = self.snapshot(); - f.debug_struct("StreamingConfigHandle") + f.debug_struct("InstalledPrecomputePlanHandle") .field( "num_agg_configs", &snap.materializations_by_policy_fingerprint.len(), @@ -650,10 +574,7 @@ pub use ActivePhysicalPlanHandle as HotReloadActivePhysicalPlan; #[deprecated(note = "Use RuntimePhysicalPlan")] pub use RuntimePhysicalPlan as ActivePhysicalPlan; -#[deprecated(note = "Use StreamingConfigHandle")] -pub use StreamingConfigHandle as HotReloadStreamingConfig; - -impl StreamingConfigHandle { +impl InstalledPrecomputePlanHandle { #[deprecated(note = "Use from_active_physical_plan")] pub fn from_active(active: ActivePhysicalPlanHandle) -> Self { Self::from_active_physical_plan(active) @@ -743,7 +664,7 @@ mod tests { }, rules: Vec::new(), }, - streaming_config: Arc::new(StreamingConfig::new(HashMap::new())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(HashMap::new())), query_plan: Arc::new(asap_types::query_plan::QueryPlan { plan_id, plan_version, @@ -775,12 +696,12 @@ mod tests { ) } - /// Build a StreamingConfig from a list of marker `id`s. After PR 5 + /// Build a InstalledPrecomputePlan from a list of marker `id`s. After PR 5 /// the map key IS the policy fingerprint, derived from /// `metric_{id}`. We return both the config and the /// dummy-id→fingerprint mapping so the assertions below can look /// up entries. - fn cfg_with_ids(ids: &[u64]) -> (StreamingConfig, std::collections::HashMap) { + fn cfg_with_ids(ids: &[u64]) -> (InstalledPrecomputePlan, std::collections::HashMap) { let mut map = HashMap::new(); let mut id_to_fp = std::collections::HashMap::new(); for &id in ids { @@ -789,13 +710,13 @@ mod tests { id_to_fp.insert(id, fp); map.insert(fp, cfg); } - (StreamingConfig::new(map), id_to_fp) + (InstalledPrecomputePlan::new(map), id_to_fp) } #[test] fn snapshot_reflects_initial_config() { let (cfg, id_to_fp) = cfg_with_ids(&[1, 2, 3]); - let hr = StreamingConfigHandle::new(cfg); + let hr = InstalledPrecomputePlanHandle::new(cfg); let snap = hr.snapshot(); assert_eq!(snap.materializations_by_policy_fingerprint.len(), 3); assert!(snap @@ -807,7 +728,7 @@ mod tests { fn swap_replaces_config_atomically() { let (cfg1, id_to_fp1) = cfg_with_ids(&[1, 2]); let (cfg2, id_to_fp2) = cfg_with_ids(&[3, 4, 5]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let old = hr.swap(cfg2); // Old snapshot still reflects pre-swap contents. assert_eq!(old.materializations_by_policy_fingerprint.len(), 2); @@ -829,7 +750,7 @@ mod tests { fn clones_share_underlying_swap() { let (cfg1, _) = cfg_with_ids(&[1]); let (cfg2, id_to_fp2) = cfg_with_ids(&[2, 3]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let hr_clone = hr.clone(); hr.swap(cfg2); // The clone sees the swap because both handles share the @@ -844,7 +765,7 @@ mod tests { #[test] fn concurrent_readers_see_consistent_snapshot() { let (cfg1, _) = cfg_with_ids(&[1, 2]); - let hr = StreamingConfigHandle::new(cfg1); + let hr = InstalledPrecomputePlanHandle::new(cfg1); let hr_writer = hr.clone(); let writer = thread::spawn(move || { for i in 0..50 { diff --git a/data_plane/src/storage_engines/types/installed_precompute_plan.rs b/data_plane/src/storage_engines/types/installed_precompute_plan.rs new file mode 100644 index 00000000..fe08d288 --- /dev/null +++ b/data_plane/src/storage_engines/types/installed_precompute_plan.rs @@ -0,0 +1,164 @@ +//! Internal execution indexes derived exclusively from a validated DAG plan. +use anyhow::Result; +use std::collections::HashMap; +use std::ops::Index; + +use super::storage_backend::StorageBackend; +use asap_types::{PolicyRegistry, PrecomputeMaterialization}; + +#[derive(Debug, Clone)] +pub struct InstalledPrecomputePlan { + pub(crate) partitioning: crate::precompute_engine::partitioning::DagPartitioning, + pub(crate) raw_programs: + HashMap>, + pub(crate) precompute_plan: Option, + pub(crate) materializations_by_policy_fingerprint: HashMap, + pub(crate) storage_backend: StorageBackend, +} + +impl InstalledPrecomputePlan { + fn derived_view(materializations: HashMap) -> Self { + Self { + partitioning: Default::default(), + raw_programs: HashMap::new(), + precompute_plan: None, + materializations_by_policy_fingerprint: materializations, + storage_backend: StorageBackend::default(), + } + } + + /// Production construction always validates the executable DAG and bindings. + pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { + let materializations = plan.runtime_materializations()?; + let mut programs = HashMap::new(); + for config in materializations.values().filter(|config| { + config.derived_input.is_none() + && plan.ingest.protocol + == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 + }) { + let program = + crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) + .map_err(anyhow::Error::msg)?; + programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); + } + let mut view = Self::derived_view(materializations); + view.partitioning = + crate::precompute_engine::partitioning::DagPartitioning::from_plan(&plan); + view.precompute_plan = Some(plan); + view.raw_programs = programs; + Ok(view) + } + + // Isolated kernel/storage fixtures can omit a physical installation. This + // constructor is absent from the production library and binary. + #[cfg(test)] + pub fn new(materializations: HashMap) -> Self { + Self::derived_view(materializations) + } + + #[cfg(test)] + pub fn with_storage_backend( + materializations: HashMap, + storage_backend: StorageBackend, + ) -> Self { + let mut view = Self::derived_view(materializations); + view.storage_backend = storage_backend; + view + } + + pub fn stored_output_reference( + &self, + definition: asap_types::sds::SummaryDefinitionId, + ) -> Option { + let selected = self.precompute_plan.as_ref().and_then(|plan| { + plan.schemas + .iter() + .find(|schema| schema.materialization == definition) + .map(|schema| schema.stored_output_reference) + }); + #[cfg(test)] + let selected = selected.or_else(|| { + self.materializations_by_policy_fingerprint + .contains_key(&definition.as_u64()) + .then(|| asap_types::sds::StoredOutputReference::for_definition(definition)) + }); + selected + } + + pub fn plan(&self) -> &asap_types::precompute_plan::PrecomputePlan { + self.precompute_plan + .as_ref() + .expect("installed plan has an authoritative source") + } + + pub fn storage_backend(&self) -> StorageBackend { + self.storage_backend + } + + pub fn get_aggregation_config(&self, fingerprint: u64) -> Option<&PrecomputeMaterialization> { + self.materializations_by_policy_fingerprint + .get(&fingerprint) + } + + pub fn materializations(&self) -> &HashMap { + &self.materializations_by_policy_fingerprint + } + + pub fn contains(&self, fingerprint: u64) -> bool { + self.materializations_by_policy_fingerprint + .contains_key(&fingerprint) + } + + pub fn policy_registry(&self) -> PolicyRegistry { + PolicyRegistry::from_configs( + self.materializations_by_policy_fingerprint + .values() + .cloned(), + ) + } +} + +impl Index for InstalledPrecomputePlan { + type Output = PrecomputeMaterialization; + fn index(&self, fingerprint: u64) -> &Self::Output { + &self.materializations_by_policy_fingerprint[&fingerprint] + } +} + +impl Default for InstalledPrecomputePlan { + fn default() -> Self { + use control_plane::physical::compiler::{ + PlanEnvelope, PrecomputePlan, BACKEND_COMPAT, PLANNER_REVISION, + }; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "empty-installation".into(), + }; + Self::from_precompute_plan( + PrecomputePlan::build(envelope, vec![], &[]).expect("valid empty plan"), + ) + .expect("valid empty installation") + } +} + +#[cfg(test)] +mod tests { + // Flat lists cannot enter through the authoritative physical-plan document. + #[test] + fn rejects_flat_aggregation_documents() { + for text in [r#"{"aggregation_configs":{}}"#, "aggregations: []"] { + assert!( + serde_yaml::from_str::( + text + ) + .is_err() + ); + } + } +} diff --git a/data_plane/src/storage_engines/types/mod.rs b/data_plane/src/storage_engines/types/mod.rs index 562d08d7..9956f5b4 100644 --- a/data_plane/src/storage_engines/types/mod.rs +++ b/data_plane/src/storage_engines/types/mod.rs @@ -8,18 +8,18 @@ pub mod enums; pub mod hot_reload_config; +pub mod installed_precompute_plan; pub mod precomputed_output; pub mod storage_backend; -pub mod streaming_config; pub use asap_physical_operators::key_by_label_values::*; pub use asap_physical_operators::measurement::*; pub use asap_physical_operators::traits::*; pub use enums::*; pub use hot_reload_config::*; +pub use installed_precompute_plan::*; pub use precomputed_output::*; pub use storage_backend::*; -pub use streaming_config::*; // Cross-module re-export of asap_types data types so callers can // write `crate::storage_engines::types::PrecomputeMaterialization` instead of diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 28e71b4c..8644f37d 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -44,7 +44,9 @@ pub struct PrecomputedOutput { /// Physical lifetime chosen before execution; never re-resolve a queued /// fragment against a newer logical-series mapping. #[serde(skip)] - pub series_id: Option, + pub storage_handle: Option, + #[serde(skip)] + pub stored_output_reference: Option, #[serde(skip)] pub catalog_generation: Option>, #[serde(skip)] @@ -86,7 +88,8 @@ impl PrecomputedOutput { policy_fp: PolicyFingerprint, ) -> Self { Self { - series_id: None, + storage_handle: None, + stored_output_reference: None, catalog_generation: None, input_revision: None, start_timestamp, @@ -115,7 +118,8 @@ impl PrecomputedOutput { policy_fp: PolicyFingerprint, ) -> Self { Self { - series_id: None, + storage_handle: None, + stored_output_reference: None, catalog_generation: None, input_revision: None, start_timestamp, diff --git a/data_plane/src/storage_engines/types/streaming_config.rs b/data_plane/src/storage_engines/types/streaming_config.rs deleted file mode 100644 index 95055c2e..00000000 --- a/data_plane/src/storage_engines/types/streaming_config.rs +++ /dev/null @@ -1,211 +0,0 @@ -use anyhow::Result; -use serde::{Deserialize, Serialize}; -use serde_yaml::Value; -use std::collections::HashMap; -use std::fs::File; -use std::io::BufReader; -use std::ops::Index; - -use asap_types::{MonitorSpec, PolicyRegistry, PrecomputeMaterialization}; - -use super::storage_backend::StorageBackend; - -/// DAG installation plus a derived in-memory routing index. The flat index is -/// never serialized as executable configuration. Raw programs are validated -/// and shared once per installed producer across all of its population states. -#[derive(Debug, Clone, Serialize)] -pub struct StreamingConfig { - #[serde(skip)] - pub(crate) raw_programs: - HashMap>, - /// Authoritative execution configuration: Planner DAGs and physical bindings. - pub precompute_plan: Option, - #[serde(skip)] - pub materializations_by_policy_fingerprint: HashMap, - /// Phase-5 capability-routing axis: which storage tier serves this - /// per-metric runtime config. The controller pushes this when planning - /// (see `docs/design-gorilla-s3-cold-engine.md` §8); pre-Phase-5 - /// configs decode with `#[serde(default)]` to `SketchStore` so - /// existing deploys keep dispatching to `ASAPQueryEngine`. - #[serde(default)] - pub storage_backend: StorageBackend, - /// Continuous-distributed-monitoring threshold specs the data-plane - /// coordinator should serve. Defaults to empty so existing configs (and the - /// vast majority of deploys, which run no monitors) decode unchanged. - #[serde(default)] - pub monitors: Vec, -} - -// Flat aggregation lists are deliberately not an accepted execution document. -impl<'de> Deserialize<'de> for StreamingConfig { - fn deserialize>(deserializer: D) -> Result { - #[derive(Deserialize)] - #[serde(deny_unknown_fields)] - struct Document { - precompute_plan: asap_types::precompute_plan::PrecomputePlan, - #[serde(default)] - storage_backend: StorageBackend, - #[serde(default)] - monitors: Vec, - } - let doc = Document::deserialize(deserializer)?; - let mut config = - Self::from_precompute_plan(doc.precompute_plan).map_err(serde::de::Error::custom)?; - config.storage_backend = doc.storage_backend; - config.monitors = doc.monitors; - Ok(config) - } -} - -impl StreamingConfig { - pub fn new( - materializations_by_policy_fingerprint: HashMap, - ) -> Self { - Self { - raw_programs: HashMap::new(), - precompute_plan: None, - materializations_by_policy_fingerprint, - storage_backend: StorageBackend::default(), - monitors: Vec::new(), - } - } - - /// Build the routing projection only after validating the DAG installation. - pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { - let materializations = plan.runtime_materializations()?; - let mut programs = HashMap::new(); - for config in materializations.values().filter(|c| { - c.derived_input.is_none() - && plan.ingest.protocol - == asap_types::precompute_plan::IngestProtocol::PrometheusRemoteWriteV1 - }) { - let program = - crate::precompute_engine::raw_dag::RawDagProgram::from_plan(&plan, config) - .map_err(anyhow::Error::msg)?; - programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); - } - let mut view = Self::new(materializations); - view.precompute_plan = Some(plan); - view.raw_programs = programs; - Ok(view) - } - - /// CDM monitor specs the data-plane coordinator should serve (may be empty). - pub fn monitors(&self) -> &[MonitorSpec] { - &self.monitors - } - - /// Phase-5 constructor: build with an explicit storage-backend pin. - /// Used by the controller-driven plan-push path; tests typically - /// stay on `Self::new(...)` and let the default land. - pub fn with_storage_backend( - materializations_by_policy_fingerprint: HashMap, - storage_backend: StorageBackend, - ) -> Self { - Self { - raw_programs: HashMap::new(), - precompute_plan: None, - materializations_by_policy_fingerprint, - storage_backend, - monitors: Vec::new(), - } - } - - /// Read-only access to the storage backend pinned at construction - /// time. The Phase-5 router consults this to pick which engine - /// answers a query. - pub fn storage_backend(&self) -> StorageBackend { - self.storage_backend - } - - pub fn get_aggregation_config( - &self, - aggregation_id: u64, - ) -> Option<&PrecomputeMaterialization> { - self.materializations_by_policy_fingerprint - .get(&aggregation_id) - } - - pub fn materializations(&self) -> &HashMap { - &self.materializations_by_policy_fingerprint - } - - pub fn contains(&self, aggregation_id: u64) -> bool { - self.materializations_by_policy_fingerprint - .contains_key(&aggregation_id) - } - - /// Derived content-addressed view. Builds a [`PolicyRegistry`] keyed - /// on [`asap_types::PolicyFingerprint`] — the merged-sid-identity-chain - /// replacement for the `aggregation_id`-keyed lookup. Cheap (O(N) - /// over `materializations_by_policy_fingerprint.len()`); call at swap time, not per - /// query, if it shows up in hot-path profiles. - /// - /// Dual-keyed transition: this method exists alongside the legacy - /// `get_aggregation_config(aggregation_id)` so callers can migrate - /// one at a time. The two views are derived from the same source — - /// they can never disagree. - pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs( - self.materializations_by_policy_fingerprint - .values() - .cloned(), - ) - } - - pub fn from_yaml_file(yaml_file: &str) -> Result { - let file = File::open(yaml_file)?; - let reader = BufReader::new(file); - let data: Value = serde_yaml::from_reader(reader)?; - - Self::from_yaml_data(&data) - } - - /// Build from the streaming-config YAML alone. Per-aggregation - /// `numAggregatesToRetain` is read directly from each - /// aggregation's YAML entry; the old InferenceConfig indirection - /// (operator-authored query→agg_ids YAML feeding a retention_map) - /// is gone — the controller drives capability matching dynamically. - pub fn from_yaml_data(data: &Value) -> Result { - serde_yaml::from_value(data.clone()).map_err(Into::into) - } -} - -impl Index for StreamingConfig { - type Output = PrecomputeMaterialization; - - fn index(&self, aggregation_id: u64) -> &Self::Output { - &self.materializations_by_policy_fingerprint[&aggregation_id] - } -} - -impl Default for StreamingConfig { - fn default() -> Self { - Self::new(HashMap::new()) - } -} - -impl StreamingConfig { - #[deprecated(note = "Use materializations")] - pub fn get_all_aggregation_configs(&self) -> &HashMap { - self.materializations() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - // Old flat lists cannot become execution authority through JSON or YAML. - #[test] - fn rejects_flat_aggregation_documents() { - for text in [ - r#"{"aggregation_configs":{}}"#, - "aggregations: []", - "aggregations: [{aggregationType: Sum, metric: m}]", - ] { - let yaml = serde_yaml::from_str(text).unwrap(); - assert!(StreamingConfig::from_yaml_data(&yaml).is_err()); - } - } -} diff --git a/data_plane/src/tests/prometheus_forwarding_tests.rs b/data_plane/src/tests/prometheus_forwarding_tests.rs index ff205f4e..bfbe765f 100644 --- a/data_plane/src/tests/prometheus_forwarding_tests.rs +++ b/data_plane/src/tests/prometheus_forwarding_tests.rs @@ -2,7 +2,7 @@ use crate::drivers::query::adapters::AdapterConfig; use crate::drivers::query::servers::http::{HttpServer, HttpServerConfig}; use crate::query_engines::{ASAPQueryEngine, QueryForwardingPolicy}; #[cfg(test)] -use crate::storage_engines::types::{QueryLanguage, StreamingConfig}; +use crate::storage_engines::types::{InstalledPrecomputePlan, QueryLanguage}; use reqwest::Client; use serde_json::Value; use std::sync::{ @@ -110,7 +110,7 @@ async fn setup_test_server(prometheus_port: u16) -> (HttpServer, u16) { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -190,7 +190,7 @@ async fn test_forwarding_disabled() { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -278,7 +278,7 @@ async fn test_prometheus_server_unreachable() { ), }; - let streaming_config = Arc::new(StreamingConfig::default()); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::default()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let idx = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 895df1b4..f842d084 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -9,8 +9,8 @@ use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; use crate::query_engines::query_result::InstantVectorElement; use crate::storage_engines::types::{ - AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, QueryLanguage, - StreamingConfig, WindowKind, + AggregationType, InstalledPrecomputePlan, KeyByLabelValues, PrecomputeMaterialization, + PrecomputedOutput, QueryLanguage, WindowKind, }; use crate::AggregateCore; use asap_types::KeyByLabelNames; @@ -118,12 +118,12 @@ pub fn create_engine_single_pop_with_aggregated( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = @@ -131,10 +131,10 @@ pub fn create_engine_single_pop_with_aggregated( let resolver = std::sync::Arc::new(SeriesIdResolver::new()); // Insert data into SketchStore via the canonical helper (M2.3.6e). - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() - .expect("agg config must be in streaming_config"); + .expect("agg config must be in installed_precompute_plan"); let timestamp = 1_000_000_u64; for (label_values_opt, acc) in data { let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); @@ -236,23 +236,23 @@ pub fn create_engine_dual_input( let keys_id = keys_agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(keys_id, keys_agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = streaming_config + let agg_cfg_1 = installed_precompute_plan .get_aggregation_config(value_id) .cloned() .expect("value agg config"); - let agg_cfg_2 = streaming_config + let agg_cfg_2 = installed_precompute_plan .get_aggregation_config(keys_id) .cloned() .expect("keys agg config"); @@ -362,22 +362,22 @@ pub fn create_engine_two_metrics( let id_b = agg_config_b.policy_fp_u64(); materializations_by_policy_fingerprint.insert(id_b, agg_config_b); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = streaming_config + let agg_cfg_1 = installed_precompute_plan .get_aggregation_config(id_a) .cloned() .expect("agg a"); - let agg_cfg_2 = streaming_config + let agg_cfg_2 = installed_precompute_plan .get_aggregation_config(id_b) .cloned() .expect("agg b"); @@ -471,12 +471,12 @@ pub fn create_engine_three_metrics( materializations_by_policy_fingerprint.insert(id, cfg); } - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = @@ -485,7 +485,7 @@ pub fn create_engine_three_metrics( let agg_cfgs: Vec<_> = ids .iter() .map(|id| { - streaming_config + installed_precompute_plan .get_aggregation_config(*id) .cloned() .expect("agg present") @@ -553,18 +553,18 @@ pub fn create_engine_multi_timestamp( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() .expect("agg"); @@ -629,18 +629,18 @@ pub fn create_engine_multi_timestamp_with_window( let agg_id = agg_config.policy_fp_u64(); materializations_by_policy_fingerprint.insert(agg_id, agg_config); - let streaming_config = Arc::new(StreamingConfig { + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { + partitioning: Default::default(), raw_programs: Default::default(), precompute_plan: None, materializations_by_policy_fingerprint, storage_backend: Default::default(), - monitors: Vec::new(), }); let summary_store = std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = streaming_config + let agg_cfg = installed_precompute_plan .get_aggregation_config(agg_id) .cloned() .expect("agg"); diff --git a/data_plane/src/utils/file_io.rs b/data_plane/src/utils/file_io.rs deleted file mode 100644 index 6f33ca87..00000000 --- a/data_plane/src/utils/file_io.rs +++ /dev/null @@ -1,52 +0,0 @@ -use crate::storage_engines::types::StreamingConfig; -use anyhow::{Context, Result}; - -pub fn read_streaming_config(yaml_file: &str) -> Result { - let yaml_data = std::fs::read_to_string(yaml_file) - .with_context(|| format!("Failed to read YAML file: {yaml_file}"))?; - let yaml_data: serde_yaml::Value = serde_yaml::from_str(&yaml_data) - .with_context(|| format!("Failed to parse YAML file: {yaml_file}"))?; - - let config = StreamingConfig::from_yaml_data(&yaml_data) - .with_context(|| format!("Failed to parse YAML config from: {yaml_file}"))?; - - Ok(config) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Write; - use tempfile::NamedTempFile; - - #[test] - fn flat_streaming_file_is_rejected() { - // PR 5: `aggregationId: 1` is silently dropped on read — the - // streaming-config map key is the policy fingerprint derived - // from content. The legacy field stays in this fixture to - // exercise the "ignored cleanly" path. - let streaming_yaml_content = r#" -aggregations: -- aggregationId: 1 - aggregationSubType: '' - aggregationType: DatasketchesKLL - labels: - aggregated: [] - grouping: - - instance - - job - rollup: [] - metric: fake_metric_total - parameters: - K: 200 - spatialFilter: '' - windowSize: 10 - numAggregatesToRetain: 6 -"#; - - let mut streaming_temp_file = NamedTempFile::new().unwrap(); - write!(streaming_temp_file, "{streaming_yaml_content}").unwrap(); - - assert!(read_streaming_config(streaming_temp_file.path().to_str().unwrap()).is_err()); - } -} diff --git a/data_plane/src/utils/mod.rs b/data_plane/src/utils/mod.rs index 5d620636..18e84ff2 100644 --- a/data_plane/src/utils/mod.rs +++ b/data_plane/src/utils/mod.rs @@ -1,5 +1,3 @@ -pub mod file_io; pub mod http; -pub use file_io::*; pub use http::*; diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index cc2d34b0..4340e615 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -6,7 +6,6 @@ //! answers are independently computed from those raw fixtures. use std::collections::{HashMap, HashSet}; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; @@ -60,7 +59,6 @@ struct Backend { client: reqwest::Client, query_base: String, otlp_url: String, - _config: tempfile::NamedTempFile, _output_dir: tempfile::TempDir, } @@ -109,22 +107,14 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(config_yaml: &str) -> Backend { +async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create data-plane output directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - config - .write_all(config_yaml.as_bytes()) - .expect("write streaming config"); - config.flush().expect("flush streaming config"); - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_str(config_yaml).unwrap(), - ) - .unwrap(); let mut physical = tempfile::NamedTempFile::new().unwrap(); - let mut install = physical_fixture::artifact(&runtime); + let mut install = + physical_fixture::artifact_from_materializations(vec![materialization.clone()]); for rule in &mut install.transmission_plan.rules { if matches!( install @@ -147,8 +137,6 @@ async fn start_backend(config_yaml: &str) -> Backend { let mut command = Command::new(env!("CARGO_BIN_EXE_data_plane")); command - .arg("--streaming-config") - .arg(config.path()) .arg("--physical-plan") .arg(physical.path()) .arg("--http-port") @@ -188,7 +176,6 @@ async fn start_backend(config_yaml: &str) -> Backend { client, query_base, otlp_url: format!("http://127.0.0.1:{otlp_http_port}/v1/metrics"), - _config: config, _output_dir: output_dir, }; } @@ -294,9 +281,11 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { .collect() } -fn config(metric: &str, kind: &str, parameters: &str) -> String { - format!( - "aggregations:\n - aggregationType: {kind}\n aggregationSubType: ''\n labels:\n grouping: [service]\n rollup: []\n aggregated: []\n metric: {metric}\n parameters:\n{parameters}\n windowSize: 1\n windowType: tumbling\n spatialFilter: ''\n" +fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { + physical_fixture::materialization( + metric, + kind.parse().unwrap(), + serde_yaml::from_str(parameters).unwrap(), ) } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 7d887235..fd8387d1 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -399,18 +399,10 @@ async fn certified_kll_state_to_query_oracle() { let port = unused_port(); let otlp_port = unused_port(); let grpc_port = unused_port(); - let mut bootstrap_config = tempfile::NamedTempFile::new().unwrap(); - serde_json::to_writer( - &mut bootstrap_config, - &serde_json::json!({"aggregations": []}), - ) - .unwrap(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .args(["--physical-plan"]) .arg(artifact_file.path()) - .arg("--streaming-config") - .arg(bootstrap_config.path()) .args(["--http-port", &port.to_string(), "--output-dir"]) .arg(output.path()) .args([ diff --git a/data_plane/tests/backend_process_e2e.rs b/data_plane/tests/backend_process_e2e.rs index 9c1082c5..f6f323f5 100644 --- a/data_plane/tests/backend_process_e2e.rs +++ b/data_plane/tests/backend_process_e2e.rs @@ -5,7 +5,9 @@ //! data plane, sends a modified-OTLP DDSketch, and verifies the resulting //! PromQL value. No server or planner is constructed in the test process. -use std::io::Write; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -374,10 +376,10 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { let output_dir = tempfile::tempdir().expect("create data-plane output directory"); let mut bootstrap = tempfile::NamedTempFile::new().expect("create bootstrap config"); - writeln!(bootstrap, "aggregations: []").expect("write bootstrap config"); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let data_child = Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") .arg(bootstrap.path()) .arg("--http-port") .arg(port(&data_api).to_string()) @@ -416,7 +418,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .env("CONTROLLER_GRPC_ADDR", &control_grpc) .env( "CONTROLLER_BACKEND_ENDPOINT", - format!("{data_base}/api/v1/streaming-config"), + format!("{data_base}/api/v1/physical-plan"), ) .env( "CONTROLLER_WORKLOADS", @@ -536,7 +538,7 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { assert_eq!(publication["collector_ids"][0], "whole-e2e-collector"); let active: serde_json::Value = client - .get(format!("{data_base}/api/v1/streaming-config")) + .get(format!("{data_base}/api/v1/physical-plan/status")) .send() .await .expect("read installed streaming config") @@ -544,12 +546,13 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .expect("decode installed streaming config"); assert_eq!( - active["aggregation_count"], 1, + active["materializations"].as_array().unwrap().len(), + 1, "physical plan was not installed: {active}" ); - let installed_aggregation = active["streaming_config"]["aggregation_configs"] - .as_object() - .and_then(|configs| configs.values().next()) + let installed_aggregation = active["precompute_plan"]["materializations"] + .as_array() + .and_then(|configs| configs.first()) .expect("installed aggregation details"); let planned_alpha = installed_aggregation["parameters"]["alpha"] .as_f64() @@ -723,8 +726,9 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .await .unwrap(); assert_eq!(first_scalar(&still_warm), Some(value)); - // Retry the staged successor while queries are in flight. Each - // request must retain a complete active snapshot through cutover. + // Retry while queries are in flight. Old-generation reads may finish, + // but the successor must stay cold until its own output is published. + // Reusing the old payload would violate StoredOutputReference identity. request["activation_unix_ms"] = (std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() @@ -747,11 +751,12 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { .json() .await .unwrap(); - assert_eq!( - first_scalar(&response), - Some(value), - "torn serving snapshot: {response}" - ); + if let Some(actual) = first_scalar(&response) { + assert_eq!(actual, value, "incorrect old-generation result: {response}"); + } else { + assert_eq!(response["status"], "error", "{response}"); + assert_eq!(response["error"], "No result for query", "{response}"); + } tokio::time::sleep(Duration::from_millis(10)).await; } }); @@ -774,6 +779,27 @@ async fn production_control_plane_to_data_plane_otlp_to_promql() { assert_eq!(activated["plan_version"], 2); let (successor, _collector_socket) = collector.await.unwrap(); readers.await.unwrap(); + let cold_successor: serde_json::Value = client + .get(format!("{data_base}/api/v1/query")) + .query(&[ + ("query", query.to_string()), + ("time", (window_end_ms as f64 / 1000.0).to_string()), + ]) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert_eq!(cold_successor["status"], "error", "{cold_successor}"); + assert_eq!( + cold_successor["error"], "No result for query", + "{cold_successor}" + ); + assert!( + first_scalar(&cold_successor).is_none(), + "successor reused old state" + ); let old_frame = client .post(format!("http://{otlp_http}/v1/metrics")) .header("content-type", "application/x-protobuf") diff --git a/data_plane/tests/clickhouse_differential_e2e.rs b/data_plane/tests/clickhouse_differential_e2e.rs index 64ad1e5e..b9a6523d 100644 --- a/data_plane/tests/clickhouse_differential_e2e.rs +++ b/data_plane/tests/clickhouse_differential_e2e.rs @@ -2,9 +2,11 @@ //! //! Set `CLICKHOUSE_URL` (for example `http://127.0.0.1:8123`) to run it. +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use std::{ collections::HashMap, - io::Write, net::TcpListener, process::{Child, Command, Stdio}, sync::Arc, @@ -63,7 +65,7 @@ async fn spawn_backend( let client = reqwest::Client::new(); let mut command = Command::new(env!("CARGO_BIN_EXE_data_plane")); command - .arg("--streaming-config") + .arg("--physical-plan") .arg(bootstrap) .arg("--http-port") .arg(api_port.to_string()) @@ -345,7 +347,7 @@ async fn run_mixed_aggregate(aggregate: &str) { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -540,7 +542,7 @@ async fn moving_windows_match_clickhouse_after_automatic_publication() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), @@ -724,7 +726,7 @@ async fn collection_sql_executes_local_elements_after_typed_exact_leaf() { let sql_port = unused_port(); let output = tempfile::tempdir().unwrap(); let mut bootstrap = tempfile::NamedTempFile::new().unwrap(); - writeln!(bootstrap, "aggregations: []").unwrap(); + serde_json::to_writer(&mut bootstrap, &empty_physical_plan::empty()).unwrap(); let _process = spawn_backend( &clickhouse_url, user.as_deref(), diff --git a/data_plane/tests/clickhouse_q05_process_e2e.rs b/data_plane/tests/clickhouse_q05_process_e2e.rs index f358dea8..57d0fa9d 100644 --- a/data_plane/tests/clickhouse_q05_process_e2e.rs +++ b/data_plane/tests/clickhouse_q05_process_e2e.rs @@ -1,6 +1,9 @@ //! Optional OS-process E2E for a q05-class bounded SQL max query. //! Run with `CLICKHOUSE_URL=http://127.0.0.1:8123 cargo test -p data_plane --test clickhouse_q05_process_e2e`. +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use control_plane::physical::compiler::{PlanEnvelope, BACKEND_COMPAT, PLANNER_REVISION}; use planner_types::pre_asap::{Column, DataType, Schema}; use std::io::Read; @@ -196,18 +199,17 @@ async fn q05_sql_is_planned_backfilled_and_served_warm_by_backend_process() { while sql_port == http_port { sql_port = free_port(); } - let streaming_config = format!( - "{}/examples/promql/streaming_config.yaml", - env!("CARGO_MANIFEST_DIR") - ); + let mut physical = tempfile::NamedTempFile::new().unwrap(); + serde_json::to_writer(&mut physical, &empty_physical_plan::empty()).unwrap(); + let physical_path = physical.path().to_str().unwrap(); let output_dir = tempfile::tempdir().unwrap(); let output_dir_arg = output_dir.path().to_str().unwrap().to_owned(); let mut command = std::process::Command::new(env!("CARGO_BIN_EXE_data_plane")); command.args([ "--http-port", &http_port.to_string(), - "--streaming-config", - &streaming_config, + "--physical-plan", + physical_path, "--clickhouse-http-port", &sql_port.to_string(), "--clickhouse-url", diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 220cdef8..f71398ba 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -7,7 +7,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -124,39 +123,18 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: component_process_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), - ) - .unwrap(); - let install = physical_fixture::artifact(&runtime); + let install = + physical_fixture::artifact_from_materializations(vec![physical_fixture::materialization( + "component_process_e2e_latency_ms", + asap_types::AggregationType::DDSketch, + [("relative_accuracy".into(), serde_json::json!(0.01))].into(), + )]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) - .arg("--streaming-config") - .arg(config.path()) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/disable_query_forwarding_process_e2e.rs b/data_plane/tests/disable_query_forwarding_process_e2e.rs index 7c8c6585..2cc1e9cf 100644 --- a/data_plane/tests/disable_query_forwarding_process_e2e.rs +++ b/data_plane/tests/disable_query_forwarding_process_e2e.rs @@ -1,6 +1,8 @@ //! The production CLI's no-forwarding mode keeps query traffic inside the backend. -use std::io::Write; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::sync::{ @@ -61,12 +63,12 @@ async fn cli_mode_blocks_instant_and_range_forwarding() { }); let mut config = tempfile::NamedTempFile::new().unwrap(); - writeln!(config, "aggregations: []").unwrap(); + serde_json::to_writer(&mut config, &empty_physical_plan::empty()).unwrap(); let output = tempfile::tempdir().unwrap(); let port = unused_port(); let mut child = ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") .arg(config.path()) .args([ "--disable-query-forwarding", diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index d9aaf592..58272c08 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -146,7 +146,7 @@ async fn post_full_config( } use control_plane::types::WorkloadCharacteristics; -use data_plane::storage_engines::types::HotReloadStreamingConfig; +use data_plane::storage_engines::types::InstalledPrecomputePlanHandle; use serde_json::Value as JsonValue; use asap_otel_proto::tonic::collector::metrics::v1::ExportMetricsServiceRequest; @@ -222,7 +222,7 @@ fn _wc_anchor() -> WorkloadCharacteristics { /// Full test stack: PrecomputeEngine + SketchStoreSink + OtlpReceiver + /// HttpServer, all sharing the same `SketchStore` and -/// `HotReloadStreamingConfig` so a controller-posted streaming-config +/// `InstalledPrecomputePlanHandle` so a controller-posted streaming-config /// is visible to the engine's accumulator routing, the engine's window /// outputs land in `SketchStore`, and the query engine reads from the /// same store. @@ -254,7 +254,7 @@ async fn start_full_stack(otlp_http_port: u16, otlp_grpc_port: u16) -> FullStack let active = data_plane::storage_engines::types::HotReloadActivePhysicalPlan::new( physical_fixture::bootstrap(), ); - let hot_reload = HotReloadStreamingConfig::from_active_physical_plan(active.clone()); + let hot_reload = InstalledPrecomputePlanHandle::from_active_physical_plan(active.clone()); let series_resolver = Arc::new(SeriesIdResolver::new()); // SketchStoreSink writes precompute output back into SketchStore so @@ -711,7 +711,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // * Modified-OTLP `DdSketchDataPoint` wire encoding + the backend's // OTLP HTTP receiver accept the payload (no 4xx/5xx). // * The full stack (PrecomputeEngine + SketchStoreSink + OtlpReceiver -// + HttpServer all sharing SketchStore + HotReloadStreamingConfig) +// + HttpServer all sharing SketchStore + InstalledPrecomputePlanHandle) // comes up and stays up under POST + query traffic. // * The OTLP-ingested sketch lands in `SketchStore` keyed by the // right `PolicyFingerprint` (or via the `instances_matching` diff --git a/data_plane/tests/monitor_process_e2e.rs b/data_plane/tests/monitor_process_e2e.rs index bf7bedbf..920d4428 100644 --- a/data_plane/tests/monitor_process_e2e.rs +++ b/data_plane/tests/monitor_process_e2e.rs @@ -4,7 +4,9 @@ //! data-plane executable, register, report different rates, and receive //! differentiated sampling grants over bidirectional gRPC streams. -use std::io::Write; +#[path = "support/empty_physical_plan.rs"] +mod empty_physical_plan; + use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -110,14 +112,18 @@ async fn production_coordinator_differentiates_edge_sampling_grants() { let monitor_port = unused_port(); let output_dir = tempfile::tempdir().expect("create output directory"); let mut config = tempfile::NamedTempFile::new().expect("create monitor config"); - write!( - config, - "aggregations: []\nmonitors:\n - agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n" + std::io::Write::write_all( + &mut config, + b"- agg_id: 1\n key: ''\n tau: 5000.0\n epsilon: 0.05\n window_ms: 60000\n", ) - .expect("write monitor config"); + .unwrap(); + let mut physical = tempfile::NamedTempFile::new().unwrap(); + serde_json::to_writer(&mut physical, &empty_physical_plan::empty()).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) - .arg("--streaming-config") + .arg("--physical-plan") + .arg(physical.path()) + .arg("--monitor-specs") .arg(config.path()) .arg("--http-port") .arg(query_port.to_string()) diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index b86830b9..a50396b4 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -8,7 +8,6 @@ #[path = "support/physical_fixture.rs"] mod physical_fixture; -use std::io::Write; use std::net::TcpListener; use std::process::{Child, Command, Stdio}; use std::time::Duration; @@ -187,39 +186,18 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); let output_dir = tempfile::tempdir().expect("create log directory"); - let mut config = tempfile::NamedTempFile::new().expect("create streaming config"); - write!( - config, - r#"aggregations: - - aggregationType: DDSketch - aggregationSubType: '' - labels: - grouping: [service] - rollup: [] - aggregated: [] - metric: differential_e2e_latency_ms - parameters: - relative_accuracy: 0.01 - windowSize: 1 - windowType: tumbling - spatialFilter: '' -"# - ) - .expect("write streaming config"); - - let runtime = data_plane::storage_engines::types::StreamingConfig::from_yaml_data( - &serde_yaml::from_slice(&std::fs::read(config.path()).unwrap()).unwrap(), - ) - .unwrap(); - let install = physical_fixture::artifact(&runtime); + let install = + physical_fixture::artifact_from_materializations(vec![physical_fixture::materialization( + "differential_e2e_latency_ms", + asap_types::AggregationType::DDSketch, + [("relative_accuracy".into(), serde_json::json!(0.01))].into(), + )]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); let child = Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(physical.path()) - .arg("--streaming-config") - .arg(config.path()) .arg("--http-port") .arg(query_port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/support/durable_summary_process.rs b/data_plane/tests/support/durable_summary_process.rs index b79aad91..5e8fa58a 100644 --- a/data_plane/tests/support/durable_summary_process.rs +++ b/data_plane/tests/support/durable_summary_process.rs @@ -30,18 +30,46 @@ async fn persisted_summary_restarts_without_live_reregistration() { }; let directory = tempfile::tempdir().unwrap(); let artifact = directory.path().join("plan.json"); - let bootstrap = directory.path().join("bootstrap.json"); let disk = directory.path().join("disk"); std::fs::create_dir_all(&disk).unwrap(); - std::fs::write(&artifact, serde_json::to_vec(&install).unwrap()).unwrap(); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); + let mut document = serde_json::to_value(&install).unwrap(); + let output_ids: std::collections::BTreeMap = install + .precompute_plan + .schemas + .iter() + .enumerate() + .map(|(index, schema)| { + ( + schema.stored_output_reference.stored_output_id.0, + 101 + index as u64, + ) + }) + .collect(); + fn assign_output_ids(value: &mut Value, ids: &std::collections::BTreeMap) { + match value { + Value::Object(fields) => { + if let Some(id) = fields.get_mut("stored_output_id") { + *id = serde_json::json!(ids[&id.as_u64().unwrap()]); + } + for nested in fields.values_mut() { + assign_output_ids(nested, ids); + } + } + Value::Array(values) => { + for nested in values { + assign_output_ids(nested, ids); + } + } + _ => {} + } + } + assign_output_ids(&mut document, &output_ids); + std::fs::write(&artifact, serde_json::to_vec(&document).unwrap()).unwrap(); let spawn = |port: u16| { ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(&artifact) - .arg("--streaming-config") - .arg(&bootstrap) .arg("--http-port") .arg(port.to_string()) .arg("--output-dir") diff --git a/data_plane/tests/support/empty_physical_plan.rs b/data_plane/tests/support/empty_physical_plan.rs new file mode 100644 index 00000000..0e453d85 --- /dev/null +++ b/data_plane/tests/support/empty_physical_plan.rs @@ -0,0 +1,31 @@ +//! Empty authoritative installation for tests that publish a workload later. +pub fn empty() -> asap_types::plan_publication::PhysicalPlanInstallRequest { + use control_plane::physical::compiler::*; + let envelope = PlanEnvelope { + plan_id: 0, + plan_version: 0, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: PLANNER_REVISION.into(), + capability_snapshot_id: "empty-bootstrap".into(), + }; + let catalog = + control_plane::physical::summary_catalog::SummaryCatalog::from_materializations(0, 0, &[]) + .unwrap(); + let mut precompute_plan = PrecomputePlan::build(envelope.clone(), vec![], &[]).unwrap(); + precompute_plan.summary_catalog = Some(catalog.reference().unwrap()); + let mut transmission_plan = + build_transmission_plan(envelope, &precompute_plan, &Default::default()).unwrap(); + transmission_plan.summary_catalog = precompute_plan.summary_catalog.clone(); + asap_types::plan_publication::PhysicalPlanInstallRequest { + summary_catalog: catalog, + precompute_plan, + transmission_plan, + query_plan: asap_types::query_plan::QueryPlan::empty(), + collector_plans: vec![], + storage_routing: None, + adaptation_evidence: vec![], + } +} diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index eeeee37d..1aa1fefc 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -103,18 +103,14 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { eprintln!("IMMUTABLE_PROCESS_ARTIFACT {}", directory.path().display()); directory.disable_cleanup(true); let artifact = directory.path().join("plan.json"); - let bootstrap = directory.path().join("bootstrap.json"); let disk = directory.path().join("disk"); std::fs::create_dir_all(&disk).unwrap(); std::fs::write(&artifact, serde_json::to_vec(&install).unwrap()).unwrap(); - std::fs::write(&bootstrap, b"{\"aggregations\":[]}").unwrap(); let spawn = |port: u16| { ChildGuard( Command::new(env!("CARGO_BIN_EXE_data_plane")) .arg("--physical-plan") .arg(&artifact) - .arg("--streaming-config") - .arg(&bootstrap) .args(["--http-port", &port.to_string(), "--output-dir"]) .arg(directory.path()) .arg("--enable-remote-write") diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index cdc04f32..fdac3939 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -4,22 +4,42 @@ use control_plane::{physical::compiler::*, query_plan::*}; use data_plane::{ drivers::query::servers::http::PhysicalPlanInstallRequest, - storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, StreamingConfig}, + storage_engines::types::{ActivePhysicalPlan, BackendStorageRouting, InstalledPrecomputePlan}, }; use std::{collections::BTreeMap, sync::Arc}; -pub fn artifact(config: &StreamingConfig) -> PhysicalPlanInstallRequest { - artifact_from_materializations( - config - .materializations_by_policy_fingerprint - .values() - .cloned() - .collect(), +/// Imported-state fixtures bind explicit storage metadata, never a flat runtime config. +#[allow(dead_code)] +pub fn materialization( + metric: &str, + family: asap_types::AggregationType, + parameters: std::collections::HashMap, +) -> asap_types::PrecomputeMaterialization { + asap_types::PrecomputeMaterialization::new( + family, + String::new(), + parameters, + asap_types::KeyByLabelNames::new(vec!["service".into()]), + asap_types::KeyByLabelNames::empty(), + asap_types::KeyByLabelNames::empty(), + String::new(), + 1, + 1, + asap_types::enums::WindowKind::Tumbling, + String::new(), + metric.into(), + None, + None, + None, ) } +pub fn artifact(config: &InstalledPrecomputePlan) -> PhysicalPlanInstallRequest { + artifact_from_materializations(config.materializations().values().cloned().collect()) +} + /// Same as [`artifact`], but from materializations the planner produced -/// directly — no legacy `StreamingConfig` document in between. +/// directly — no legacy `InstalledPrecomputePlan` document in between. pub fn artifact_from_materializations( mut configs: Vec, ) -> PhysicalPlanInstallRequest { @@ -174,7 +194,7 @@ pub fn artifact_from_materializations( #[allow(dead_code)] pub fn bootstrap() -> ActivePhysicalPlan { let mut plan = data_plane::drivers::query::servers::http::build_active_physical_plan( - artifact(&StreamingConfig::default()), + artifact(&InstalledPrecomputePlan::default()), Arc::new(BackendStorageRouting::empty()), ) .unwrap(); diff --git a/docs/design_docs/precompute-dag-execution.md b/docs/design_docs/precompute-dag-execution.md index 7504c948..e16c9db5 100644 --- a/docs/design_docs/precompute-dag-execution.md +++ b/docs/design_docs/precompute-dag-execution.md @@ -1,24 +1,428 @@ -# Precompute execution from post-ASAP IR +# Precompute DAG execution -Audience: backend developers and reviewers of issue #762. +Audience: backend designers and developers. -The execution installation is `PrecomputePlan`: selected Planner DAGs, their node bindings, and physical window/storage placement. The former standalone `AggregationConfig` type is removed. `PrecomputeMaterialization` describes storage and routing; it is not independently executable. The streaming configuration serializes the DAG plan and derives its routing index after validation. Flat `aggregations` / `aggregation_configs` documents are rejected. Publish the complete physical plan through `/api/v1/physical-plan` and activate its generation; partial streaming configuration updates are removed. +This document specifies the precompute engine design: how it receives a +Planner-selected computation, runs it across data partitions, and publishes the +stored outputs consumed by query plans. The [plan split](asapplanner-integration.md) +and [SDS contract](summary-catalog-sds-architecture.md) define the compiler and +storage contracts used here. + +## 1. Intuition: compute once, reuse across queries, amortize the cost + +Precompute performs shared work ahead of query execution and stores its selected +outputs. Later queries reuse those outputs repeatedly, amortizing the cost of +constructing and maintaining them across the queries they serve. The purpose is +to reduce both total resource cost and query latency: repeated queries avoid +repeating expensive input scans and computation, while the work left on the +query path is a stored-state read and the remaining query operators. + +For example, repeated p50 and p99 requests for request latency by service over +the same five-minute range can share one Planner-selected KLL producer. +Precompute builds the KLL state once per service and scheduled window. Every +compatible request then reads that stored state and applies its percentile +readout, rather than scanning the samples and rebuilding the summary for each +request. Reuse applies both to repeated executions of a query and to different +queries that share the selected output. + +Over a workload, the resource cost is the cost of construction, maintenance and +storage plus the reads and remaining query computation. Reuse must save enough +repeated work to offset those costs; this is what makes precomputation worthwhile. +Once the required output is ready, query latency excludes its construction work. +“Compute once” refers to a particular output, population and window: subsequent +windows still require construction or updates under the selected schedule. + +The physical compiler splits the selected DAG at its stored outputs: ```mermaid flowchart LR - P[Selected Planner post-ASAP DAG] --> I[Validate DAG and physical bindings] - I --> R[Raw source → SummaryAgg streaming kernel] - I --> M[Maintenance dependency scheduler] - R --> S[Stored summary frontier] - S --> M - S --> Q[Query projection and readout] - M --> S + subgraph P[PrecomputePlan] + I[Read latency samples] --> G[Group by service] + G --> K[Build KLL state] + K --> W[Write latency-kll] + end + W --> S[SummaryStore] + subgraph Q[QueryPlans] + R[Read latency-kll] --> P50[Estimate p50] + R --> P99[Estimate p99] + end + S --> R +``` + +These labels describe the computation, not a separate backend operator language. +The executable nodes preserve Planner operator semantics and their provenance; +the compiler adds concrete source and storage bindings. + +A worker runs the **whole precompute subgraph for its assigned data partition**. +For this example, one worker handles `service=api` and another can handle +`service=worker`. Both execute the same graph. Each worker runs its nodes +sequentially in dependency order; workers process independent partitions +concurrently. A worker is an execution task, not necessarily a dedicated OS +thread. + +Only outputs selected for persistence become stored summaries. Intermediate +values remain in worker memory unless the plan explicitly stores them. A single +precompute subgraph can have several stored outputs, and several query plans +can read the same output. + +## 2. Inputs and outputs + +The physical compiler receives the selected post-ASAP DAG and the selected +deployment guarantee and schedule/retention. It emits one coherent plan version +containing definition rows, a PrecomputePlan, and matching QueryPlans. + +| Input to the precompute engine | Purpose | +| --- | --- | +| `PrecomputePlan` | Defines the precompute subgraph, node IDs, operator parameters, dependency edges, input bindings and stored-output bindings. Carries the selected deployment guarantee and schedule/retention. | +| Definition rows | Describe the input, Planner family, parameters, grouping and time semantics of each selected stored output. Installed in `SummaryStore.summary_definitions`. | +| Source data | Raw samples or rows, or explicitly referenced committed summary state. The plan determines which input type each node accepts. | +| Execution triggers | Input arrival, scheduled evaluation endpoints, and input-completion signals, according to the selected maintenance mode. | +| `PrecomputeEngineConfig` | Supplies worker count, queue capacity and operational settings such as flush polling interval. These settings cannot change computation, logical windows or the selected schedule/retention. | + +The engine produces committed `StoredSummary` records for the selected outputs. +It also reports completion or failure for scheduled work and drain requests. +Installing a plan does not imply that its output records are ready. + +A QueryPlan consumes these records through a `StoredOutputReference`. That +reference names a stored output and definition within the installed plan +version; population and window selection identify the records to read. + +## 3. Install the plan once + +The backend derives an immutable `InstalledPrecomputePlan` from the supplied +PrecomputePlan. This runtime representation avoids validating the graph and +binding operators again for every input batch. + +Installation performs four steps: + +1. Validate the graph and operator input/output types, including ordered edge + roles for operators with multiple inputs. Validate source, definition and + stored-output bindings together with the matching QueryPlans. +2. Bind supported runtime operators and compile a dependency execution order. + Preserve Planner families and parameters: Rate and Increase remain distinct, + and grouping does not create a separate backend family. +3. Derive a partitioning rule that keeps all required dependencies and + reductions local to a worker, as described below. Check that execution can + satisfy the selected deployment guarantee and schedule/retention. +4. Build node and source-routing indexes, register the validated definition + rows, and activate the coherent plan version for routing and execution. + +Unsupported operators, incompatible bindings, cycles and unsatisfied deployment +requirements fail installation before the plan becomes active. + +| Object | Lifetime and contents | +| --- | --- | +| `PrecomputePlan` | Compiler-supplied computation and bindings for one plan version. | +| `InstalledPrecomputePlan` | Backend-derived execution order, immutable operator programs, bindings, routing indexes and validated partitioning rule. Shared by the router and workers. | +| Worker execution state | Mutable node state, input ordering buffers, windows and intermediate results for that worker's assigned partitions. | + +`InstalledPrecomputePlan` is internal runtime data, not another serialized +configuration or independently installable plan. All execution lookup tables +come from the validated DAG and its bindings. There is no separately maintained +aggregation list. Operator fusion is permitted only when it preserves the DAG's +semantics and physical-to-semantic provenance. + +## 4. Route data so each worker can execute the whole subgraph + +Partitioning is a property of the entire precompute subgraph. All inputs that a +node must combine must reach the same worker. The router applies the installed +partitioning rule to canonical input labels and assigns that partition to a +worker for the lifetime of the active execution state. + +| Computation | Required data placement | +| --- | --- | +| KLL or Sum grouped by service | Keep all inputs for one service together. | +| Per-series Rate | Keep each series together and preserve its sample-time semantics. | +| Per-series Rate followed by Sum by service | Partition by service; keep separate Rate state for each series within the worker, then reduce locally. | +| Global Sum | Place all inputs on one worker in the simple execution model. | + +For Rate followed by Sum, distributing individual series independently could +put two series of the same service on different workers. Neither worker could +then compute the complete service result. Partitioning by service satisfies +both the per-series state requirement and the downstream reduction. + +The simple design uses one validated partitioning rule for the installed +precompute subgraph. If independent partitions cannot be established, use one +worker when the deployment requirements permit it; otherwise reject the plan. +There is no implicit shuffle or cross-worker partial-result merge. A hot +partition therefore remains limited by its owning worker. + +Partition ownership is separate from stored-output identity. Hashing a stored +output ID does not establish that its input dependencies are local. Multiple +stored outputs can share one data partition and one worker. + +```mermaid +flowchart TD + I[Bound input data] --> R[Route by validated data partition] + P[InstalledPrecomputePlan] -. shared program .-> A + P -. shared program .-> B + R --> A[Worker 0: entire subgraph for service api] + R --> B[Worker 1: entire subgraph for service worker] + A --> S[SummaryStore] + B --> S +``` + +Worker queues are bounded. Admission applies backpressure when a destination +queue is full; an atomic input batch must reserve its required queue capacity +before being acknowledged. Sequential queue consumption does not itself sort +event timestamps. Input ordering, allowed lateness and counter-reset handling +must follow the bound operators' time semantics. + +## 5. Execute ready work inside the owning worker + +Each worker holds the same installed program and separate mutable state. An +input batch updates only the state for its assigned data partition. A scheduled +evaluation identifies the logical window whose outputs are due. Nodes run when +the inputs required for that evaluation are available. + +The following diagram expands the KLL example for one scheduled window. The +router assigns partition work; each worker reads only its bound partition and +executes every required node of the same installed precompute subgraph. Dashed +arrows denote shared immutable program access; solid arrows denote work or data. + +```mermaid +flowchart TB + D[Selected Planner DAG] --> C[Physical compiler: split at stored outputs] + C --> P[PrecomputePlan: Read -> Group -> KLL -> Write] + C --> QP[QueryPlans: Read stored KLL -> Estimate] + P --> IP[InstalledPrecomputePlan: shared immutable program] + T[Scheduled window and source partitions] --> R[Route by service] + + subgraph W0[Worker 0: service=api] + direction TB + A0[Read partition input] --> A1[Group by service] + A1 --> A2[Build KLL: private state] + A2 --> A3[Write latency-kll for api and window] + end + subgraph W1[Worker 1: service=worker] + direction TB + B0[Read partition input] --> B1[Group by service] + B1 --> B2[Build KLL: private state] + B2 --> B3[Write latency-kll for worker and window] + end + + IP -. same program .-> A0 + IP -. same program .-> B0 + R -->|api partition| A0 + R -->|worker partition| B0 + A3 --> S[SummaryStore: separate committed population/window records] + B3 --> S + S --> QR[Query engine: bound stored-state read] + QP -. query program .-> QR + QR --> E[Estimate p50 or p99 on each query] ``` -For a raw producer, installation checks its `SummaryAgg` payload, input edge, source selection, reduction, family, and supported update expressions. The worker executes that validated projection with Planner-owned family and update parameters. Ingestion retains physical window management and routes populations using the validated binding. Shared producers have one installed program and one state per population/window. An unsupported raw path fails installation; the worker cannot choose Sum as a fallback. Backfill uses the same program and update evaluator. Derived summaries continue through the production maintenance scheduler, which observes stored frontiers, dependency roles and shared-node memoization. +Worker 0 and Worker 1 can run concurrently. Within each worker the arrows +establish dependency order: KLL construction follows its input computation, +and publication follows completion of the required state. Neither worker owns +an exclusive operator stage; both execute Read, Group, KLL and Write. They share +the program but not mutable KLL state. The two records use the same plan version, +stored-output ID and definition, with different population keys. Query execution +starts from the committed output rather than running this precompute path again. + +The selected maintenance mode determines how state is constructed. An +incremental operator updates its window state as input arrives. A batch operator +reads the bound range and builds its result at the scheduled endpoint. The +runtime cannot replace one mode with another merely because both produce the +same family of summary. + +Within one evaluation, the worker follows this workflow: + +### Shared physical operator execution + +The shared library lives in ASAPPlanner alongside post-ASAP IR. Backend #770 +pins that library and IR to the same revision; this PR integrates ingestion. +Installed precompute DAGs execute through its `PhysicalDag` runtime. The backend supplies +storage frontiers, declared edge order, window completeness and durable commit +keys. It does not own a second dependency walker. + +For completed-window DAGs, SummaryAgg uses the native summary builder, +SummaryMerge uses the native state merge, finalization uses native typed readout, +and Binary lowers aligned rows to native Project using the Planner binary contract, including checked division. Batch conversion +preserves the installed population and timestamp bindings. Native calls receive +the surrounding execution context, so they share its memory budget and +cancellation. Query execution uses the same library's operations. + +Raw ingestion retains per-window accumulator state through shared-library +updaters; worker routing and window completion remain backend responsibilities. +Storage publication occurs only after successful DAG execution. It is separate +from the library's request-local caching of intermediate results. + +```text +execute(partition, evaluation_window, bound_inputs): + check input identities and required completeness + create one intermediate-result map for this evaluation + for node in installed dependency order: + if this evaluation does not require node: continue + obtain inputs in their declared edge-role order + if a required input is incomplete: keep dependent work pending + otherwise: + execute node using this partition's node state + retain its result for every downstream consumer + publish completed selected outputs with their stored-output bindings +``` + +A shared upstream node is evaluated once for the same partition, window and +input revision, even when several downstream sinks consume it. Its result stays +available until those consumers finish. A later input revision or evaluation +window requires fresh evaluation; the cache is not keyed only by node ID across +unrelated work. + +Readiness belongs to dependencies, not merely to elapsed wall-clock time. For a +node consuming several sources, all required sources must satisfy the bound +window and population-completion rules. A flush timer cannot make missing input +complete. On a finite input run, admission closes and completion barriers follow +all accepted input; workers then finish eligible downstream work and acknowledge +drain only after the required output commits finish. + +A derived summary reads explicitly referenced committed source records and +checks their coverage before executing downstream operators. That read is a +frontier: the worker does not repeat the source records' upstream computation. +Derived work follows the same partition ownership and worker execution model. + +Nodes within a worker execute sequentially. Parallelism comes from independent +workers, including when they construct derived summaries. Store commit +coordination protects publication without requiring unrelated workers to hold +one global lock while evaluating their DAGs. + +## 6. Publish the selected DAG outputs + +One `SummaryStore` owns two logical tables: + +| Table | Record contents | +| --- | --- | +| `summary_definitions` | `SummaryDefinition`: canonical input, Planner family, parameters, grouping and time semantics. Shared across compatible records. | +| `stored_summaries` | `StoredSummary`: record key, definition ID, actual format and coverage, and payload. | + +The compiler assigns a `stored_output_id` to each output selected for +persistence. The precompute writer and query readers carry the same +`StoredOutputReference`. The reference is a plan binding, not a third catalog +object. + +A concrete stored record has the key: + +```text +(plan_id, plan_version, stored_output_id, population_key, window) +``` + +`population_key` preserves canonical label names and values. `window` identifies +the intended time partition and its boundary convention; actual coverage is +validated separately. Worker IDs and transient runtime handles are not record +identities. A store may use a local numeric row handle for indexing or wire +compression; that handle cannot select another output, definition or plan +version. The writer supplies a `StoredSummaryKey` and the reader supplies the +same selected `StoredOutputReference` within its installed plan version. +Durable metadata retains this binding so recovery validates it before exposing +payloads. V1 does not implicitly reuse records from another plan version. +The durable binding format is version 4; earlier SID metadata is rejected rather +than inferred to refer to a selected output. + +Before publication, validate that the output matches its bound definition, +format, population and window. Payload and identifying metadata become visible +as one committed record. A reader must not observe metadata pointing to an +uncommitted payload. Repeating the publication of the same completed result must +not add that result twice; conflicting writes must not silently overwrite a +record under the same key. Any permitted replacement follows the selected +update policy and preserves coherent reads. + +QueryPlan and derived precompute readers perform indexed lookup using the +installed reference and requested population/window. They check definition, +format, plan version and coverage before consuming a record. Missing or +incomplete state remains unavailable; query fallback is governed by QueryPlan. + +## 7. Worked example: one KLL producer, two percentile readers + +Use the latency example with this selected deployment: + +- Plan version `42`; group input by `service`. +- Build KLL with `k=200` over `(T - 5m, T]`. +- Rebuild from data at rest every minute, aligned to the Unix epoch. +- Retain completed summaries for ten minutes. +- Store output `latency-kll`, defined by `def-api-latency-kll`. + +The following YAML illustrates the bindings and stored record, not a Rust wire +schema: + +```yaml +plan_version: 42 +writer_reference: + stored_output_id: latency-kll + definition_id: def-api-latency-kll +reader_references: + p50: {stored_output_id: latency-kll, definition_id: def-api-latency-kll} + p99: {stored_output_id: latency-kll, definition_id: def-api-latency-kll} + +summary_definitions: + def-api-latency-kll: + input: {metric: request_latency_seconds, value: sample_value} + family: {kind: Sketch, algorithm: KLL, parameters: {k: 200}} + group_by: [service] + time_semantics: {range: 5m, bounds: "(start, end]"} + output_type: kll_state + +stored_summaries: + - key: + plan_version: 42 + stored_output_id: latency-kll + population_key: {service: api} + window: {start_exclusive: '12:00', end_inclusive: '12:05'} + definition_id: def-api-latency-kll + format: {schema: kll-v1, encoding: kll-binary-v1} + coverage: {start_exclusive: '12:00', end_inclusive: '12:05'} + payload: +``` + +At the `12:05` evaluation endpoint: + +1. The schedule triggers construction for `(12:00, 12:05]`. The bound source + supplies the required data and completion evidence for that range. +2. Routing sends `service=api` data to its owning worker. That worker executes + the read, grouping and KLL construction nodes for the partition. Another + worker can execute the same graph for `service=worker` concurrently. +3. The first worker commits the record illustrated above. The other worker + commits a separate record with `population_key: {service: worker}`. +4. Both percentile QueryPlans select the `service=api` record for endpoint + `12:05`. They read the same KLL state and apply their different estimates. + +At `12:06`, the same graph builds new records for `(12:01, 12:06]`. The definition, +stored-output ID and plan version stay the same; the window part of the key +changes. Ten-minute retention controls how long completed records remain +available, not how much input each KLL summarizes. Building a percentile result +at query time does not create another stored output. + +## 8. Plan changes, recovery and validation + +Routing, operator bindings and output references belong to one coherent plan +version. Accepted work retains that version until completion. Activation drains +or isolates incompatible old worker state before new work uses a different +partition assignment. Changing worker count requires coordinated reassignment; +changing a hash modulus while state is active is insufficient. + +A newly activated version is cold until its own stored outputs are available. +Queries must not substitute the previous version's payload to hide that gap. +Continuous warm service across activation requires an explicit readiness or +state-migration protocol; atomic plan publication alone does not provide it. + +Recovery validates persisted record identities, definitions, formats and +coverage before exposing them to readers. A matching definition alone does not +authorize reuse across plan versions. Runtime caches and routing indexes are +rebuilt from the installed plan. Input replay and durable operator-state +checkpointing require an explicit source/update contract; record publication +alone does not guarantee exactly-once ingestion. -`SummaryAgg` is the operator; Sum, Count, Min, Max, Rate and Increase are its exact families. `ExactAccumulator` retains the family and population layout across updates, reset, merge and serialization. Counter arithmetic can be shared internally, while a Rate state still rejects Increase readout or merge. Keyed layout does not introduce `MultipleX` Planner families. Config-based dispatch remains only in isolated kernel test fixtures and cannot execute in a production build. +Acceptance checks follow the workflow: -Catalog schema version 3 carries Planner family in SDS. Installation rejects disagreement between DAG and storage descriptors; storage admission rejects wrong exact families. The persisted `PlannerExactAccumulatorV1` encoding includes family and population layout. Tests cover a real Planner-selected DAG through worker execution and query readout, all six exact families through disk eviction/restart, invalid installations, and the native backend process Remote Write/HTTP query suite. +| Check | Required observation | +| --- | --- | +| Planner-to-runtime binding | Node semantics, edge roles, families and stored-output references agree; invalid plans fail installation. | +| One worker versus several | Identical results for grouped Sum, per-series Rate and Rate followed by service reduction, including counter resets and window boundaries. | +| Partition locality | Every dependency needed for a partition stays with its owner; a rule splitting a required reduction is rejected. | +| Shared computation | A common upstream node runs once per evaluation/input revision across its consumers, including multiple stored sinks. | +| Readiness and backpressure | Incomplete inputs do not produce readable complete records; admission and drain honor queued work and commits. | +| Store and query integration | Writer and reader use the same composite identity; format, coverage and version mismatches fail validation. | +| Restart and activation | Committed records recover coherently; retries do not duplicate completed publication; old and new plan state do not mix. | -The runtime supports explicit subsets of Planner operators. Shared Hydra grouping and unsupported raw input programs are rejected rather than silently assigned another algorithm. Existing imported collector state and isolated payload kernels are not alternate executable configuration formats. +The simple execution model does not require node-level parallel scheduling, +cross-worker shuffle, or separate metadata and payload services. Its unit of +parallel work is a complete precompute subgraph over an independent data +partition. diff --git a/docs/design_docs/summary-catalog-sds-architecture.md b/docs/design_docs/summary-catalog-sds-architecture.md index beb0f432..be6f4466 100644 --- a/docs/design_docs/summary-catalog-sds-architecture.md +++ b/docs/design_docs/summary-catalog-sds-architecture.md @@ -290,9 +290,10 @@ must still satisfy the reader. V1 needs no additional instance UUID. A `StoredOutputReference` identifies the output across its records, not a pointer to one payload; reader partition/time selection supplies the rest of the lookup. Schema/encoding IDs identify supported formats. -Human-readable names are diagnostics, not join keys. Reuse across plan versions -requires an explicit compatibility decision; a matching definition ID is -insufficient. +Human-readable names and local numeric row handles are not join keys. V1 reads +only the installed plan version; a matching definition ID does not authorize +reading another version’s payload. Cross-version reuse would require an explicit +reader binding and is outside the V1 storage path. A `StoredOutputReference` identifies a stored output and definition within the enclosing plan version. The reader/writer binding constrains acceptable partition, schema,