diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index d7488a927..557f198f5 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -25,24 +25,20 @@ struct Row { } fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { + let family = AggregationType::Max.planner_exact_family().unwrap(); let mut materialization = PrecomputeMaterialization::new( - AggregationType::Max, - String::new(), - std::collections::HashMap::from([("variant".into(), json!(2))]), + "raw_samples.value", KeyByLabelNames::new(vec!["labels".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - "raw_samples.value".into(), - None, - Some("raw_samples".into()), - Some("value".into()), ); + materialization.table_name = Some("raw_samples".into()); + materialization.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); materialization.pane_origin_ms = Some(0); + materialization.allocate_stored_output_id(&family); let envelope = PlanEnvelope { plan_id: 27, plan_version: 1, @@ -54,7 +50,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { capability_snapshot_id: "sql27-main-eval".into(), }; let mut precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) + PrecomputePlan::build_backend_local(envelope.clone(), vec![(materialization, family)]) .unwrap(); let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, @@ -62,12 +58,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { &Default::default(), ) .unwrap(); - let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( - 27, - 1, - &[materialization], - ) - .unwrap(); + let sds = asap_types::summary_catalog::SummaryCatalog::from_plan(&precompute_plan).unwrap(); let reference = sds.reference().unwrap(); precompute_plan.bind_catalog(&sds).unwrap(); transmission_plan.summary_catalog = Some(reference); diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index dc421e778..289dc9fe8 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -260,7 +260,7 @@ pub async fn compile_automatic_clickhouse_workload( let selection_trace = std::mem::take(&mut planned.selection_trace); let template = planned.canonical_sql.clone(); let (entry, installed) = compile_selected_sql(query, planned, |node, family| { - let config = materialize_selected_sql(node, family, query) + let (config, family) = materialize_selected_sql(node, family, query) .map_err(crate::query_plan::QueryPlanError::Invalid)?; let binding = MaterializationBinding { full_window_slide_ms: matches!( @@ -276,11 +276,12 @@ pub async fn compile_automatic_clickhouse_workload( window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(query.end_ms - query.start_ms), - item_labels: config.aggregated_labels.labels.clone(), + // SQL table summaries project no per-item dimension. + item_labels: Vec::new(), }; materializations .entry(config.policy_fingerprint()) - .or_insert(config); + .or_insert((config, family)); Ok(binding) })?; index_sql_template(&mut window_templates, template, &entry); @@ -300,15 +301,11 @@ pub async fn compile_automatic_clickhouse_workload( ); } let configs: Vec<_> = materializations.into_values().collect(); - let sds = SummaryCatalog::from_materializations( - request.envelope.plan_id, - request.envelope.plan_version, - &configs, - ) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut precompute = PrecomputePlan::build_backend_local(request.envelope.clone(), configs) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; precompute.executable_dags = installed_dags; + let sds = SummaryCatalog::from_plan(&precompute) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; precompute .bind_catalog(&sds) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; @@ -350,7 +347,13 @@ fn materialize_selected_sql( node: &planner_types::post_asap::SummaryNode, family: &planner_types::post_asap::SummaryFamilyType, query: &ClickHouseSqlWorkloadEntry, -) -> Result { +) -> Result< + ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ), + String, +> { use crate::physical::backend_stage::{AggregationInput, BackendAggregation}; use planner_types::{post_asap::SummaryExpr, pre_asap::Reduction}; let SummaryExpr::SummaryAgg { @@ -401,11 +404,12 @@ fn materialize_selected_sql( heap_update_mode: None, aggregation_input: AggregationInput::Raw, }; - let mut config = crate::physical::compiler::aggregation_config_for_materialization( - &aggregation, - asap_types::QueryLanguage::ClickHouseSql, - ) - .map_err(|error| error.to_string())?; + let (mut config, computation) = + crate::physical::compiler::aggregation_config_for_materialization( + &aggregation, + asap_types::QueryLanguage::ClickHouseSql, + ) + .map_err(|error| error.to_string())?; config.table_name = Some(table); config.grouping_labels = grouping; config.value_projection = Some(value); @@ -418,7 +422,8 @@ fn materialize_selected_sql( .map_err(|_| "SQL evaluation timestamp exceeds runtime range")?, ); config.num_aggregates_to_retain = Some(2); - Ok(config) + config.allocate_stored_output_id(&computation); + Ok((config, computation.0)) } pub async fn compile_clickhouse_workload( @@ -620,7 +625,7 @@ fn bind_selected_node( .map_err(crate::query_plan::QueryPlanError::Invalid)?; let expected = family.clone(); let selected = select_materialization( - &request.precompute_plan.materializations, + &request.precompute_plan, &table_ref, &value_column, &spatial_filter.canonical(), @@ -646,7 +651,7 @@ fn bind_selected_node( window_ms: selected.stored_window_ms(), pane_origin_ms: selected.pane_origin_ms, readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1000)), - item_labels: selected.aggregated_labels.labels.clone(), + item_labels: Vec::new(), }) } @@ -920,21 +925,18 @@ fn clickhouse_materialization_leaf_contract( } fn select_materialization<'a>( - materializations: &'a [asap_types::PrecomputeMaterialization], + plan: &'a asap_types::precompute_plan::PrecomputePlan, table_ref: &str, value_projection: &asap_types::sds::ValueProjectionIdentity, spatial_filter: &str, expected: &planner_types::post_asap::SummaryFamilyType, semantic_window_seconds: u64, ) -> Result<&'a asap_types::PrecomputeMaterialization, crate::query_plan::QueryPlanError> { - let mut matches = materializations.iter().filter(|candidate| { + let mut matches = plan.materializations.iter().filter(|candidate| { candidate.table_name.as_deref() == Some(table_ref) && candidate.effective_value_projection() == value_projection - && candidate.population_filter_canonical().ok().as_deref() == Some(spatial_filter) - && candidate - .accumulator_spec() - .ok() - .is_some_and(|spec| spec.family == *expected) + && candidate.table_population_canonical().ok().as_deref() == Some(spatial_filter) + && plan.state_family(candidate.stored_output_id) == Some(expected) && semantic_window_seconds .checked_mul(1000) .is_some_and(|window| window % candidate.window_size.saturating_mul(1000) == 0) @@ -956,6 +958,7 @@ fn select_materialization<'a>( mod tests { use super::*; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::SummaryFamilyType; use planner_types::pre_asap::{Column, DataType, Schema}; // A dashboard refresh must reuse the installed identity without treating @@ -1042,33 +1045,54 @@ mod tests { } } + fn exact(agg: AggregationType) -> SummaryFamilyType { + agg.planner_exact_family().unwrap() + } + + fn ddsketch(alpha: f64) -> SummaryFamilyType { + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + }; + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), + GroupingStrategy::PerSubpopulationInstance, + ) + } + fn materialization( - agg: AggregationType, + family: SummaryFamilyType, value_column: &str, window: u64, slide: u64, - parameter: (&str, serde_json::Value), - ) -> PrecomputeMaterialization { + ) -> (PrecomputeMaterialization, SummaryFamilyType) { let mut value = PrecomputeMaterialization::new( - agg, - String::new(), - std::collections::HashMap::from([(parameter.0.into(), parameter.1)]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + format!("telemetry.{value_column}"), KeyByLabelNames::empty(), - String::new(), window, slide, WindowKind::Tumbling, - String::new(), - format!("telemetry.{value_column}"), - None, - Some("telemetry".into()), - Some(value_column.into()), ); + value.table_name = Some("telemetry".into()); + value.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: value_column.into(), + }); value.pane_origin_ms = Some(0); value.table_timestamp_column = Some("timestamp_ms".into()); - value + value.allocate_stored_output_id(&family); + (value, family) + } + + fn envelope(plan_id: u64) -> crate::physical::compiler::PlanEnvelope { + crate::physical::compiler::PlanEnvelope { + plan_id, + plan_version: 1, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: crate::physical::compiler::BACKEND_COMPAT.into(), + planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), + capability_snapshot_id: "clickhouse-test".into(), + } } /// A table leaf whose value column has the given producer typing. @@ -1108,16 +1132,7 @@ mod tests { ], schema, }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; + let family = exact(AggregationType::Sum); let summary_schema = SummarySchema { fields: vec![], time_index: None, @@ -1208,16 +1223,7 @@ mod tests { fields: vec![], time_index: None, }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; + let family = exact(AggregationType::Sum); let node = SummaryNode { expr: SummaryExpr::SummaryAgg { child: std::rc::Rc::new(SummaryNode { @@ -1246,130 +1252,55 @@ mod tests { #[test] fn selected_nodes_bind_unique_family_parameters_source_and_window() { - let sum_60 = materialization( - AggregationType::Sum, - "requests", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let count_60 = materialization( - AggregationType::Max, - "requests", - 60, - 10, - ("variant", serde_json::json!(2)), - ); - let sum_300 = materialization( - AggregationType::Sum, - "requests", - 300, - 30, - ("variant", serde_json::json!(3)), - ); - let other = materialization( - AggregationType::Sum, - "latency", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let dd_2 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.02)), - ); - let dd_5 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.05)), - ); - let configs = vec![ - sum_60.clone(), - count_60.clone(), - sum_300, - other, - dd_2.clone(), - dd_5, - ]; - let sum_family = sum_60.accumulator_spec().unwrap().family; - let count_family = count_60.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - sum_60.policy_fingerprint() - ); - let dd_family = dd_2.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &dd_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - dd_2.policy_fingerprint() - ); - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &count_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - count_60.policy_fingerprint() - ); - assert!(select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "missing".into() - }, - "", - &sum_family, - 60 - ) - .is_err()); - let mut ambiguous = configs.clone(); - ambiguous.push(sum_60); - assert!(select_materialization( - &ambiguous, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 + let sum_60 = materialization(exact(AggregationType::Sum), "requests", 60, 60); + let count_60 = materialization(exact(AggregationType::Max), "requests", 60, 60); + let sum_300 = materialization(exact(AggregationType::Sum), "requests", 300, 300); + let other = materialization(exact(AggregationType::Sum), "latency", 60, 60); + let dd_2 = materialization(ddsketch(0.02), "requests", 60, 60); + let dd_5 = materialization(ddsketch(0.05), "requests", 60, 60); + let plan = PrecomputePlan::build_backend_local( + envelope(1), + vec![ + sum_60.clone(), + count_60.clone(), + sum_300, + other, + dd_2.clone(), + dd_5, + ], ) - .unwrap_err() - .to_string() - .contains("ambiguous")); + .unwrap(); + let requests = asap_types::sds::ValueProjectionIdentity::Column { + name: "requests".into(), + }; + let select = |plan, projection, family| { + select_materialization(plan, "telemetry", projection, "", family, 60) + .map(|selected| selected.policy_fingerprint()) + }; + for expected in [&sum_60, &count_60, &dd_2] { + assert_eq!( + select(&plan, &requests, &expected.1).unwrap(), + expected.0.policy_fingerprint() + ); + } + let missing = asap_types::sds::ValueProjectionIdentity::Column { + name: "missing".into(), + }; + assert!(select(&plan, &missing, &sum_60.1).is_err()); + let mut ambiguous = plan.clone(); + let mut duplicate = sum_60.0.clone(); + duplicate.stored_output_id = asap_types::sds::StoredOutputId(1); + ambiguous.materializations.push(duplicate.clone()); + ambiguous + .schemas + .push(asap_types::precompute_plan::StateSchemaContract { + materialization: duplicate.stored_output_id, + ..plan.schemas[0].clone() + }); + assert!(select(&ambiguous, &requests, &sum_60.1) + .unwrap_err() + .to_string() + .contains("ambiguous")); } #[test] @@ -1414,27 +1345,11 @@ mod tests { #[tokio::test] async fn rejects_mixed_snapshots_and_preserves_local_sql_contracts() { - let config = materialization( - AggregationType::Sum, - "value", - 2, - 2, - ("variant", serde_json::json!(1)), - ); - let sds = - SummaryCatalog::from_materializations(71, 1, std::slice::from_ref(&config)).unwrap(); - let envelope = crate::physical::compiler::PlanEnvelope { - plan_id: 71, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: crate::physical::compiler::BACKEND_COMPAT.into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "clickhouse-mixed-compile-test".into(), - }; + let config = materialization(exact(AggregationType::Sum), "value", 2, 2); + let envelope = envelope(71); let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); + let sds = SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&sds).unwrap(); let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, @@ -1734,11 +1649,14 @@ mod tests { value: planner_types::pre_asap::ScalarValue::Utf8("requests".into()), }], }); - request.summary_catalog = - SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); + config.allocate_stored_output_id(&exact(AggregationType::Sum)); let envelope = request.precompute_plan.envelope.clone(); - request.precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); + request.precompute_plan = PrecomputePlan::build_backend_local( + envelope.clone(), + vec![(config, exact(AggregationType::Sum))], + ) + .unwrap(); + request.summary_catalog = SummaryCatalog::from_plan(&request.precompute_plan).unwrap(); request .precompute_plan .bind_catalog(&request.summary_catalog) diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs index 33759262a..dea69a85a 100644 --- a/control_plane/src/opamp/mod.rs +++ b/control_plane/src/opamp/mod.rs @@ -1004,7 +1004,7 @@ mod tests { generated_at_unix_ms: 1, activation_unix_ms: 1, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "caps-1".into(), }, @@ -1033,7 +1033,7 @@ mod tests { transmission_rules: vec![crate::physical::compiler::TransmissionRule { materialization: asap_types::PolicyFingerprint(1).into(), producer_id: collector_id.into(), - schema_id: "asap-query-backend.v1:summary-state:v1:1".into(), + schema_id: "asap-query-backend.v2:summary-state:v1:1".into(), mode: crate::physical::compiler::TransmissionMode::Full, encoding: crate::physical::compiler::StateEncoding::SketchlibProtobufV1, emit_every_ms: 60_000, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index b04e57d7f..251856e99 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -852,7 +852,7 @@ fn state_fingerprints( &physical_aggregation(query, state, query.query_id.clone(), target), &state.node, ) - .map(|config| config.policy_fingerprint()) + .map(|(config, _)| config.policy_fingerprint()) .map_err(|error| error.to_string()) }) .collect() @@ -1634,6 +1634,7 @@ impl DeploymentPlanCompiler { } let population_operators = super::maintained_population::operators(&request)?; let mut compiled_materializations = Vec::with_capacity(request.queries.len()); + let mut output_computations = BTreeMap::<_, OutputComputation>::new(); let mut collector_materializations = Vec::with_capacity(request.queries.len()); let mut plan_materializations = Vec::with_capacity(request.queries.len()); let mut shared_materializations = BTreeMap::new(); @@ -1843,7 +1844,7 @@ impl DeploymentPlanCompiler { .or_default() .push(kind.algorithm().clone()); } - let precompute_materialization = + let (precompute_materialization, _) = scoped_materialization(&aggregation, &selected.node)?; let materialization = precompute_materialization.policy_fingerprint(); let consumer_indices = consumers[&materialization] @@ -1911,7 +1912,7 @@ impl DeploymentPlanCompiler { // Preserve semantic window and evaluation cadence independently // from the selected storage representation. aggregation.window_secs = window_implementation.window_secs; - let mut runtime_materialization = + let (mut runtime_materialization, computation) = scoped_materialization(&aggregation, &selected.node)?; if canonical_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { runtime_materialization.population_key_encoding = @@ -2059,7 +2060,9 @@ impl DeploymentPlanCompiler { reason, })?, ); + runtime_materialization.allocate_stored_output_id(&computation); let materialization = runtime_materialization.policy_fingerprint(); + output_computations.insert(materialization, computation.clone()); let consumer_query_ids = state_consumers .iter() .map(|query| query.query_id.clone()) @@ -2173,6 +2176,7 @@ impl DeploymentPlanCompiler { &mut node_bindings, &mut runtime_policies, &mut lifecycle_estimates, + &mut output_computations, ); } @@ -2293,11 +2297,15 @@ impl DeploymentPlanCompiler { })?; let (_, source_window, _) = selected_input_contract(node) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let materialization_family = materialization.accumulator_spec() - .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? - .family; + let materialization_family = &output_computations + .get(&fingerprint) + .ok_or_else(|| crate::query_plan::QueryPlanError::Invalid(format!( + "compiled binding {} has no state family", + fingerprint.0 + )))? + .0; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != *node_family + if materialization_family != node_family || window_ms == 0 || source_window.map(|seconds| seconds.saturating_mul(1_000)).unwrap_or(query.query_lookback_ms) % window_ms != 0 @@ -2316,7 +2324,12 @@ impl DeploymentPlanCompiler { output_grouping: PhysicalGrouping::Reduce( materialization.grouping_labels.names(), ), - item_labels: materialization.aggregated_labels.labels.clone(), + item_labels: output_computations[&fingerprint].1["item_labels"] + .as_array() + .into_iter() + .flatten() + .filter_map(|label| label.as_str().map(str::to_owned)) + .collect(), window_ms: stored_interval_ms, pane_origin_ms: materialization.pane_origin_ms, }) @@ -2703,7 +2716,7 @@ impl DeploymentPlanCompiler { } } validate_retained_summary_footprint( - &materializations, + &with_families(materializations.clone(), &output_computations), request .data_workload .as_ref() @@ -2746,19 +2759,20 @@ impl DeploymentPlanCompiler { }) .collect::>()?; let mut precompute_plan = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => { - PrecomputePlan::build(envelope.clone(), materializations, &producer_ids).and_then( - |mut plan| { - plan.executable_dags = installed_dags; - plan.validate()?; - Ok(plan) - }, - ) - } + PhysicalDeploymentTarget::DistributedCollectors => PrecomputePlan::build( + envelope.clone(), + with_families(materializations, &output_computations), + &producer_ids, + ) + .and_then(|mut plan| { + plan.executable_dags = installed_dags; + plan.validate()?; + Ok(plan) + }), PhysicalDeploymentTarget::BackendLocalRemoteWrite => { PrecomputePlan::build_backend_local_with_dags( envelope.clone(), - materializations, + with_families(materializations, &output_computations), installed_dags, ) } @@ -2792,12 +2806,8 @@ impl DeploymentPlanCompiler { materializations: collector_materializations.clone(), }) .collect::>(); - let summary_catalog = super::summary_catalog::SummaryCatalog::from_materializations( - envelope.plan_id, - envelope.plan_version, - &precompute_plan.materializations, - ) - .map_err(|error| CompileError::Query { + let summary_catalog = super::summary_catalog::SummaryCatalog::from_plan(&precompute_plan) + .map_err(|error| CompileError::Query { query_id: "summary-catalog".into(), reason: error.to_string(), })?; @@ -3796,11 +3806,36 @@ pub(super) fn retained_state_count( /// Estimate the encoded bytes retained by one physical state. Sketch matrix /// cells use two words here, covering the counter plus observed serialization /// overhead. Heap and exact-state estimates include container slack. -fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) -> u128 { - estimated_state_bytes( - &materialization.aggregation_type, - &materialization.parameters, - ) +fn retained_state_bytes(family: &SummaryFamilyType) -> u128 { + let parameters = match family { + SummaryFamilyType::Sketch(kind, _) => sketch_params_json(kind.params()), + _ => json!({}), + }; + asap_types::aggregation_type_for_family(family).map_or(256, |aggregation_type| { + estimated_state_bytes( + &aggregation_type, + ¶meters + .as_object() + .into_iter() + .flatten() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ) + }) +} + +/// Pair each output with its state family for the installed plan. +fn with_families( + materializations: Vec, + computations: &BTreeMap, +) -> Vec<(asap_types::PrecomputeMaterialization, SummaryFamilyType)> { + materializations + .into_iter() + .map(|config| { + let family = computations[&config.policy_fingerprint()].0.clone(); + (config, family) + }) + .collect() } pub(crate) fn estimated_state_bytes( @@ -3849,23 +3884,25 @@ pub(crate) fn estimated_state_bytes( pub(crate) fn retained_partition_count( materialization: &asap_types::PrecomputeMaterialization, + family: &SummaryFamilyType, input_cardinality: Option, ) -> u128 { use asap_types::AggregationType as A; + let aggregation_type = asap_types::aggregation_type_for_family(family); // Reset-aware and min/max state remains source-series scoped even with an // empty output grouping. Grouped states have at most one partition per // input series. Other empty groupings are the Reduce([]) global singleton. if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( - materialization.aggregation_type, - A::Increase | A::Rate | A::Min | A::Max + aggregation_type, + Some(A::Increase | A::Rate | A::Min | A::Max) ) || !materialization.grouping_labels.names().is_empty() || (materialization.derived_input.is_some() && matches!( - materialization.aggregation_type, - A::CountMinSketchWithHeap | A::CountSketchWithHeap | A::Sum + aggregation_type, + Some(A::CountMinSketchWithHeap | A::CountSketchWithHeap | A::Sum) )) { u128::from(input_cardinality.unwrap_or(1).max(1)) @@ -3875,19 +3912,23 @@ pub(crate) fn retained_partition_count( } fn validate_retained_summary_footprint( - materializations: &[asap_types::PrecomputeMaterialization], + outputs: &[(asap_types::PrecomputeMaterialization, SummaryFamilyType)], input_cardinality: Option, budget_bytes: u64, ) -> Result<(), CompileError> { - let estimated_bytes = materializations + let estimated_bytes = outputs .iter() - .fold(0u128, |total, materialization| { + .fold(0u128, |total, (materialization, family)| { total.saturating_add( - retained_state_bytes(materialization) + retained_state_bytes(family) .saturating_mul(u128::from( materialization.num_aggregates_to_retain.unwrap_or(1), )) - .saturating_mul(retained_partition_count(materialization, input_cardinality)), + .saturating_mul(retained_partition_count( + materialization, + family, + input_cardinality, + )), ) }); if estimated_bytes > u128::from(budget_bytes) { @@ -4156,71 +4197,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -pub fn raw_time_series_input_contract( - expr: &QueryExpr, - exact: bool, -) -> Result<(String, Option, String), String> { - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let (source, window_secs) = match expr { - QueryExpr::TimeRange { child, range } => { - if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { - return Err("warm producer requires a positive whole-second range".into()); - } - (child.as_ref(), Some(range.as_secs())) - } - QueryExpr::Scan { .. } if exact => { - return Err( - "instantaneous sample selection is not a temporal accumulator readout".into(), - ); - } - source => (source, None), - }; - match source { - QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } if !metric.is_empty() => { - let mut matchers = Vec::with_capacity(predicates.len()); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - return Err("materialization filter is not a label comparison".into()); - }; - let (QueryExpr::Column(column), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return Err("materialization filter must compare a label with a string".into()); - }; - let label = schema - .columns - .get(*column) - .map(|field| field.name.as_str()) - .ok_or_else(|| { - "materialization filter references an unknown column".to_string() - })?; - let operator = match op { - CompareOpKind::Eq => "=", - CompareOpKind::Ne => "!=", - CompareOpKind::Regex => "=~", - CompareOpKind::NotRegex => "!~", - _ => return Err("materialization filter uses a non-PromQL comparison".into()), - }; - let encoded = serde_json::to_string(value).map_err(|error| error.to_string())?; - matchers.push(format!("{label}{operator}{encoded}")); - } - matchers.sort(); - let spatial_filter = if matchers.is_empty() { - String::new() - } else { - format!("{{{}}}", matchers.join(",")) - }; - Ok((metric.clone(), window_secs, spatial_filter)) - } - _ => { - Err("source predicates or temporal modifiers require a Prometheus exact subtree".into()) - } - } -} +pub use asap_types::precompute_plan::raw_time_series_input_contract; /// Immutable inputs may combine explicit exact accumulator readouts using /// maintenance-time arithmetic. Population and window closure are checked by @@ -4438,11 +4415,11 @@ fn physical_aggregation( fn scoped_materialization( aggregation: &BackendAggregation, node: &SummaryNode, -) -> anyhow::Result { - let mut config = +) -> anyhow::Result<(asap_types::PrecomputeMaterialization, OutputComputation)> { + let (mut config, computation) = aggregation_config_for_materialization(aggregation, asap_types::QueryLanguage::PromQl)?; if !matches!(aggregation.aggregation_input, AggregationInput::Raw) { - return Ok(config); + return Ok((config, computation)); } let SummaryExpr::SummaryAgg { reduction, .. } = &node.expr else { anyhow::bail!("materialization lacks SummaryAgg partition contract"); @@ -4471,33 +4448,58 @@ fn scoped_materialization( asap_types::sds::PopulationPartitioning::Grouped } }); - Ok(config) + config.allocate_stored_output_id(&computation); + Ok((config, computation)) } -pub(crate) fn aggregation_config_for_materialization( +/// The Planner computation a stored output id is allocated from: the +/// output's state family and its canonical producer description (update, +/// input predicate and item projection). The compiler keeps it beside each +/// output so later layout decisions can reallocate the id; installed plans +/// carry neither, since the bound DAG node defines the computation. +pub(crate) type OutputComputation = (SummaryFamilyType, Value); + +pub(crate) fn output_computation( aggregation: &BackendAggregation, language: asap_types::QueryLanguage, -) -> anyhow::Result { - use anyhow::Context as _; - let mut json = crate::emit::backend_wire::build_backend_aggregation_json(aggregation); - // The selected physical duration is authoritative. The legacy edge emitter's - // 5..60 second clamp must not silently change a backend materialization. - json["windowSize"] = serde_json::json!(aggregation.window_secs); +) -> OutputComputation { + let json = crate::emit::backend_wire::build_backend_aggregation_json(aggregation); + let mut parameters = json["parameters"].clone(); if language == asap_types::QueryLanguage::ClickHouseSql { // Raw input does not imply PromQL's (start,end] convention. SQL bounds // are normalized and bound explicitly by the SQL compiler. - json["parameters"] + parameters .as_object_mut() .expect("emitter parameters are an object") .remove("promql_right_closed"); } - let yaml = serde_yaml::to_value(json).context("convert physical aggregation fields")?; - asap_types::PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - asap_types::QueryLanguage::PromQl, + // Deployment fields (metric, grouping, window) are hashed from the + // materialization itself, so layout changes can reallocate the id. + ( + aggregation.family.clone(), + serde_json::json!({ + "aggregation_type": json["aggregationType"], + "parameters": parameters, + "item_labels": json["labels"]["aggregated"], + "spatial_filter": asap_types::utils::normalize_spatial_filter(&aggregation.spatial_filter), + }), ) - .context("build materialization from physical aggregation") +} + +pub(crate) fn aggregation_config_for_materialization( + aggregation: &BackendAggregation, + language: asap_types::QueryLanguage, +) -> anyhow::Result<(asap_types::PrecomputeMaterialization, OutputComputation)> { + let computation = output_computation(aggregation, language); + let mut config = asap_types::PrecomputeMaterialization::new( + aggregation.metric_name.clone(), + asap_types::KeyByLabelNames::new(aggregation.grouping.clone()), + aggregation.window_secs, + aggregation.window_secs, + asap_types::WindowKind::Tumbling, + ); + config.allocate_stored_output_id(&computation); + Ok((config, computation)) } fn materialization_consumers<'a>( @@ -4532,7 +4534,7 @@ fn materialization_consumers<'a>( { continue; } - let config = scoped_materialization( + let (config, _) = scoped_materialization( &physical_aggregation(query, &state, query.query_id.clone(), target), &state.node, )?; @@ -5030,6 +5032,29 @@ impl QueryFrontend { } #[cfg(test)] pub(crate) mod tests { + /// The Planner update of the DAG node that produces `output`. + fn bound_summary_update( + plan: &PrecomputePlan, + output: asap_types::sds::StoredOutputId, + ) -> planner_types::post_asap::SummaryUpdate { + use asap_types::executable_plan::BackendNodeBinding; + plan.executable_dags + .values() + .find_map(|installed| { + let (node, _) = installed.binding.nodes.iter().find(|(_, binding)| { + matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == output) + })?; + let dag = installed.document.decode().unwrap(); + match &dag.nodes.iter().find(|candidate| candidate.id == *node)?.payload { + planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { input, .. } => { + Some(input.clone()) + } + _ => None, + } + }) + .expect("output has a Planner SummaryAgg producer") + } + use super::*; /// Multiple current-value quantiles and TopK limits share one maintained population. @@ -5200,8 +5225,12 @@ pub(crate) mod tests { let plan = DeploymentPlanCompiler.compile_promql(input, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - asap_types::AggregationType::Min + asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(plan.precompute_plan.materializations[0].stored_output_id) + .unwrap() + ), + Some(asap_types::AggregationType::Min) ); assert!(plan .query_plan @@ -5844,10 +5873,18 @@ pub(crate) mod tests { .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); - assert_eq!( - plan.precompute_plan.materializations[0].parameters["weight_mode"], mode, - "{query}" - ); + let weight = bound_summary_update( + &plan.precompute_plan, + plan.precompute_plan.materializations[0].stored_output_id, + ) + .weight; + let expected = match mode { + "count" => planner_types::post_asap::SummaryInputExpr::Constant(1.0), + _ => planner_types::post_asap::SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + }; + assert_eq!(weight, expected, "{query}"); } } @@ -5935,38 +5972,71 @@ pub(crate) mod tests { environment(10_000), ) .unwrap(); - let mut materializations = plan.precompute_plan.materializations; - let cms = materializations + let mut outputs = plan + .precompute_plan + .materializations + .iter() + .map(|config| { + let family = plan + .precompute_plan + .state_family(config.stored_output_id) + .unwrap(); + (config.clone(), family.clone()) + }) + .collect::>(); + let heap_size = outputs + .iter() + .find_map(|(_, family)| match family { + SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::CmsWithHeap { heap_size, .. } => { + Some(*heap_size) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + let heap = |width| { + SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::CmsWithHeap, + planner_types::post_asap::SketchParams::CmsWithHeap { + width, + depth: 7, + heap_size, + }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ) + }; + let cms = outputs .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap + .find(|(_, family)| { + asap_types::aggregation_type_for_family(family) + == Some(asap_types::AggregationType::CountMinSketchWithHeap) }) .unwrap(); - cms.parameters.remove("width"); - cms.parameters.remove("depth"); - cms.parameters.insert("w".into(), json!(524_288)); - cms.parameters.insert("d".into(), json!(7)); - cms.num_aggregates_to_retain = Some(80); + cms.1 = heap(524_288); + cms.0.num_aggregates_to_retain = Some(80); let error = validate_retained_summary_footprint( - &materializations, + &outputs, Some(5), DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, ) .unwrap_err(); assert!(error.to_string().contains("exceeds budget")); - let cms = materializations + let cms = outputs .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap + .find(|(_, family)| { + asap_types::aggregation_type_for_family(family) + == Some(asap_types::AggregationType::CountMinSketchWithHeap) }) .unwrap(); - cms.parameters.insert("w".into(), json!(4_096)); + cms.1 = heap(4_096); validate_retained_summary_footprint( - &materializations, + &outputs, Some(5), DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, ) @@ -6055,7 +6125,7 @@ pub(crate) mod tests { plan_version: 1, activation_unix_ms: now, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), } } @@ -7554,12 +7624,12 @@ pub(crate) mod tests { .compile_promql(request("q", "sum_over_time(m[1m])"), environment(10_000)) .unwrap(); let mut source = bundle.precompute_plan.materializations[0].clone(); - source.stored_output_id = Some(asap_types::sds::StoredOutputId(1)); + source.stored_output_id = asap_types::sds::StoredOutputId(1); source.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; let mut configs = vec![source.clone()]; for id in [3, 4] { let mut output = source.clone(); - output.stored_output_id = Some(asap_types::sds::StoredOutputId(id)); + output.stored_output_id = asap_types::sds::StoredOutputId(id); output.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([asap_types::sds::StoredOutputId(1)]), program_sha256: "0".repeat(64), @@ -8876,9 +8946,19 @@ pub(crate) mod tests { environment(10_000), ) .unwrap(); - let mut materializations = bundle.precompute_plan.materializations; - materializations[0].aggregation_type = asap_types::AggregationType::HLL; - materializations[0].parameters.clear(); + let hll = SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + planner_types::post_asap::SketchParams::Hll { precision: 14 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let materializations = bundle + .precompute_plan + .materializations + .into_iter() + .map(|config| (config, hll.clone())) + .collect::>(); let mut envelope_plan = PrecomputePlan::build( bundle.envelope.clone(), materializations.clone(), @@ -8909,7 +8989,15 @@ pub(crate) mod tests { .expect("compile"); let plan = PrecomputePlan::build_backend_local( bundle.envelope.clone(), - bundle.precompute_plan.materializations.clone(), + bundle + .precompute_plan + .materializations + .iter() + .map(|config| { + let family = bundle.precompute_plan.state_family(config.stored_output_id); + (config.clone(), family.unwrap().clone()) + }) + .collect(), ) .expect("backend-local contract"); @@ -9106,7 +9194,10 @@ pub(crate) mod tests { if expected_readout == crate::query_plan::ExactReadout::Rate { let materialization = plan.precompute_plan.materializations.first().unwrap(); assert_eq!( - materialization.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(materialization.stored_output_id) + .unwrap() + .clone(), SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Rate, planner_types::post_asap::ExactParams::Rate, diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 72dfb3e2a..0a59d97dc 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -396,17 +396,17 @@ impl ErpPlanningInput { .summary_descriptors .get(&materialization.summary_descriptor_id) .ok_or("ERP summary descriptor is absent from the active catalog")?; - let actual_semantics = match &summary.operator { + let algorithm = match &summary.operator { asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::HLL, - .. - } => Some( + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), + } => Some(kind.algorithm()), + _ => None, + }; + let actual_semantics = match algorithm { + Some(planner_types::post_asap::SketchAlgorithm::Hll) => Some( asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue, ), - asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::UnivMon, - .. - } => Some( + Some(planner_types::post_asap::SketchAlgorithm::UnivMon) => Some( asap_types::erp_observation::ErpObservationInputSemantics::UnitSampleFrequency, ), _ => None, @@ -1468,14 +1468,24 @@ mod tests { // Catalog resolution is independent of Planner selection. Build an // HLL catalog fixture from a compiled source identity; production // selection cannot deploy HLL without a known confidence guarantee. - let mut materialization = plan.precompute_plan.materializations[0].clone(); - materialization.aggregation_type = asap_types::AggregationType::HLL; - materialization.parameters = - serde_json::from_value(serde_json::json!({"precision": 14})).unwrap(); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let materialization = &plan.precompute_plan.materializations[0]; + let hll = planner_types::post_asap::SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( plan.summary_catalog.plan_id, plan.summary_catalog.plan_version, - &[materialization], + vec![( + materialization, + &hll, + plan.precompute_plan + .population_filter(materialization) + .unwrap(), + )], ) .unwrap(); let (mut policy, mut observed) = online_population_fixture(); diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index e1e29a4dc..c7aad57c2 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -1,6 +1,7 @@ use super::compiler::{ derived_window_cost, gcd, retained_state_count, CollectorMaterialization, - MaterializationLifecycleEstimate, PhysicalCompilationRequest, RuntimeRulePolicy, + MaterializationLifecycleEstimate, OutputComputation, PhysicalCompilationRequest, + RuntimeRulePolicy, }; use asap_types::WindowMaterializationLayout; use planner_types::post_asap::{PostAsapNodeId, SummaryWindowFramework}; @@ -17,8 +18,10 @@ pub(super) fn share_additive_panes( bindings: &mut BTreeMap<(usize, PostAsapNodeId), asap_types::PolicyFingerprint>, policies: &mut BTreeMap, estimates: &mut BTreeMap, + computations: &mut BTreeMap, ) { - use asap_types::{AggregationType, WindowKind}; + use asap_types::WindowKind; + use planner_types::post_asap::{ExactKind, SummaryFamilyType}; let derived_sources = materializations .iter() .filter_map(|m| m.derived_input.as_ref()) @@ -34,7 +37,10 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!(m.aggregation_type, AggregationType::Sum) + || !matches!( + computations.get(&old), + Some((SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), _)) + ) { continue; } @@ -87,6 +93,7 @@ pub(super) fn share_additive_panes( canonical.window_type = WindowKind::Tumbling; canonical.window_layout = WindowMaterializationLayout::Pane { pane_secs: 1 }; canonical.pane_origin_ms = Some(0); + canonical.allocate_stored_output_id(&computations[&old]); let key = ( canonical.policy_fingerprint(), serde_json::to_string(&lifecycle).unwrap(), @@ -114,6 +121,8 @@ pub(super) fn share_additive_panes( pane_secs: group.pane_secs, }; canonical.pane_origin_ms = Some(group.origin); + let computation = &computations[&physical[index].0]; + physical[index].1.allocate_stored_output_id(computation); } } let mut target_counts = BTreeMap::new(); @@ -154,6 +163,8 @@ pub(super) fn share_additive_panes( combined.expected_updates = 0.0; combined.lifecycle_cost = group.cost; combined.window_realization_id = format!("shared-pane-{}", new.0); + let computation = computations[&physical[group.members[0]].0].clone(); + computations.insert(new, computation); for index in group.members { let old = physical[index].0; if let Some(estimate) = estimates.remove(&old) { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 8db586aa6..faf45a98a 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -918,26 +918,25 @@ mod catalog_binding_tests { use crate::physical::summary_catalog::SummaryCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - fn fixture() -> (QueryPlan, SummaryCatalog) { + /// A catalog over one `m` output grouped by `job`, with `kind` state. + fn output(kind: AggregationType) -> (PrecomputeMaterialization, SummaryCatalog) { + let family = kind.planner_exact_family().unwrap(); let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + "m", KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, ); config.pane_origin_ms = Some(0); - let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); + config.allocate_stored_output_id(&family); + let catalog = + SummaryCatalog::from_outputs(7, 2, vec![(&config, &family, String::new())]).unwrap(); + (config, catalog) + } + + fn fixture() -> (QueryPlan, SummaryCatalog) { + let (config, catalog) = output(AggregationType::Sum); let entry = QueryPlanEntry { physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, @@ -1062,26 +1061,7 @@ mod catalog_binding_tests { .to_string() .contains("exact counter SDS")); - let mut counter = PrecomputeMaterialization::new( - AggregationType::Increase, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - counter.pane_origin_ms = Some(0); - let counter_catalog = - SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); + let (counter, counter_catalog) = output(AggregationType::Increase); let (mut counter_plan, _) = fixture(); binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); binding(&mut counter_plan).stored_output_reference = counter_catalog diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index bdf5949b4..20c9a0658 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,51 +1,6 @@ -//! Validate stored materialization descriptors against Planner summary families. -//! -//! This projection supports catalog identity and imported state metadata. It is -//! not an execution program. Raw and maintenance execution dispatch directly -//! on the selected post-ASAP DAG payload; the descriptor must agree with it. - -use serde_json::Value; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::key_by_label_names::KeyByLabelNames; -use crate::AggregationType; -use planner_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, - SketchParams, SummaryFamilyType, -}; - -/// Data_plane's typed replacement for -/// `(aggregation_type, aggregation_sub_type, parameters)`: which -/// accumulator to run (`kind`), with what tuning (`params`), and -/// whether it's keyed by a group-by label set (`grouping`). -/// -/// Computed on demand from an [`PrecomputeMaterialization`] via -/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config -/// itself, so there is exactly one source of truth for the fields that -/// feed [`crate::policy_fingerprint::PolicyFingerprint`]. -#[derive(Debug, Clone, PartialEq)] -pub struct AccumulatorSpec { - /// Planner-owned committed summary identity. For sketches this is a - /// validated `SketchKind` (category + algorithm + params), following the - /// ASAP-aware-mapping vocabulary. - pub family: SummaryFamilyType, - /// Physical keyed-state layout, independent of semantic family. - pub grouping: Option, -} - -/// Typed rule for turning an input sample into an accumulator update. -/// -/// This is execution semantics, separate from the summary family: the same -/// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Stored descriptors encode the rule in -/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the -/// runtime does not branch on ad-hoc strings. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SampleUpdateRule { - Value { scale: f64 }, - Count, - CounterDelta { scale: f64 }, -} +//! Planner summary-update domains implemented by the raw accumulator kernels. +//! Stored outputs carry no update semantics; these predicates inspect the +//! bound Planner DAG node's `SummaryUpdate`. /// The implemented raw frequency domain counts occurrences of sample values. pub fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { @@ -74,747 +29,3 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) ) } - -impl PrecomputeMaterialization { - pub fn sample_update_rule(&self) -> SampleUpdateRule { - let scale = self - .parameters - .get("weight_scale") - .and_then(Value::as_f64) - .filter(|scale| scale.is_finite() && *scale > 0.0) - .unwrap_or(1.0); - match self - .parameters - .get("weight_mode") - .or_else(|| self.parameters.get("topk_weight")) - .and_then(Value::as_str) - .map(str::to_ascii_lowercase) - .as_deref() - { - Some("count" | "frequency" | "freq") => SampleUpdateRule::Count, - Some("counter_delta") => SampleUpdateRule::CounterDelta { scale }, - _ => SampleUpdateRule::Value { scale }, - } - } -} - -/// A storage descriptor cannot be resolved to a supported Planner family. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AccumulatorSpecError { - /// `aggregation_type` was `SingleSubpopulation` with an - /// `aggregation_sub_type` string not in the recognized alias list. - UnknownSingleSubpopulationSubType(String), - /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. - UnknownMultipleSubpopulationSubType(String), - /// `aggregation_type` itself has no accumulator-dispatch mapping. - /// Also returned for an invalid HLL precision. A resolved family identifies - /// stored state; it does not imply raw-value updater support. - UnmappedAggregationType(AggregationType), -} - -impl std::fmt::Display for AccumulatorSpecError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnknownSingleSubpopulationSubType(s) => { - write!(f, "Unknown SingleSubpopulation sub_type '{s}'") - } - Self::UnknownMultipleSubpopulationSubType(s) => { - write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") - } - Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), - } - } -} - -impl std::error::Error for AccumulatorSpecError {} - -impl PrecomputeMaterialization { - /// Resolve this config's `(aggregation_type, aggregation_sub_type, - /// parameters)` triple into a typed [`AccumulatorSpec`]. - /// - /// Unsupported descriptors return an error; this projection never chooses - /// a fallback family and cannot authorize DAG execution. - pub fn accumulator_spec(&self) -> Result { - use AggregationType::*; - - let sub_type = self.aggregation_sub_type.as_str(); - - let independent_sketch = |algorithm, params| { - SummaryFamilyType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::PerSubpopulationInstance, - ) - }; - let (family, keyed) = match self.aggregation_type { - Sum | Count | Increase | Rate | Min | Max => ( - self.aggregation_type.planner_exact_family().unwrap(), - !self.aggregated_labels.is_empty(), - ), - DatasketchesKLL => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - HydraKLL => { - let k = kll_k_param(self) as u32; - ( - SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraKll, - params: HydraParams::HydraKll { - k, - shared_buckets: k, - }, - }, - ), - true, - ) - } - CountMinSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountMinSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - CountSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - DDSketch => ( - independent_sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { - alpha: ddsketch_alpha_param(self), - }, - ), - false, - ), - UnivMon => { - let param = - |name: &str, default: u64, max: u64| -> Result { - let value = self - .parameters - .get(name) - .map_or(Some(default), |v| v.as_u64()) - .ok_or(AccumulatorSpecError::UnmappedAggregationType(UnivMon))?; - if value == 0 || value > max { - return Err(AccumulatorSpecError::UnmappedAggregationType(UnivMon)); - } - Ok(value as u32) - }; - ( - independent_sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: param("heap_size", 32, u32::MAX as u64)?, - sketch_rows: param("sketch_rows", 5, 20)?, - sketch_cols: param("sketch_cols", 1024, u32::MAX as u64)?, - layers: param("layers", 4, 64)? as u8, - }, - ), - false, - ) - } - HLL => { - let precision = match self.parameters.get("precision") { - None => 14, - Some(value) => value - .as_u64() - .ok_or(AccumulatorSpecError::UnmappedAggregationType(HLL))?, - }; - if !(4..=18).contains(&precision) { - return Err(AccumulatorSpecError::UnmappedAggregationType(HLL)); - } - ( - independent_sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: precision as u8, - }, - ), - false, - ) - } - SingleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - "DatasketchesKLL" | "datasketches_kll" | "KLL" | "kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - other => { - return Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType( - other.to_string(), - )) - } - }, - MultipleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - "CountMinSketch" | "count_min_sketch" | "CMS" | "cms" => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - "HydraKLL" | "hydra_kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - true, - ), - other => { - return Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType( - other.to_string(), - )) - } - }, - }; - - let grouping = if keyed { - Some(self.grouping_labels.label_names()) - } else { - None - }; - - Ok(AccumulatorSpec { family, grouping }) - } -} - -// --------------------------------------------------------------------------- -// Raw-parameter extraction helpers. -// -// Relocated verbatim from `data_plane::precompute_engine::accumulator_factory` -// (same names, same behavior, same defaults) — this is now their one -// definition; `accumulator_factory.rs` re-exports them via `use` so its -// existing unit tests (`cms_params_reads_canonical_w_d_keys`, -// `test_kll_k_param_capital_k`, ...) keep passing unchanged. -// --------------------------------------------------------------------------- - -/// Extract the KLL `k` parameter. Capital `"K"` takes precedence over -/// lowercase `"k"` to match the convention used by the top-level -/// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { - config - .parameters - .get("K") - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .and_then(|v| u16::try_from(v).ok()) - .unwrap_or(200) -} - -/// Extract `(row_num, col_num)` for CMS / HydraKLL configs. -/// -/// Reads canonical `d` (depth = rows) / `w` (width = cols) keys — -/// matches what the control plane's `sketch_params_to_json` emits and -/// what `sketch_config_to_params` uses for OTLP policy_fp content -/// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { - let row_num = config - .parameters - .get("d") - .and_then(|v| v.as_u64()) - .unwrap_or(4) as usize; - let col_num = config - .parameters - .get("w") - .and_then(|v| v.as_u64()) - .unwrap_or(1000) as usize; - (row_num, col_num) -} - -/// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` -/// from `parameters`; defaults to 20 (the heap holds the top-k -/// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { - config - .parameters - .get("heap_size") - .or_else(|| config.parameters.get("k")) - .or_else(|| config.parameters.get("K")) - .and_then(|v| v.as_u64()) - .map(|v| v as usize) - .filter(|&v| v > 0) - .unwrap_or(20) -} - -/// Pull `relativeAccuracy` (or canonical aliases) out of a -/// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, -/// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { - let parsed = param_f64(config, "relativeAccuracy") - .or_else(|| param_f64(config, "relative_accuracy")) - .or_else(|| param_f64(config, "alpha")) - .unwrap_or(0.01); - if parsed > 0.0 && parsed < 1.0 { - parsed - } else { - tracing::warn!( - "DDSketch relativeAccuracy {} out of (0,1); using default 0.01", - parsed - ); - 0.01 - } -} - -fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { - config.parameters.get(key).and_then(Value::as_f64) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::key_by_label_names::KeyByLabelNames; - use std::collections::HashMap; - - fn assert_exact(spec: &AccumulatorSpec, expected: ExactKind) { - assert!(matches!( - &spec.family, - SummaryFamilyType::ExactAggregate(kind, _) if kind == &expected - )); - } - - fn assert_sketch( - spec: &AccumulatorSpec, - expected_algorithm: planner_types::post_asap::SketchAlgorithm, - expected_params: planner_types::post_asap::SketchParams, - ) { - match &spec.family { - SummaryFamilyType::Sketch(kind, _) => { - assert_eq!(kind.algorithm(), &expected_algorithm); - assert_eq!(kind.params(), &expected_params); - } - other => panic!("expected sketch family, got {other:?}"), - } - } - - #[allow(clippy::too_many_arguments)] - fn make_config( - agg_type: AggregationType, - sub_type: &str, - params: HashMap, - grouping_labels: Vec<&str>, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - sub_type.to_string(), - params, - KeyByLabelNames::new(grouping_labels.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) - } - - // ---- direct (non-wrapper) variants ------------------------------ - - #[test] - fn sum_is_unkeyed_sum() { - let cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - - #[test] - fn keyed_layout_preserves_sum_family() { - let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); - cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert_eq!( - spec.grouping, - Some(KeyByLabelNames::new(vec!["zone".to_string()])) - ); - } - - #[test] - fn datasketches_kll_reads_k_param() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(128)); - let cfg = make_config(AggregationType::DatasketchesKLL, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 128 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hydra_kll_is_keyed_kll() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(64)); - let cfg = make_config(AggregationType::HydraKLL, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 64 }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_maps_to_cms_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(7)); - params.insert("w".to_string(), serde_json::json!(2048)); - let cfg = make_config(AggregationType::CountMinSketch, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Cms, - planner_types::post_asap::SketchParams::Cms { - width: 2048, - depth: 7, - }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_with_heap_maps_to_cms_with_heap_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(4)); - params.insert("w".to_string(), serde_json::json!(256)); - params.insert("heap_size".to_string(), serde_json::json!(8)); - let cfg = make_config( - AggregationType::CountMinSketchWithHeap, - "", - params, - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::CmsWithHeap, - planner_types::post_asap::SketchParams::CmsWithHeap { - width: 256, - depth: 4, - heap_size: 8, - }, - ); - } - - /// Documented existing quirk (see `accumulator_factory.rs`): bare - /// `CountSketch` gets its own `SketchAlgorithm` identity here, but - /// `accumulator_factory` routes it through the same - /// `CmsAccumulatorUpdater` as bare CMS — no dedicated heap-less - /// Count-Sketch accumulator exists. This test locks in the *identity* - /// resolution only; the shared-accumulator behavior is exercised in - /// `accumulator_factory.rs`'s own tests. - #[test] - fn count_sketch_gets_its_own_kind_identity() { - let cfg = make_config(AggregationType::CountSketch, "", HashMap::new(), vec!["h"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CountSketch - )); - } - - #[test] - fn ddsketch_reads_relative_accuracy_alpha() { - let mut params = HashMap::new(); - params.insert("relativeAccuracy".to_string(), serde_json::json!(0.02)); - let cfg = make_config(AggregationType::DDSketch, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::DDSketch, - planner_types::post_asap::SketchParams::DDSketch { alpha: 0.02 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hll_has_catalog_identity_for_sketch_envelope_ingest() { - let cfg = make_config(AggregationType::HLL, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().unwrap(); - assert_sketch( - &spec, - SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, - ); - for precision in [ - serde_json::json!(3), - serde_json::json!(19), - serde_json::json!(4.5), - serde_json::json!("bad"), - ] { - let cfg = make_config( - AggregationType::HLL, - "", - HashMap::from([("precision".into(), precision)]), - vec![], - ); - assert!(cfg.accumulator_spec().is_err()); - } - } - - // ---- wrapper (Single/MultipleSubpopulation) variants ------------ - - #[test] - fn single_subpopulation_sum_alias() { - for alias in ["Sum", "sum"] { - let cfg = make_config( - AggregationType::SingleSubpopulation, - alias, - HashMap::new(), - vec![], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - } - - #[test] - fn multiple_subpopulation_cms_alias() { - for alias in ["CountMinSketch", "count_min_sketch", "CMS", "cms"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Cms - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn multiple_subpopulation_hydra_kll_alias() { - for alias in ["HydraKLL", "hydra_kll"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Kll - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn unknown_single_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::SingleSubpopulation, - "Bogus", - HashMap::new(), - vec![], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn unknown_multiple_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - "Bogus", - HashMap::new(), - vec!["host"], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn error_display_matches_pre_step5_warning_text() { - assert_eq!( - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL'" - ); - } - - #[test] - fn sample_update_rule_decodes_legacy_parameters_once() { - let mut parameters = HashMap::new(); - parameters.insert("weight_mode".into(), Value::String("counter_delta".into())); - parameters.insert("weight_scale".into(), Value::from(1_000_000.0)); - let counter = make_config( - AggregationType::CountMinSketchWithHeap, - "", - parameters, - vec![], - ); - assert_eq!( - counter.sample_update_rule(), - SampleUpdateRule::CounterDelta { scale: 1_000_000.0 } - ); - - let mut count_parameters = HashMap::new(); - count_parameters.insert("topk_weight".into(), Value::String("frequency".into())); - let count = make_config( - AggregationType::CountSketchWithHeap, - "", - count_parameters, - vec![], - ); - assert_eq!(count.sample_update_rule(), SampleUpdateRule::Count); - } - - // ---- PolicyFingerprint stability guard --------------------------- - - /// `accumulator_spec()` must be a pure, additional *read* of - /// `PrecomputeMaterialization` — it must not change what - /// `PolicyFingerprint::from_config` hashes. This locks in a fixed - /// fingerprint for a fixed config as a tripwire: if this test ever - /// needs its expected constant updated, `policy_fingerprint.rs` - /// changed in a way that breaks on-disk sid compatibility, which is - /// exactly what Step 5 was required not to do. - #[test] - fn accumulator_spec_does_not_perturb_policy_fingerprint() { - let cfg = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - let fp_before = cfg.policy_fp_u64(); - let _ = cfg.accumulator_spec(); // takes &self — must not mutate `cfg` - let fp_after = cfg.policy_fp_u64(); - assert_eq!( - fp_before, fp_after, - "calling accumulator_spec() must not change the fingerprint \ - PolicyFingerprint::from_config computes from this config" - ); - - // An independently-built config with identical content must - // still agree — proves accumulator_spec() reads, never writes, - // the fields PolicyFingerprint::from_config hashes. - let cfg2 = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - assert_eq!(fp_after, cfg2.policy_fp_u64()); - } -} diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index f256d74fd..a2ce6dd08 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -1,13 +1,8 @@ use serde::{Deserialize, Serialize}; -use serde_json::Value; -use serde_yaml; -use std::collections::HashMap; -use crate::enums::{QueryLanguage, WindowKind}; +use crate::enums::WindowKind; use crate::policy_fingerprint::PolicyFingerprint; -use crate::utils::normalize_spatial_filter; use crate::AggregationType; -use crate::KeyByLabelNames; /// Physical maintenance layout for one semantic windowed summary. /// @@ -86,21 +81,19 @@ impl WindowMaterializationLayout { } } -/// Physical materialization metadata with content-derived identity. -/// This descriptor cannot authorize execution: the enclosing PrecomputePlan -/// must bind it to a compatible Planner DAG producer. +/// Deployment metadata for one stored output: its identity, source binding, +/// window cadence and layout, retention and storage codec. What the output +/// computes (state family, update, input predicate and reduction) is defined +/// only by the Planner DAG node the enclosing PrecomputePlan binds to it. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PrecomputeMaterialization { - /// Explicit deployment output allocation; independent of semantic identity. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stored_output_id: Option, + /// Deployment output allocation. Several outputs may share one semantic + /// definition; see [`Self::allocate_stored_output_id`]. + pub stored_output_id: crate::sds::StoredOutputId, /// Planner-selected dependency closure ending at the persisted output. #[serde(default, skip_serializing_if = "Option::is_none")] pub semantic_fragment: Option, - pub aggregation_type: AggregationType, - pub aggregation_sub_type: String, - pub parameters: HashMap, #[serde(serialize_with = "crate::grouping_projection::serialize_config_grouping")] pub grouping_labels: crate::GroupingProjection, #[serde( @@ -112,9 +105,6 @@ pub struct PrecomputeMaterialization { pub partitioning: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub derived_input: Option, - pub aggregated_labels: KeyByLabelNames, - pub rollup_labels: KeyByLabelNames, - pub original_yaml: String, pub window_size: u64, // Window size in seconds (e.g., 900s for 15m) pub slide_interval: u64, // Slide/hop interval in seconds (e.g., 30s) @@ -126,8 +116,6 @@ pub struct PrecomputeMaterialization { #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, - pub spatial_filter: String, - pub spatial_filter_normalized: String, pub metric: String, // PromQL mode: metric name; SQL mode: derived from table_name.value_column pub num_aggregates_to_retain: Option, @@ -145,13 +133,11 @@ pub struct PrecomputeMaterialization { /// /// **Why this is separate from [`Self::value_projection`]**: the /// projection is the materialization's *identity* — which column or - /// constant is summarised, and part of the policy fingerprint. This is how - /// the ingest path must *read* that column, which the identity does not - /// determine: an integer column needs an exactness guard on its way into - /// f64 summary state, and a nullable column needs SQL's "aggregates skip - /// NULL" rule reproduced at the reader rather than a decode failure on the - /// first NULL row. Two materializations over the same column are the same - /// policy either way, so this deliberately stays out of the fingerprint. + /// constant is summarised. This is how the ingest path must *read* that + /// column, which the identity does not determine: an integer column needs + /// an exactness guard on its way into f64 summary state, and a nullable + /// column needs SQL's "aggregates skip NULL" rule reproduced at the reader + /// rather than a decode failure on the first NULL row. /// /// `None` ⇒ PromQL-mode materializations and legacy SQL definitions, which /// keep the pre-typed behaviour (read the column as it comes). @@ -163,17 +149,11 @@ pub struct PrecomputeMaterialization { /// query. For single-population queries, key and value share the same /// fingerprint and type. For multi-population queries (e.g. Topk), they /// differ. -/// -/// **PR 5**: the `aggregation_id_for_*: u64` fields now carry the -/// `PolicyFingerprint::as_u64()` form of the matched config, NOT a -/// controller-allocated id. Callers that need typed identity can call -/// [`AggregationIdInfo::policy_fp_for_key`] / -/// [`AggregationIdInfo::policy_fp_for_value`]. #[derive(Debug, Clone)] pub struct AggregationIdInfo { - /// `PolicyFingerprint::as_u64()` of the key aggregation's config. + /// Stored output id of the key aggregation. pub key_policy_fingerprint: u64, - /// `PolicyFingerprint::as_u64()` of the value aggregation's config. + /// Stored output id of the value aggregation. pub value_policy_fingerprint: u64, pub aggregation_type_for_key: AggregationType, pub aggregation_type_for_value: AggregationType, @@ -182,6 +162,43 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} impl PrecomputeMaterialization { + /// An unallocated output over `metric`; the caller must allocate its id + /// with [`Self::allocate_stored_output_id`] once the deployment fields are final. + pub fn new( + metric: impl Into, + grouping_labels: impl Into, + window_size: u64, + slide_interval: u64, + window_type: WindowKind, + ) -> Self { + Self { + stored_output_id: crate::sds::StoredOutputId(0), + semantic_fragment: None, + grouping_labels: grouping_labels.into(), + population_key_encoding: Default::default(), + partitioning: None, + derived_input: None, + window_size, + slide_interval, + window_type, + window_layout: WindowMaterializationLayout::Pane { + pane_secs: if slide_interval == 0 { + window_size + } else { + slide_interval + }, + }, + pane_origin_ms: None, + metric: metric.into(), + num_aggregates_to_retain: None, + table_name: None, + value_projection: None, + table_timestamp_column: None, + table_population: None, + value_source_column: None, + } + } + pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection .as_ref() @@ -214,13 +231,15 @@ impl PrecomputeMaterialization { } } - pub fn population_filter_canonical(&self) -> Result { + /// Validate the source binding. Returns the canonical typed table + /// population, which is part of the table source binding; a time-series + /// predicate belongs to the Planner DAG's scan. + pub fn table_population_canonical(&self) -> Result { if let Some(input) = &self.derived_input { input.validate()?; if self.table_name.is_some() || self.table_population.is_some() || self.table_timestamp_column.is_some() - || !self.spatial_filter.is_empty() { return Err("derived inputs cannot also declare a raw source/filter".into()); } @@ -238,78 +257,15 @@ impl PrecomputeMaterialization { } crate::table_population::validate_column_name(column)?; } - if self.table_name.is_some() && !self.spatial_filter.is_empty() { - return Err("table populations cannot use a PromQL label filter".into()); - } - if let Some(population) = &self.table_population { - if self.table_name.is_none() || !self.spatial_filter.is_empty() { - return Err( - "typed table population requires a table and no PromQL label filter".into(), - ); + match &self.table_population { + Some(population) => { + if self.table_name.is_none() { + return Err("typed table population requires a table".into()); + } + population.validate()?; + Ok(population.canonical()) } - population.validate()?; - Ok(population.canonical()) - } else { - Ok(normalize_spatial_filter(&self.spatial_filter)) - } - } - - #[allow(clippy::too_many_arguments)] - pub fn new( - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: HashMap, - grouping_labels: impl Into, - aggregated_labels: KeyByLabelNames, - rollup_labels: KeyByLabelNames, - original_yaml: String, - window_size: u64, - slide_interval: u64, - window_type: WindowKind, - spatial_filter: String, - metric: String, - num_aggregates_to_retain: Option, - // SQL-specific fields - table_name: Option, - value_column: Option, - ) -> Self { - // Generate normalized spatial filter (placeholder implementation) - let spatial_filter_normalized = normalize_spatial_filter(&spatial_filter); - - Self { - stored_output_id: None, - semantic_fragment: None, - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels: grouping_labels.into(), - population_key_encoding: Default::default(), - partitioning: None, - derived_input: None, - aggregated_labels, - rollup_labels, - original_yaml, - window_size, - slide_interval, - window_type, - window_layout: WindowMaterializationLayout::Pane { - pane_secs: if slide_interval == 0 { - window_size - } else { - slide_interval - }, - }, - pane_origin_ms: None, - spatial_filter, - spatial_filter_normalized, - metric, - num_aggregates_to_retain, - table_name, - value_projection: value_column - .map(|name| crate::sds::ValueProjectionIdentity::Column { name }), - table_population: None, - table_timestamp_column: None, - value_source_column: None, + None => Ok(String::new()), } } @@ -319,223 +275,127 @@ impl PrecomputeMaterialization { PolicyFingerprint::from_config(self) } - /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the - /// policy-fingerprint-keyed call sites (e.g. `InstalledPrecomputePlan`'s - /// `HashMap` keys). **Always** equal to - /// `self.policy_fingerprint().as_u64()`. The value is content- - /// addressed identity, NOT a controller-allocated counter id. + /// The stored output id as a raw u64. pub fn policy_fp_u64(&self) -> u64 { self.policy_fingerprint().as_u64() } - pub fn from_yaml_data( - aggregation_data: &serde_yaml::Value, - num_aggregates_to_retain: Option, - query_language: QueryLanguage, - ) -> Result { - // `aggregationId` is silently dropped — identity is - // content-addressed via PolicyFingerprint (PR 5). Pre-PR-5 - // fixtures that still spell out the field parse cleanly. + /// Allocate the deterministic default id from the current deployment + /// fields and `computation`, the producer's computation identity taken + /// from the Planner DAG (state family, update, input predicate). Outputs + /// with equal deployment fields and computation share one id; any + /// difference, including pane layout or phase, yields a distinct output. + pub fn allocate_stored_output_id(&mut self, computation: &impl Serialize) { + let computation = crate::semantic_fragment::canonical_bytes(computation) + .expect("computation identity serializes"); + let id = self.deployment_digest(&computation); + // Zero is the unallocated sentinel. + self.stored_output_id = crate::sds::StoredOutputId(id.max(1)); + } - let labels = &aggregation_data["labels"]; - let grouping_labels: crate::GroupingProjection = - serde_yaml::from_value(labels["grouping"].clone())?; - let aggregated_labels = KeyByLabelNames::new( - labels["aggregated"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing aggregated labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), + fn deployment_digest(&self, computation: &[u8]) -> u64 { + use xxhash_rust::xxh64::xxh64; + let mut buf: Vec = Vec::with_capacity(512); + if !self.population_key_encoding.is_legacy() { + // UTF-8 raw metrics cannot alias this nonlegacy domain prefix. + buf.extend_from_slice(b"\xffpopulation-key-canonical-labels-v1\0"); + } + if self.derived_input.is_some() { + // Raw policies start with UTF-8 metric bytes; 0xff is impossible + // there, so a metric cannot impersonate this source domain. + buf.extend_from_slice(b"\xffderived-input-v1:"); + buf.extend_from_slice( + &serde_json::to_vec(&self.source_identity()).expect("typed source identity"), + ); + } else { + buf.extend_from_slice(self.metric.as_bytes()); + } + buf.push(0); + buf.extend_from_slice(computation); + buf.push(0); + if let Some(partitioning) = self.partitioning { + buf.extend_from_slice(format!("partition:{partitioning:?}\0").as_bytes()); + } + for label in &self.grouping_labels.names() { + buf.extend_from_slice(label.as_bytes()); + buf.push(b','); + } + buf.push(0); + if self.table_name.is_some() && !self.grouping_labels.is_empty() { + buf.extend_from_slice( + crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS.as_bytes(), + ); + buf.push(0); + } + if !self.grouping_labels.is_legacy_labels() { + buf.extend_from_slice(b"typed-grouping:"); + buf.extend_from_slice( + serde_json::to_string(&self.grouping_labels) + .expect("group projection serializes") + .as_bytes(), + ); + buf.push(0); + } + buf.extend_from_slice(&self.window_size.to_le_bytes()); + buf.push(0); + buf.extend_from_slice(&self.slide_interval.to_le_bytes()); + buf.push(0); + buf.extend_from_slice( + serde_json::to_string(&self.window_type) + .unwrap_or_default() + .as_bytes(), ); - let rollup_labels = KeyByLabelNames::new( - labels["rollup"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing rollup labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), + buf.push(0); + // A full overlapping window and a mergeable pane layout may share + // semantic descriptors but never share payload instances or lifecycle + // accounting. + buf.extend_from_slice( + serde_json::to_string(&self.window_layout) + .unwrap_or_default() + .as_bytes(), ); - - let aggregation_type: AggregationType = aggregation_data["aggregationType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationType"))? - .parse() - .map_err(|e: String| anyhow::anyhow!(e))?; - - let aggregation_sub_type = aggregation_data["aggregationSubType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationSubType"))? - .to_string(); - - // Convert serde_yaml::Value to serde_json::Value for parameters - let parameters: HashMap = aggregation_data["parameters"] - .as_mapping() - .ok_or_else(|| anyhow::anyhow!("Missing parameters"))? - .iter() - .map(|(k, v)| { - let key = k.as_str().unwrap_or("").to_string(); - let value = serde_json::to_value(v).unwrap_or(Value::Null); - (key, value) - }) - .collect(); - - let window_size = aggregation_data["windowSize"] - .as_u64() - .ok_or_else(|| anyhow::anyhow!("Missing windowSize"))?; - - let window_type = aggregation_data - .get("windowType") - .and_then(|v| v.as_str()) - .unwrap_or("tumbling") - .parse::() - .unwrap_or_default(); - - let slide_interval = aggregation_data - .get("slideInterval") - .and_then(|v| v.as_u64()) - .unwrap_or(window_size); - - let pane_origin_ms = aggregation_data - .get("paneOriginMs") - .or_else(|| aggregation_data.get("pane_origin_ms")) - .and_then(|v| v.as_i64()); - - let spatial_filter = aggregation_data["spatialFilter"] - .as_str() - .unwrap_or("") - .to_string(); - - if [ - "valueColumn", - "value_column", - "valueProjection", - "value_projection", - ] - .iter() - .filter(|key| { - aggregation_data - .get(**key) - .is_some_and(|value| !value.is_null()) - }) - .count() - > 1 - { - return Err(anyhow::anyhow!( - "multiple value projection fields are not allowed" - )); + buf.push(0); + // Presence is explicit so an unknown phase cannot alias an + // epoch-aligned definition. + match self.pane_origin_ms { + Some(origin) => { + buf.push(1); + buf.extend_from_slice(&origin.to_le_bytes()); + } + None => buf.push(0), } - let typed_projection: Option = aggregation_data - .get("valueProjection") - .or_else(|| aggregation_data.get("value_projection")) - .filter(|value| !value.is_null()) - .map(|value| serde_json::to_value(value).and_then(serde_json::from_value)) - .transpose()?; - let (metric, table_name, value_column) = match query_language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl => { - let metric = aggregation_data["metric"] - .as_str() - .ok_or_else(|| { - anyhow::anyhow!("Missing metric for time-series query language") - })? - .to_string(); - (metric, None, None) + buf.push(0); + if let Some(table) = &self.table_name { + buf.extend_from_slice(b"\0sql-source-v1\0"); + buf.extend_from_slice(table.as_bytes()); + buf.push(0); + if let Some(column) = self.effective_value_projection().column() { + buf.extend_from_slice(column.as_bytes()); } - QueryLanguage::ClickHouseSql => { - let table = aggregation_data["tableName"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing tableName for ClickHouse SQL"))? - .to_string(); - let column = aggregation_data["valueColumn"] - .as_str() - .or_else(|| { - typed_projection - .as_ref() - .and_then(|projection| projection.column()) - }) - .map(str::to_owned); - if column.is_none() && typed_projection.is_none() { - return Err(anyhow::anyhow!( - "Missing value projection for ClickHouse SQL" - )); - } - ( - format!("{table}.{}", column.as_deref().unwrap_or("constant")), - Some(table), - column, - ) + if matches!( + self.effective_value_projection(), + crate::sds::ValueProjectionIdentity::Constant { .. } + ) { + buf.extend_from_slice(b"\0constant-projection-v1\0"); + buf.extend_from_slice( + serde_json::to_string(self.effective_value_projection()) + .expect("finite validated projection serializes") + .as_bytes(), + ); } - }; - - let mut config = Self::new( - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels, - aggregated_labels, - rollup_labels, - String::new(), // original_yaml - empty as in Python - window_size, - slide_interval, - window_type, - spatial_filter, - metric, - num_aggregates_to_retain, - table_name, - value_column, - ); - if aggregation_data.get("windowLayout").is_some() - && aggregation_data.get("window_layout").is_some() - { - anyhow::bail!("multiple window layout fields are not allowed"); } - if let Some(layout) = aggregation_data - .get("windowLayout") - .or_else(|| aggregation_data.get("window_layout")) - { - config.window_layout = serde_yaml::from_value(layout.clone())?; - config - .window_layout - .validate(config.window_size, config.slide_interval) - .map_err(anyhow::Error::msg)?; + if let Some(population) = &self.table_population { + let canonical = population.canonical(); + if !canonical.is_empty() { + buf.push(0); + buf.extend_from_slice(canonical.as_bytes()); + } } - config.population_key_encoding = aggregation_data - .get("population_key_encoding") - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()? - .unwrap_or_default(); - config.derived_input = aggregation_data - .get("derived_input") - .filter(|v| !v.is_null()) - .map(|v| serde_yaml::from_value(v.clone())) - .transpose()?; - config.partitioning = aggregation_data - .get("partitioning") - .filter(|value| !value.is_null()) - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()?; - if let Some(projection) = typed_projection { - config.value_projection = Some(projection); + if let Some(column) = &self.table_timestamp_column { + buf.extend_from_slice(b"\0timestamp-ms\0"); + buf.extend_from_slice(column.as_bytes()); } - config.pane_origin_ms = pane_origin_ms; - config.table_timestamp_column = aggregation_data - .get("tableTimestampColumn") - .or_else(|| aggregation_data.get("table_timestamp_column")) - .and_then(serde_yaml::Value::as_str) - .map(str::to_owned); - config.table_population = aggregation_data - .get("tablePopulation") - .or_else(|| aggregation_data.get("table_population")) - .filter(|value| !value.is_null()) - .cloned() - .map(serde_yaml::from_value) - .transpose()?; - config - .population_filter_canonical() - .map_err(anyhow::Error::msg)?; - Ok(config) + xxh64(&buf, 0) } } @@ -575,68 +435,22 @@ mod window_layout_tests { mod tests { use super::*; - fn sample_yaml(with_id: bool) -> serde_yaml::Value { - let id_line = if with_id { "aggregationId: 42\n" } else { "" }; - let yaml = format!( - "{id_line}aggregationType: DDSketch\naggregationSubType: ''\nmetric: http_latency_ms\nlabels:\n grouping: [zone]\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n", - id_line = id_line - ); - serde_yaml::from_str(&yaml).expect("yaml parses") - } - - /// PR 5: `aggregationId` in the YAML is silently dropped — identity - /// is derived from content. A fixture with the field parses to the - /// SAME config as a fixture without it. - #[test] - fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(true), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - let without = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - assert_eq!( - with.policy_fingerprint(), - without.policy_fingerprint(), - "explicit aggregationId in the YAML must not change the policy fingerprint", - ); - } - - /// Round-tripping the same content yields the same fingerprint. - #[test] - fn fingerprint_is_deterministic_per_content() { - let a = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse a"); - let b = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse b"); - assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); - assert_ne!( - a.policy_fingerprint().as_u64(), - 0, - "fingerprint is never the 0 sentinel for a real config", + fn config() -> PrecomputeMaterialization { + let mut config = PrecomputeMaterialization::new( + "http_latency_ms", + crate::KeyByLabelNames::new(vec!["zone".into()]), + 60, + 10, + WindowKind::Sliding, ); + config.allocate_stored_output_id(&"ddsketch"); + config } + // Window layout and pane origin survive the canonical JSON wire, and + // derived-style spellings are rejected. #[test] - fn explicit_window_layout_survives_custom_json_and_yaml_transport() { - use super::WindowMaterializationLayout; - let mut yaml = sample_yaml(false); - yaml["windowSize"] = serde_yaml::to_value(60).unwrap(); - yaml["slideInterval"] = serde_yaml::to_value(10).unwrap(); + fn window_layout_and_pane_origin_survive_canonical_json() { for layout in [ WindowMaterializationLayout::FullWindow, WindowMaterializationLayout::Pane { pane_secs: 5 }, @@ -645,110 +459,39 @@ mod tests { levels_secs: vec![10, 30], }, ] { - yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) - .unwrap(); - assert_eq!(config.window_layout, layout); + let mut config = config(); + config.window_layout = layout.clone(); + config.pane_origin_ms = Some(7_000); let mut wire = serde_json::to_value(&config).unwrap(); let decoded: PrecomputeMaterialization = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(decoded.window_layout, layout); + assert_eq!(decoded.pane_origin_ms, Some(7_000)); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); wire["windowLayout"] = wire["window_layout"].clone(); assert!(serde_json::from_value::(wire).is_err()); } - yaml.as_mapping_mut() - .unwrap() - .remove(serde_yaml::Value::from("windowLayout")); - let legacy = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); - assert_eq!( - legacy.window_layout, - WindowMaterializationLayout::Pane { pane_secs: 10 } - ); - yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!( - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() - ); } - #[test] - fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let unknown = epoch.policy_fingerprint(); - epoch.pane_origin_ms = Some(7_000); - let planned = epoch.policy_fingerprint(); - assert_ne!(unknown, planned); - - let wire = serde_json::to_value(&epoch).unwrap(); - assert_eq!(wire["pane_origin_ms"], serde_json::json!(7_000)); - let mut derived = serde_json::to_value(&epoch).unwrap(); - let origin = derived - .as_object_mut() - .unwrap() - .remove("pane_origin_ms") - .unwrap(); - derived - .as_object_mut() - .unwrap() - .insert("paneOriginMs".into(), origin); - assert!(serde_json::from_value::(derived).is_err()); - let decoded: PrecomputeMaterialization = - serde_json::from_value(serde_json::to_value(&epoch).unwrap()).unwrap(); - assert_eq!(decoded.pane_origin_ms, Some(7_000)); - } - - /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); - } - - /// Installed materializations reject the removed externally assigned identity. + // The removed externally assigned identity is not accepted on the wire. #[test] fn canonical_wire_rejects_aggregation_id() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let mut json = serde_json::to_value(&cfg).unwrap(); - assert!( - json.get("aggregationId").is_none(), - "aggregationId must not appear in the canonical wire format" - ); + let mut json = serde_json::to_value(config()).unwrap(); + assert!(json.get("aggregationId").is_none()); json["aggregationId"] = serde_json::json!(42); assert!(serde_json::from_value::(json).is_err()); } + // Table value projections are typed, and a non-finite constant is invalid. #[test] fn typed_projection_roundtrips_and_untyped_column_is_rejected() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .unwrap(); + let mut config = config(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), }); - let column_identity = config.policy_fingerprint(); let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); @@ -759,34 +502,16 @@ mod tests { config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), }); - let wire = serde_json::to_value(&config).unwrap(); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); - assert_eq!( - decoded.effective_value_projection(), - config.effective_value_projection() - ); - let mut yaml = sample_yaml(false); - yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); - yaml["valueProjection"] = - serde_yaml::to_value(serde_json::to_value(&config.value_projection).unwrap()).unwrap(); - let decoded = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) - .unwrap(); + let decoded: PrecomputeMaterialization = + serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap(); assert_eq!( decoded.effective_value_projection(), config.effective_value_projection() ); - yaml["valueColumn"] = serde_yaml::to_value("other_column").unwrap(); - assert!(PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - QueryLanguage::ClickHouseSql - ) - .is_err()); - assert_ne!(config.policy_fingerprint(), column_identity); + assert!(config.table_population_canonical().is_ok()); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Float64(f64::NAN), }); - assert!(config.population_filter_canonical().is_err()); + assert!(config.table_population_canonical().is_err()); } } diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ebcc60d21..71d70e615 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -1,2 +1,36 @@ //! Kernel identity of backend-stored summary state. pub use asap_summary_state::AggregationType; + +use planner_types::post_asap::{ExactKind, GroupingStrategy, SketchAlgorithm, SummaryFamilyType}; + +/// The backend kernel tag that stores state of the Planner `family`. `None` +/// for families the backend has no stored-state kernel for. +pub fn aggregation_type_for_family(family: &SummaryFamilyType) -> Option { + Some(match family { + SummaryFamilyType::ExactAggregate(kind, _) => match kind { + ExactKind::Sum => AggregationType::Sum, + ExactKind::Count => AggregationType::Count, + ExactKind::Min => AggregationType::Min, + ExactKind::Max => AggregationType::Max, + ExactKind::Increase => AggregationType::Increase, + ExactKind::Rate => AggregationType::Rate, + ExactKind::IRate => return None, + }, + SummaryFamilyType::Sketch(kind, grouping) => match (kind.algorithm(), grouping) { + (SketchAlgorithm::Kll, GroupingStrategy::SharedMultiSubpopulation { .. }) => { + AggregationType::HydraKLL + } + (_, GroupingStrategy::SharedMultiSubpopulation { .. }) => return None, + (SketchAlgorithm::Kll, _) => AggregationType::DatasketchesKLL, + (SketchAlgorithm::Cms, _) => AggregationType::CountMinSketch, + (SketchAlgorithm::CmsWithHeap, _) => AggregationType::CountMinSketchWithHeap, + (SketchAlgorithm::CountSketch, _) => AggregationType::CountSketch, + (SketchAlgorithm::CountSketchWithHeap, _) => AggregationType::CountSketchWithHeap, + (SketchAlgorithm::DDSketch, _) => AggregationType::DDSketch, + (SketchAlgorithm::Hll, _) => AggregationType::HLL, + (SketchAlgorithm::UnivMon, _) => AggregationType::UnivMon, + (SketchAlgorithm::Kmv | SketchAlgorithm::Theta, _) => return None, + }, + _ => return None, + }) +} diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index ffd1edea0..d7a79de6a 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -138,25 +138,40 @@ impl DerivedInputIdentity { mod tests { use super::*; use crate::summary_catalog::SummaryCatalog; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use crate::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + } + + fn allocated(mut config: PrecomputeMaterialization) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&"sum"); + config + } fn config() -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + allocated(PrecomputeMaterialization::new( + "m", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, + )) + } + + fn catalog( + plan_version: u64, + outputs: &[PrecomputeMaterialization], + ) -> Result { + let family = sum(); + SummaryCatalog::from_outputs( + 1, + plan_version, + outputs + .iter() + .map(|c| (c, &family, String::new())) + .collect(), ) } @@ -169,15 +184,18 @@ mod tests { inputs: BTreeSet::from([raw_id]), program_sha256: "a".repeat(64), }); + let derived = allocated(derived); assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); - let a = - SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); - let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); + let a = catalog(1, &[raw.clone(), derived.clone()]).unwrap(); + let b = catalog(9, &[raw, derived.clone()]).unwrap(); assert_eq!(a.outputs, b.outputs); assert_eq!(a.data_descriptors, b.data_descriptors); let mut renamed = derived.clone(); renamed.metric = "output_alias".into(); - assert_eq!(renamed.policy_fingerprint(), derived.policy_fingerprint()); + assert_eq!( + allocated(renamed).policy_fingerprint(), + derived.policy_fingerprint() + ); let json = serde_json::to_value(&derived).unwrap(); let decoded: PrecomputeMaterialization = serde_json::from_value(json).unwrap(); assert_eq!(decoded.policy_fingerprint(), derived.policy_fingerprint()); @@ -190,13 +208,17 @@ mod tests { inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "d".repeat(64), }); + let derived = allocated(derived); let mut raw = derived.clone(); raw.metric = format!( "derived-input-v1:{}", serde_json::to_string(&derived.source_identity()).unwrap() ); raw.derived_input = None; - assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + assert_ne!( + allocated(raw).policy_fingerprint(), + derived.policy_fingerprint() + ); } #[test] @@ -206,9 +228,11 @@ mod tests { inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "b".repeat(64), }); - assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); + let derived = allocated(derived); + assert!(catalog(1, std::slice::from_ref(&derived)).is_err()); + let mut derived = derived; derived.table_name = Some("table".into()); - assert!(derived.population_filter_canonical().is_err()); + assert!(derived.table_population_canonical().is_err()); } fn program(source: u32, root: u32) -> OwnedPostAsapDag { use crate::executable_plan::{OwnedPostAsapEdge, OwnedPostAsapNode}; @@ -322,7 +346,7 @@ mod tests { 1, [( config.policy_fingerprint(), - SummaryDescriptor::from_config(&config).unwrap(), + SummaryDescriptor::from_family(&sum()).unwrap(), data, )] ) @@ -342,12 +366,16 @@ mod tests { generated_at_unix_ms: 0, activation_unix_ms: 0, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: crate::precompute_plan::BACKEND_COMPAT.into(), planner_revision: "test".into(), capability_snapshot_id: "test".into(), }; - let error = - PrecomputePlan::build(envelope, vec![config], &["producer".into()]).unwrap_err(); + let error = PrecomputePlan::build( + envelope, + vec![(allocated(config), sum())], + &["producer".into()], + ) + .unwrap_err(); assert!(error .to_string() .contains("installed aligned immutable maintenance DAG")); diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index 5cd902b7d..134e961a4 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -3,7 +3,7 @@ //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` //! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, -//! `PolicyRegistry`, `capability_matching`) is its real center of +//! `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime //! index key than a planning IR node — ASAPController's diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 2275c0a4a..7c19837bd 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -12,10 +12,8 @@ pub mod monitor_spec; pub mod physical_plan_codec; pub mod plan_publication; pub mod policy_fingerprint; -pub mod policy_registry; pub mod producer_plan; pub mod query_requirements; -pub mod routing_index; pub mod sds; pub mod semantic_fragment; pub mod storage_backend; @@ -24,18 +22,15 @@ pub mod summary_semantics; pub mod table_population; pub mod utils; -pub use accumulator_spec::{AccumulatorSpec, AccumulatorSpecError, SampleUpdateRule}; pub use accuracy::{AccuracyKind, AccuracyProfile}; pub use aggregation_config::*; -pub use aggregation_type::AggregationType; +pub use aggregation_type::{aggregation_type_for_family, AggregationType}; pub use enums::*; pub use grouping_projection::{GroupingProjection, PopulationKeyEncoding}; pub use key_by_label_names::KeyByLabelNames; pub use monitor_spec::{MonitorFunctional, MonitorSpec}; pub use policy_fingerprint::PolicyFingerprint; -pub use policy_registry::PolicyRegistry; pub use query_requirements::*; -pub use routing_index::RoutingIndex; pub use storage_backend::*; pub mod precompute_plan; diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index 42fa2e475..69060cdcf 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -1,16 +1,11 @@ //! Legacy routing wrapper for a deployed stored output. //! -//! An explicit `PrecomputeMaterialization::stored_output_id` takes precedence. -//! Otherwise the compiler allocates a deterministic default from the existing -//! policy fields (including pane layout and cadence). This identifier is not -//! semantic identity: `SummaryDefinitionId` hashes the versioned semantic -//! definition, and several deployed outputs may share that definition. -//! Catalog installation checks that an output is never assigned conflicting -//! computation or format contracts. +//! The value is `PrecomputeMaterialization::stored_output_id`, allocated by the +//! compiler from the deployment fields and the Planner producer's computation +//! identity. It is not semantic identity: `SummaryDefinitionId` hashes the +//! versioned semantic definition, and several deployed outputs may share it. use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use xxhash_rust::xxh64::xxh64; use crate::aggregation_config::PrecomputeMaterialization; @@ -22,178 +17,17 @@ use crate::aggregation_config::PrecomputeMaterialization; pub struct PolicyFingerprint(pub u64); impl PolicyFingerprint { - /// Unset sentinel returned by `Default`. Sinks cannot resolve an unset policy; - /// construct valid fingerprints with [`Self::from_config`]. + /// Unset sentinel returned by `Default`. Sinks cannot resolve an unset policy. pub const UNSET: PolicyFingerprint = PolicyFingerprint(0); /// True when this fingerprint is the [`Self::UNSET`] sentinel. pub fn is_unset(self) -> bool { self.0 == 0 } -} -impl PolicyFingerprint { - /// Compute the fingerprint of an [`PrecomputeMaterialization`]. - /// - /// Hash inputs are concatenated with `\0` byte separators and - /// canonicalized so that map/iteration order can't affect the - /// outcome. Parameter values are rendered via `serde_json::to_string` - /// for nested-shape determinism (matches the existing - /// `parameters_canonical` form used in `AggKind::ExactAgg`). + /// The stored output id of `cfg`. pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { - if let Some(output) = cfg.stored_output_id { - return output.fingerprint(); - } - let mut buf: Vec = Vec::with_capacity(512); - - if !cfg.population_key_encoding.is_legacy() { - // UTF-8 raw metrics cannot alias this nonlegacy domain prefix. - buf.extend_from_slice(b"\xffpopulation-key-canonical-labels-v1\0"); - } - // 1. metric name - if cfg.derived_input.is_some() { - // Raw policies start with UTF-8 metric bytes; 0xff is impossible - // there, so a metric cannot impersonate this source domain. - buf.extend_from_slice(b"\xffderived-input-v1:"); - buf.extend_from_slice( - &serde_json::to_vec(&cfg.source_identity()).expect("typed source identity"), - ); - } else { - buf.extend_from_slice(cfg.metric.as_bytes()); - } - buf.push(0); - - // 2. aggregation_type (Serialize impl is the stable form) - buf.extend_from_slice( - serde_json::to_string(&cfg.aggregation_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 3. aggregation_sub_type - buf.extend_from_slice(cfg.aggregation_sub_type.as_bytes()); - buf.push(0); - - // 4. parameters — canonicalized (sorted keys, JSON-rendered values) - let sorted: BTreeMap<&String, &serde_json::Value> = cfg.parameters.iter().collect(); - for (k, v) in sorted { - buf.extend_from_slice(k.as_bytes()); - buf.push(b'='); - buf.extend_from_slice(serde_json::to_string(v).unwrap_or_default().as_bytes()); - buf.push(b';'); - } - buf.push(0); - - if let Some(partitioning) = cfg.partitioning { - buf.extend_from_slice(format!("partition:{partitioning:?}\0").as_bytes()); - } - - // 5. grouping_labels (already sorted at construction per - // KeyByLabelNames invariant; encode as `,`-joined list) - for l in &cfg.grouping_labels.names() { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - if cfg.table_name.is_some() && !cfg.grouping_labels.is_empty() { - buf.extend_from_slice( - crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS.as_bytes(), - ); - buf.push(0); - } - if !cfg.grouping_labels.is_legacy_labels() { - buf.extend_from_slice(b"typed-grouping:"); - buf.extend_from_slice( - serde_json::to_string(&cfg.grouping_labels) - .expect("group projection serializes") - .as_bytes(), - ); - buf.push(0); - } - - // 6. aggregated_labels - for l in &cfg.aggregated_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 7. rollup_labels - for l in &cfg.rollup_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 8. window_size + slide_interval + window_type (cadence) - buf.extend_from_slice(&cfg.window_size.to_le_bytes()); - buf.push(0); - buf.extend_from_slice(&cfg.slide_interval.to_le_bytes()); - buf.push(0); - buf.extend_from_slice( - serde_json::to_string(&cfg.window_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // Physical representation is part of state identity. A full overlapping - // window and a mergeable pane layout may share semantic descriptors but - // never share payload instances or lifecycle accounting. - buf.extend_from_slice( - serde_json::to_string(&cfg.window_layout) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 9. pane origin. Presence is explicit so a legacy definition with - // unknown phase cannot alias an epoch-aligned definition. - match cfg.pane_origin_ms { - Some(origin) => { - buf.push(1); - buf.extend_from_slice(&origin.to_le_bytes()); - } - None => buf.push(0), - } - buf.push(0); - - // 10. spatial_filter_normalized — canonicalized predicate - buf.extend_from_slice(cfg.spatial_filter_normalized.as_bytes()); - if let Some(table) = &cfg.table_name { - buf.extend_from_slice(b"\0sql-source-v1\0"); - buf.extend_from_slice(table.as_bytes()); - buf.push(0); - if let Some(column) = cfg.effective_value_projection().column() { - buf.extend_from_slice(column.as_bytes()); - } - if matches!( - cfg.effective_value_projection(), - crate::sds::ValueProjectionIdentity::Constant { .. } - ) { - buf.extend_from_slice(b"\0constant-projection-v1\0"); - buf.extend_from_slice( - serde_json::to_string(cfg.effective_value_projection()) - .expect("finite validated projection serializes") - .as_bytes(), - ); - } - } - if let Some(population) = &cfg.table_population { - let canonical = population.canonical(); - if !canonical.is_empty() { - buf.push(0); - buf.extend_from_slice(canonical.as_bytes()); - } - } - if let Some(column) = &cfg.table_timestamp_column { - buf.extend_from_slice(b"\0timestamp-ms\0"); - buf.extend_from_slice(column.as_bytes()); - } - - Self(xxh64(&buf, 0)) + cfg.stored_output_id.fingerprint() } /// The raw u64. Use sparingly — prefer comparing `PolicyFingerprint` @@ -215,55 +49,40 @@ impl std::fmt::Display for PolicyFingerprint { mod tests { use super::*; use crate::enums::WindowKind; - use crate::AggregationType; use crate::KeyByLabelNames; - use std::collections::HashMap; - fn cfg( - metric: &str, - agg_type: AggregationType, - params: HashMap, - group_by: Vec<&str>, - window_size: u64, - spatial_filter: &str, - ) -> PrecomputeMaterialization { + fn cfg(metric: &str, group_by: Vec<&str>, window_size: u64) -> PrecomputeMaterialization { PrecomputeMaterialization::new( - agg_type, - String::new(), - params, + metric, KeyByLabelNames::new(group_by.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), window_size, window_size, WindowKind::Tumbling, - spatial_filter.to_string(), - metric.to_string(), - None, - None, - None, ) } + fn allocated( + mut config: PrecomputeMaterialization, + computation: &str, + ) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&computation); + config + } + + // The canonical wire keeps the legacy encoding implicit, and the + // population key encoding separates allocated output identities. #[test] fn population_encoding_preserves_legacy_wire_and_separates_identity() { use crate::grouping_projection::PopulationKeyEncoding; - let legacy = cfg( - "m", - AggregationType::Sum, - HashMap::new(), - vec!["host"], - 60, - "", - ); + let legacy = allocated(cfg("m", vec!["host"], 60), "sum"); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); - let mut canonical = legacy.clone(); + let mut canonical = cfg("m", vec!["host"], 60); canonical.population_key_encoding = PopulationKeyEncoding::CanonicalLabelsV1; + let canonical = allocated(canonical, "sum"); assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); @@ -271,234 +90,52 @@ mod tests { assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); } + // Equal deployment fields and computation allocate the same output. #[test] - fn same_config_yields_same_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn physical_window_layout_is_part_of_state_identity() { - let panes = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let mut full = panes.clone(); - full.window_layout = crate::WindowMaterializationLayout::FullWindow; - assert_ne!( - PolicyFingerprint::from_config(&panes), - PolicyFingerprint::from_config(&full) - ); - } - - #[test] - fn different_metric_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "cpu_pct", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_window_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 300, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_spatial_filter_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200""#, - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_group_by_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone", "service"], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was - /// excluded from the fingerprint hash. PR 5 deletes the field - /// entirely — identity *is* the fingerprint — so this is now - /// vacuously true. Kept as a doc-comment anchor; no runtime test - /// is needed. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_eq!( - a.policy_fp_u64(), - PolicyFingerprint::from_config(&a).as_u64(), - ); + fn same_deployment_and_computation_yield_same_output() { + let a = allocated(cfg("http_lat", vec!["zone"], 60), "sum"); + let b = allocated(cfg("http_lat", vec!["zone"], 60), "sum"); + assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); + assert!(!a.policy_fingerprint().is_unset()); } + // Any deployment or computation difference allocates a distinct output. #[test] - fn num_aggregates_to_retain_does_not_affect_fingerprint() { - let mut a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let mut b = a.clone(); - a.num_aggregates_to_retain = Some(100); - b.num_aggregates_to_retain = Some(500); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b), - "retention is a separate concern from policy identity" - ); + fn deployment_and_computation_differences_yield_distinct_outputs() { + let base = allocated(cfg("m", vec!["zone"], 60), "sum"); + let mut panes = cfg("m", vec!["zone"], 60); + panes.window_layout = crate::WindowMaterializationLayout::FullWindow; + let mut phased = cfg("m", vec!["zone"], 60); + phased.pane_origin_ms = Some(7_000); + for other in [ + allocated(cfg("other", vec!["zone"], 60), "sum"), + allocated(cfg("m", vec!["zone"], 30), "sum"), + allocated(cfg("m", vec!["host"], 60), "sum"), + allocated(cfg("m", vec!["zone"], 60), "count"), + allocated(panes, "sum"), + allocated(phased, "sum"), + ] { + assert_ne!(base.policy_fingerprint(), other.policy_fingerprint()); + } } + // Retention is not part of an output's identity. #[test] - fn parameter_map_order_does_not_affect_fingerprint() { - // HashMap iteration order is non-deterministic; the fingerprint - // must be order-independent. - let mut p1 = HashMap::new(); - p1.insert("a".into(), serde_json::json!(1)); - p1.insert("b".into(), serde_json::json!(2)); - let mut p2 = HashMap::new(); - p2.insert("b".into(), serde_json::json!(2)); - p2.insert("a".into(), serde_json::json!(1)); - let a = cfg("http_lat", AggregationType::Sum, p1, vec![], 60, ""); - let b = cfg("http_lat", AggregationType::Sum, p2, vec![], 60, ""); + fn retention_does_not_affect_output_identity() { + let a = allocated(cfg("m", vec![], 60), "sum"); + let mut b = cfg("m", vec![], 60); + b.num_aggregates_to_retain = Some(100); assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) + a.policy_fingerprint(), + allocated(b, "sum").policy_fingerprint() ); } #[test] - fn spatial_filter_canonicalization_drives_fingerprint() { - // Two filters that differ only in matcher ordering produce the - // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `PrecomputeMaterialization::new` (via - // `normalize_spatial_filter`) sorts matchers by key. - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200",zone="us""#, - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"zone="us",status="200""#, - ); + fn display_format_is_hex_with_prefix() { assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) + format!("{}", PolicyFingerprint(0xdead_beef)), + "policy_fp:00000000deadbeef" ); } - - #[test] - fn display_format_is_hex_with_prefix() { - let fp = PolicyFingerprint(0xdeadbeef); - assert_eq!(format!("{}", fp), "policy_fp:00000000deadbeef"); - } } diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs deleted file mode 100644 index 90138b139..000000000 --- a/crates/asap_types/src/policy_registry.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Content-addressed policy registry. -//! -//! Derived view over a collection of `PrecomputeMaterialization`s that maps -//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the -//! merged-sid-identity-chain replacement for the controller-allocated -//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `InstalledPrecomputePlan` -//! carries (see `data_plane::storage_engines::types::installed_precompute_plan`'s -//! module doc for why that type lives there, not here). -//! -//! ## Dual-keyed transition -//! -//! PR 3 (where this lives): the registry exists alongside the -//! `aggregation_id`-keyed map. Callers can opt into either lookup. -//! Construction is `O(N)` over the source configs; computation is -//! pure (no I/O, no mutation of the source). -//! -//! Subsequent PRs migrate one set of callers at a time off -//! `aggregation_id` → `PolicyFingerprint`, until the final PR can -//! delete the legacy index. -//! -//! ## Identity invariants -//! -//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` -//! ARE the same policy. The registry treats this as a *deduplication* -//! invariant — if two distinct entries in the source `materializations_by_output` -//! map produce the same fingerprint, the later one wins (last-write -//! semantics). In practice the source should never contain duplicates; -//! if it does, that's a control-plane bug worth surfacing in telemetry -//! (see [`PolicyRegistry::from_configs_with_collisions`]). - -use std::collections::HashMap; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; - -/// Content-addressed lookup table for active aggregation policies. -#[derive(Debug, Clone, Default)] -pub struct PolicyRegistry { - policies: HashMap, -} - -impl PolicyRegistry { - /// Construct from a list of configs. Duplicates (same fingerprint) - /// collapse to the last entry; use - /// [`Self::from_configs_with_collisions`] when you want to detect - /// them. - pub fn from_configs(configs: I) -> Self - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - for cfg in configs { - policies.insert(PolicyFingerprint::from_config(&cfg), cfg); - } - Self { policies } - } - - #[cfg(test)] - /// Construct + report the count of duplicate fingerprints (entries - /// where the source contained two configs producing the same - /// fingerprint and the later one displaced the earlier). Zero in - /// the well-formed case; non-zero is a control-plane bug worth - /// surfacing. - pub fn from_configs_with_collisions(configs: I) -> (Self, usize) - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - let mut collisions = 0usize; - for cfg in configs { - let fp = PolicyFingerprint::from_config(&cfg); - if policies.insert(fp, cfg).is_some() { - collisions += 1; - } - } - (Self { policies }, collisions) - } - - /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.policies.get(&fp) - } - - /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { - self.policies.iter() - } - - /// Live policy count. - pub fn len(&self) -> usize { - self.policies.len() - } - - pub fn is_empty(&self) -> bool { - self.policies.is_empty() - } - - /// All fingerprints currently registered. Useful for diffing two - /// registries during a hot-reload swap. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.policies.keys().copied() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is derived from - // content. Kept as a parameter so existing call sites in the - // tests below don't churn. - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn from_configs_round_trips_lookup() { - let a = cfg(1, "http_lat"); - let b = cfg(2, "cpu_pct"); - let fp_a = PolicyFingerprint::from_config(&a); - let fp_b = PolicyFingerprint::from_config(&b); - let reg = PolicyRegistry::from_configs(vec![a.clone(), b.clone()]); - assert_eq!(reg.len(), 2); - assert_eq!(reg.get(fp_a).unwrap().metric, "http_lat"); - assert_eq!(reg.get(fp_b).unwrap().metric, "cpu_pct"); - } - - #[test] - fn identical_configs_collapse_to_one_entry() { - // PR 5: identity IS the content. Two configs that are - // byte-for-byte identical on the policy-relevant fields - // collapse to ONE entry — there's no way to distinguish them - // anymore. - let a = cfg(1, "http_lat"); - let b = cfg(99, "http_lat"); - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 1); - assert_eq!(collisions, 1); - } - - #[test] - fn distinct_policies_keep_distinct_entries() { - let a = cfg(1, "http_lat"); - let b = cfg(1, "cpu_pct"); // different metric → distinct policies - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 2); - assert_eq!(collisions, 0); - } -} diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 3da190514..25d96a267 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -3,7 +3,7 @@ mod catalog; -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; +use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; use planner_types::pre_asap::Source; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -87,7 +87,9 @@ pub fn validated_source_window_cohort<'a>( Ok(sources.to_vec()) } -pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; +/// Installed-plan schema. v2 materializations carry only deployment fields; +/// v1 plans duplicated computation semantics and are rejected on decode. +pub const BACKEND_COMPAT: &str = "asap-query-backend.v2"; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PlanEnvelope { @@ -105,7 +107,7 @@ pub struct PlanEnvelope { /// edges define execution; materializations attach storage/window placement. /// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths /// execute through the maintenance DAG scheduler at stored-state frontiers. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, @@ -119,6 +121,42 @@ pub struct PrecomputePlan { pub executable_dags: BTreeMap, } +/// Serde mirror of [`PrecomputePlan`]; the compiler checks it stays in sync. +#[derive(Deserialize)] +#[serde(remote = "PrecomputePlan")] +struct PrecomputePlanDef { + #[serde(default)] + summary_catalog: Option, + envelope: PlanEnvelope, + ingest: IngestContract, + schemas: Vec, + producers: Vec, + materializations: Vec, + #[serde(default)] + executable_dags: BTreeMap, +} + +impl<'de> Deserialize<'de> for PrecomputePlan { + /// Check the schema before the body so an older plan fails with its + /// version rather than with its first unrecognized field. + fn deserialize>(deserializer: D) -> Result { + use serde::de::Error; + let value = serde_json::Value::deserialize(deserializer)?; + let compat = value + .get("envelope") + .and_then(|envelope| envelope.get("backend_compat")) + .and_then(serde_json::Value::as_str); + if compat != Some(BACKEND_COMPAT) { + return Err(D::Error::custom(format!( + "unsupported installed precompute plan schema {}: this backend requires \ + {BACKEND_COMPAT}; recompile the plan", + compat.unwrap_or("") + ))); + } + PrecomputePlanDef::deserialize(value).map_err(D::Error::custom) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum IngestProtocol { @@ -218,7 +256,9 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { } } -/// Decoder/schema contract for one content-addressed materialization. +/// Decoder/schema contract for one stored output. `family` is the stored +/// state's type, which readers need without executing the producer; when a +/// Planner DAG node produces the output, validation requires its family. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StateSchemaContract { @@ -226,7 +266,7 @@ pub struct StateSchemaContract { pub schema_id: String, pub schema_version: u32, pub materialization: crate::sds::StoredOutputId, - pub family: StateFamilyContract, + pub family: SummaryFamilyType, pub source: Source, pub value_projection: crate::sds::ValueProjectionIdentity, pub group_by: crate::GroupingProjection, @@ -274,6 +314,11 @@ pub enum PrecomputePlanError { InvalidSchema { schema_id: String }, #[error("materialization {0} uses a summary family unsupported by the runtime schema")] UnsupportedFamily(u64), + #[error("materialization {materialization} has no consistent Planner computation: {reason}")] + InvalidComputation { + materialization: u64, + reason: String, + }, #[error("materialization {materialization} has an invalid window layout: {reason}")] InvalidWindowLayout { materialization: u64, @@ -288,36 +333,29 @@ pub enum PrecomputePlanError { } impl PrecomputePlan { + /// Each output is paired with its stored state family. pub fn build( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, producer_ids: &[String], ) -> Result { - Self::build_complete( - envelope, - materializations, - producer_ids, - false, - BTreeMap::new(), - ) + Self::build_complete(envelope, outputs, producer_ids, false, BTreeMap::new()) } fn build_complete( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, producer_ids: &[String], backend_local: bool, executable_dags: BTreeMap, ) -> Result { - let schemas = materializations + let schemas = outputs .iter() - .map(|materialization| { + .map(|(materialization, family)| { let fingerprint = materialization.policy_fingerprint(); - let accumulator = materialization - .accumulator_spec() - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; - let family = StateFamilyContract::try_from(&accumulator.family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; + if StateFamilyContract::try_from(family).is_err() { + return Err(PrecomputePlanError::UnsupportedFamily(fingerprint.0)); + } let source = materialization.table_name.as_ref().map_or_else( || Source::TimeSeries { metric: materialization.metric.clone(), @@ -334,7 +372,7 @@ impl PrecomputePlan { schema_id: state_schema_id(fingerprint), schema_version: 1, materialization: fingerprint.into(), - family, + family: family.clone(), source, value_projection, group_by: materialization.grouping_labels.clone(), @@ -350,10 +388,11 @@ impl PrecomputePlan { }, pane_origin_ms: materialization.pane_origin_ms, }, - encodings: state_encodings(&accumulator.family), + encodings: state_encodings(family), }) }) .collect::, PrecomputePlanError>>()?; + let materializations: Vec<_> = outputs.into_iter().map(|(config, _)| config).collect(); let producers = producer_ids .iter() .flat_map(|producer_id| { @@ -407,19 +446,141 @@ impl PrecomputePlan { /// are precomputed inside ASAPQuery rather than by ASAPCollector. pub fn build_backend_local( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, ) -> Result { - Self::build_backend_local_with_dags(envelope, materializations, BTreeMap::new()) + Self::build_backend_local_with_dags(envelope, outputs, BTreeMap::new()) } /// Construct the complete backend-local contract before validation. Derived /// definitions are never validated without their actual executable bindings. pub fn build_backend_local_with_dags( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, executable_dags: BTreeMap, ) -> Result { - Self::build_complete(envelope, materializations, &[], true, executable_dags) + Self::build_complete(envelope, outputs, &[], true, executable_dags) + } + + /// The stored state family of `output`, from its schema contract. + pub fn state_family(&self, output: crate::sds::StoredOutputId) -> Option<&SummaryFamilyType> { + self.schemas + .iter() + .find(|schema| schema.materialization == output) + .map(|schema| &schema.family) + } + + /// The Planner SummaryAgg node that produces `output`, with the source + /// expression feeding it when that input is a raw source. `None` when no + /// installed DAG produces the output with a SummaryAgg. Every DAG that + /// produces the output must agree on both, or the plan is rejected: the + /// runtime routes, filters and catalogs the output once. + #[allow(clippy::type_complexity)] + pub fn summary_producer( + &self, + output: crate::sds::StoredOutputId, + ) -> Result< + Option<( + planner_types::post_asap::PostAsapDagNode, + Option, + )>, + String, + > { + use planner_types::post_asap::PostAsapOperatorPayload; + let mut selected: Option<( + planner_types::post_asap::PostAsapDagNode, + Option, + )> = None; + for installed in self.executable_dags.values() { + let Some(node) = installed.binding.nodes.iter().find_map(|(node, binding)| { + matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } if *stored_output == output).then_some(*node) + }) else { + continue; + }; + let dag = installed.document.decode()?; + let producer = dag + .nodes + .iter() + .find(|candidate| candidate.id == node) + .ok_or("bound materialization node is absent from its DAG")?; + if !matches!(producer.payload, PostAsapOperatorPayload::SummaryAgg { .. }) { + continue; + } + let inputs: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node).collect(); + let source = match inputs.as_slice() { + [edge] => dag + .nodes + .iter() + .find(|candidate| candidate.id == edge.producer) + .and_then(|source| match &source.payload { + PostAsapOperatorPayload::Fallback { expression } => { + Some(expression.clone()) + } + _ => None, + }), + _ => None, + }; + if let Some((first, first_source)) = &selected { + // Shared outputs may be read by scans that project different + // columns; what must agree is the update and its population. + let population = + |node: &planner_types::post_asap::PostAsapDagNode, + source: &Option| { + let PostAsapOperatorPayload::SummaryAgg { family, .. } = &node.payload + else { + return None; + }; + source.as_ref().map(|expression| { + raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + ) + .map(|(metric, _, filter)| { + (metric, crate::utils::normalize_spatial_filter(&filter)) + }) + }) + }; + if first.payload != producer.payload + || population(first, first_source) != population(producer, &source) + { + return Err(format!( + "stored output {} is produced by DAGs that disagree on its computation", + output.as_u64() + )); + } + } else { + selected = Some((producer.clone(), source)); + } + } + Ok(selected) + } + + /// Canonical population predicate of `config`'s input: the typed table + /// population, or the label filter of its Planner DAG time-series scan. + pub fn population_filter( + &self, + config: &crate::PrecomputeMaterialization, + ) -> Result { + let table = config.table_population_canonical()?; + if config.table_name.is_some() || config.derived_input.is_some() { + return Ok(table); + } + let expression = match self.summary_producer(config.stored_output_id)? { + Some((_, Some(expression))) => expression, + // A bound raw output whose input cannot be read must not be + // treated as unfiltered. + Some((_, None)) => { + return Err("raw output's Planner producer does not read a source scan".into()) + } + // Only plans without a Planner DAG for this output (imported or + // fixture outputs) have no predicate to read. + None => return Ok(String::new()), + }; + let family = self.state_family(config.stored_output_id); + let (_, _, filter) = raw_time_series_input_contract( + &expression, + matches!(family, Some(SummaryFamilyType::ExactAggregate(..))), + )?; + Ok(crate::utils::normalize_spatial_filter(&filter)) } pub fn runtime_materializations( @@ -630,11 +791,16 @@ impl PrecomputePlan { .map_err(|e| PrecomputePlanError::CatalogContract(e.to_string())) .map(|contract| !asap_physical_operators::physical_planner::precompute::is_population_schema(&contract.schema))? .then_some(native); - if sources.iter().any(|source| { - !matches!(source.aggregation_type, crate::AggregationType::Sum) - && !(native.is_some() - && matches!(source.aggregation_type, crate::AggregationType::Rate)) - }) { + let exact = |config: &crate::PrecomputeMaterialization, kinds: &[ExactKind]| { + matches!(self.state_family(config.stored_output_id), + Some(SummaryFamilyType::ExactAggregate(kind, _)) if kinds.contains(kind)) + }; + let source_kinds: &[ExactKind] = if native.is_some() { + &[ExactKind::Sum, ExactKind::Rate] + } else { + &[ExactKind::Sum] + }; + if sources.iter().any(|source| !exact(source, source_kinds)) { return Err(invalid()); } if native.is_none() { @@ -643,12 +809,11 @@ impl PrecomputePlan { } validate_maintenance_reduction(config, target_node) .map_err(PrecomputePlanError::CatalogContract)?; - } else if !matches!( - config.aggregation_type, - crate::AggregationType::CountMinSketchWithHeap - | crate::AggregationType::CountSketchWithHeap - | crate::AggregationType::Sum - ) { + } else if !exact(config, &[ExactKind::Sum]) + && !matches!(self.state_family(config.stored_output_id), + Some(SummaryFamilyType::Sketch(kind, _)) if matches!(kind.algorithm(), + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + { return Err(invalid()); } let inputs: Vec<_> = dag @@ -773,32 +938,38 @@ impl PrecomputePlan { "DAG materialization has no runtime configuration".into(), )); }; - if self.ingest.protocol == IngestProtocol::PrometheusRemoteWriteV1 - && matches!( - config.aggregation_type, - crate::AggregationType::HLL | crate::AggregationType::UnivMon - ) + if let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { + family, + input, + .. + } = &node.payload { - if let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { - input, - .. - } = &node.payload - { - let supported = match config.aggregation_type { - crate::AggregationType::HLL => { - crate::accumulator_spec::is_scalar_sample_value(input) - || crate::accumulator_spec::is_unit_sample_frequency(input) - } - crate::AggregationType::UnivMon => { - crate::accumulator_spec::is_unit_sample_frequency(input) - } - _ => unreachable!(), - }; - if !supported { - return Err(PrecomputePlanError::CatalogContract( - "raw materialization input does not match its accumulator update semantics".into(), - )); + if self.state_family(config.stored_output_id) != Some(family) { + return Err(PrecomputePlanError::CatalogContract( + "stored output schema family differs from its Planner producer".into(), + )); + } + let algorithm = match family { + SummaryFamilyType::Sketch(kind, _) => Some(kind.algorithm()), + _ => None, + }; + let supported = match algorithm { + _ if self.ingest.protocol != IngestProtocol::PrometheusRemoteWriteV1 => { + true + } + Some(SketchAlgorithm::Hll) => { + crate::accumulator_spec::is_scalar_sample_value(input) + || crate::accumulator_spec::is_unit_sample_frequency(input) } + Some(SketchAlgorithm::UnivMon) => { + crate::accumulator_spec::is_unit_sample_frequency(input) + } + _ => true, + }; + if !supported { + return Err(PrecomputePlanError::CatalogContract( + "raw materialization input does not match its accumulator update semantics".into(), + )); } } if let Some(partitioning) = config.partitioning { @@ -827,6 +998,11 @@ impl PrecomputePlan { } let mut materializations = BTreeSet::new(); for materialization in &self.materializations { + if materialization.stored_output_id.as_u64() == 0 { + return Err(PrecomputePlanError::CatalogContract( + "materialization has no allocated stored output id".into(), + )); + } if materialization.table_name.is_none() && !materialization.grouping_labels.is_legacy_labels() { @@ -899,12 +1075,35 @@ impl PrecomputePlan { candidate.policy_fingerprint() == schema.materialization.fingerprint() }) .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let accumulator = materialization.accumulator_spec().map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; - let family = StateFamilyContract::try_from(&accumulator.family).map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; + if StateFamilyContract::try_from(&schema.family).is_err() + || crate::aggregation_type_for_family(&schema.family).is_none() + { + return Err(PrecomputePlanError::UnsupportedFamily( + schema.materialization.as_u64(), + )); + } + let invalid_computation = |reason: String| PrecomputePlanError::InvalidComputation { + materialization: schema.materialization.as_u64(), + reason, + }; + let raw = + materialization.table_name.is_none() && materialization.derived_input.is_none(); + // Once a plan carries Planner DAGs, a raw output's input predicate + // exists only on its producer; an unbound output would silently + // read every series of its metric. + if raw + && !self.executable_dags.is_empty() + && self + .summary_producer(schema.materialization) + .map_err(invalid_computation)? + .is_none() + { + return Err(invalid_computation( + "raw output has no Planner producer".into(), + )); + } + self.population_filter(materialization) + .map_err(invalid_computation)?; let source = materialization.table_name.as_ref().map_or_else( || Source::TimeSeries { metric: materialization.metric.clone(), @@ -915,7 +1114,6 @@ impl PrecomputePlan { ); let value_projection = materialization.effective_value_projection().clone(); if schema.schema_id != state_schema_id(schema.materialization.fingerprint()) - || schema.family != family || schema.source != source || schema.value_projection != value_projection || schema.group_by != materialization.grouping_labels @@ -930,7 +1128,7 @@ impl PrecomputePlan { crate::WindowKind::Session => None, } || schema.window.pane_origin_ms != materialization.pane_origin_ms - || !state_encodings_match(&accumulator.family, &schema.encodings) + || !state_encodings_match(&schema.family, &schema.encodings) { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), @@ -974,6 +1172,74 @@ impl PrecomputePlan { } } +/// Decompose a Planner raw time-series input expression into its metric, +/// optional whole-second range and PromQL label filter. +pub fn raw_time_series_input_contract( + expr: &planner_types::pre_asap::QueryExpr, + exact: bool, +) -> Result<(String, Option, String), String> { + use planner_types::pre_asap::{CompareOpKind, QueryExpr, ScalarValue}; + let (source, window_secs) = match expr { + QueryExpr::TimeRange { child, range } => { + if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { + return Err("warm producer requires a positive whole-second range".into()); + } + (child.as_ref(), Some(range.as_secs())) + } + QueryExpr::Scan { .. } if exact => { + return Err( + "instantaneous sample selection is not a temporal accumulator readout".into(), + ); + } + source => (source, None), + }; + match source { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + predicates, + schema, + } if !metric.is_empty() => { + let mut matchers = Vec::with_capacity(predicates.len()); + for predicate in predicates { + let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { + return Err("materialization filter is not a label comparison".into()); + }; + let (QueryExpr::Column(column), QueryExpr::Literal(ScalarValue::Utf8(value))) = + (left.as_ref(), right.as_ref()) + else { + return Err("materialization filter must compare a label with a string".into()); + }; + let label = schema + .columns + .get(*column) + .map(|field| field.name.as_str()) + .ok_or_else(|| { + "materialization filter references an unknown column".to_string() + })?; + let operator = match op { + CompareOpKind::Eq => "=", + CompareOpKind::Ne => "!=", + CompareOpKind::Regex => "=~", + CompareOpKind::NotRegex => "!~", + _ => return Err("materialization filter uses a non-PromQL comparison".into()), + }; + let encoded = serde_json::to_string(value).map_err(|error| error.to_string())?; + matchers.push(format!("{label}{operator}{encoded}")); + } + matchers.sort(); + let spatial_filter = if matchers.is_empty() { + String::new() + } else { + format!("{{{}}}", matchers.join(",")) + }; + Ok((metric.clone(), window_secs, spatial_filter)) + } + _ => { + Err("source predicates or temporal modifiers require a Prometheus exact subtree".into()) + } + } +} + pub(crate) fn state_schema_id(fingerprint: crate::PolicyFingerprint) -> String { format!("{}:summary-state:v1:{}", BACKEND_COMPAT, fingerprint.0) } @@ -1046,16 +1312,34 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec mod source_window_cohort_tests { use super::*; fn full_window() -> crate::PrecomputeMaterialization { - serde_json::from_value(serde_json::json!({ - "aggregation_type":"Sum", "aggregation_sub_type":"", "parameters":{}, - "grouping_labels":{"labels":[]}, "aggregated_labels":{"labels":[]}, - "rollup_labels":{"labels":[]}, "original_yaml":"", - "window_size":60, "slide_interval":10, "window_type":"sliding", - "window_layout":{"kind":"full_window"}, "pane_origin_ms":0, - "spatial_filter":"", "spatial_filter_normalized":"", "metric":"m", - "num_aggregates_to_retain":null, "table_name":null, "value_projection":null - })) - .unwrap() + let mut config: crate::PrecomputeMaterialization = + serde_json::from_value(serde_json::json!({ + "stored_output_id":0, "grouping_labels":{"labels":[]}, + "window_size":60, "slide_interval":10, "window_type":"sliding", + "window_layout":{"kind":"full_window"}, "pane_origin_ms":0, "metric":"m", + "num_aggregates_to_retain":null, "table_name":null, "value_projection":null + })) + .unwrap(); + config.allocate_stored_output_id(&"sum"); + config + } + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate( + ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ) + } + fn envelope(plan_version: u64) -> PlanEnvelope { + PlanEnvelope { + plan_id: 1, + plan_version, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: "test".into(), + capability_snapshot_id: "test".into(), + } } // New native-format support must not invalidate persisted older codec subsets. #[test] @@ -1089,20 +1373,11 @@ mod source_window_cohort_tests { #[test] fn producer_roster_roundtrip_and_watermark_scope() { - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 2, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; let mut config = full_window(); config.slide_interval = config.window_size; config.window_type = crate::WindowKind::Tumbling; - let mut plan = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap(); + let mut plan = + PrecomputePlan::build_backend_local(envelope(2), vec![(config, sum())]).unwrap(); let generation = crate::sds::CatalogGeneration { schema_version: 1, plan_id: 1, @@ -1222,17 +1497,8 @@ mod source_window_cohort_tests { config.slide_interval = config.window_size; config.population_key_encoding = crate::grouping_projection::PopulationKeyEncoding::CanonicalLabelsV1; - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; - let error = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap_err(); + let error = + PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap_err(); assert!( error.to_string().contains("population key encoding"), "{error}" @@ -1252,6 +1518,7 @@ mod source_window_cohort_tests { assert!(std::ptr::eq(result[0], &source)); let mut other = source.clone(); other.metric = "other".into(); + other.allocate_stored_output_id(&"sum"); assert_eq!( validated_source_window_cohort(&target, &[&source, &other]) .unwrap() @@ -1269,8 +1536,65 @@ mod source_window_cohort_tests { crate::WindowMaterializationLayout::Pane { pane_secs: 10 } } } + changed.allocate_stored_output_id(&"sum"); assert_ne!(source.policy_fingerprint(), changed.policy_fingerprint()); assert!(validated_source_window_cohort(&target, &[&changed]).is_err()); } } + + // A plan compiled for the v1 schema, which duplicated computation fields + // on materializations, is rejected by version before its body is read. + #[test] + fn rejects_plans_from_the_computation_carrying_schema_by_version() { + let mut config = full_window(); + config.slide_interval = config.window_size; + config.window_type = crate::WindowKind::Tumbling; + let plan = PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap(); + let mut wire = serde_json::to_value(&plan).unwrap(); + assert!(serde_json::from_value::(wire.clone()).is_ok()); + wire["envelope"]["backend_compat"] = "asap-query-backend.v1".into(); + wire["materializations"][0]["aggregation_type"] = "Sum".into(); + let error = serde_json::from_value::(wire).unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported installed precompute plan schema asap-query-backend.v1"), + "{error}" + ); + } + + // Materializations name no computation: the removed v1 fields are unknown. + #[test] + fn materializations_reject_removed_computation_fields() { + let wire = serde_json::to_value(full_window()).unwrap(); + for field in [ + "aggregation_type", + "aggregation_sub_type", + "parameters", + "spatial_filter", + "spatial_filter_normalized", + "aggregated_labels", + "rollup_labels", + "original_yaml", + ] { + let mut legacy = wire.clone(); + legacy[field] = serde_json::json!(""); + assert!( + serde_json::from_value::(legacy).is_err(), + "{field}" + ); + } + } + + // An output without an allocated id cannot be installed. + #[test] + fn rejects_unallocated_stored_output_ids() { + let mut config = full_window(); + config.slide_interval = config.window_size; + config.window_type = crate::WindowKind::Tumbling; + config.stored_output_id = crate::sds::StoredOutputId(0); + let error = + PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap_err(); + assert!(error.to_string().contains("stored output id"), "{error}"); + } } diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 80bfbcaa3..eaf49df13 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -44,12 +44,7 @@ impl PrecomputePlan { catalog: &SummaryCatalog, ) -> Result<(), PrecomputePlanError> { catalog.validate().map_err(|e| invalid(e.to_string()))?; - let expected = SummaryCatalog::from_materializations( - catalog.plan_id, - catalog.plan_version, - &self.materializations, - ) - .map_err(|e| invalid(e.to_string()))?; + let expected = SummaryCatalog::from_plan(self).map_err(|e| invalid(e.to_string()))?; if catalog.outputs != expected.outputs || catalog.definitions != expected.definitions { return Err(invalid( "stored output semantics differ from installed writer computation", @@ -118,8 +113,11 @@ impl PrecomputePlan { for config in &self.materializations { let id = StoredOutputId::from(config.policy_fingerprint()); let binding = &catalog.outputs[&id]; + let family = self + .state_family(id) + .ok_or(PrecomputePlanError::SchemaSetMismatch)?; let expected = - SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; + SummaryDescriptor::from_family(family).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { return Err(invalid( "summary operator/update contract differs from catalog", @@ -149,27 +147,16 @@ impl PrecomputePlan { || data.source != expected_source || &data.value_projection != expected_projection || data.population_filter_canonical - != config.population_filter_canonical().map_err(invalid)? + != self.population_filter(config).map_err(invalid)? || data.group_by_keys != config.grouping_labels { return Err(invalid("source/population/grouping differs from catalog")); } - if config.spatial_filter_normalized - != crate::utils::normalize_spatial_filter(&config.spatial_filter) - { - return Err(invalid("normalized population predicate drift")); - } let schema = self .schemas .iter() .find(|s| s.materialization == id) .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let family = config - .accumulator_spec() - .map_err(|e| invalid(e.to_string()))? - .family; - let expected_family = StateFamilyContract::try_from(&family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(id.as_u64()))?; let source = if let Some(table) = &config.table_name { Source::Table { table_ref: table.clone(), @@ -202,7 +189,6 @@ impl PrecomputePlan { .output_reference(id) .map_err(|e| invalid(e.to_string()))? || schema.schema_id != state_schema_id(id.fingerprint()) - || schema.family != expected_family || schema.source != source || &schema.value_projection != projection || schema.group_by != config.grouping_labels @@ -215,7 +201,7 @@ impl PrecomputePlan { schema_id: schema.schema_id.clone(), }); } - if !state_encodings_match(&family, &schema.encodings) { + if !state_encodings_match(family, &schema.encodings) { return Err(invalid("encoding does not match state family")); } if config.num_aggregates_to_retain == Some(0) { diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 846c4df83..dd5a7cfd9 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -386,7 +386,9 @@ impl TransmissionPlan { if !schema.encodings.contains(&rule.encoding) { return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); } - validate_runtime_rule_policy(rule, &schema.family)?; + let family = StateFamilyContract::try_from(&schema.family) + .map_err(|()| TransmissionPlanError::InvalidRule(rule.producer_id.clone()))?; + validate_runtime_rule_policy(rule, &family)?; } Ok(()) } diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs deleted file mode 100644 index ec20b7ef2..000000000 --- a/crates/asap_types/src/routing_index.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! `RoutingIndex` — a metric-bucketed structural index over a -//! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `InstalledPrecomputePlan`'s `PrecomputeMaterialization`s, so it represents planned policy -//! rather than a reconstruction from ingest side effects. -//! -//! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] -//! itself — already O(1), nothing to add here. -//! -//! **Tier 2** (structural match: "every policy registered for this metric") -//! is what this type adds. Before this existed, -//! `control_plane::asap_tier_analysis::find_matching_policies` scanned -//! *every* policy in the registry for every candidate, checking each one's -//! metric name first — i.e. it paid for every OTHER metric's policies on -//! every lookup. `RoutingIndex` buckets by metric once, at construction -//! time, so a lookup only ever touches the policies that could possibly -//! match. -//! -//! ## Lifecycle -//! -//! Built fresh from a `PolicyRegistry` snapshot — construction is `O(N)` -//! over the registry, same order as `PolicyRegistry::from_configs` itself. -//! Callers that build one per query (mirroring today's -//! `streaming_snap.policy_registry()` call) still get the Tier-2 win for -//! any query with more than one candidate sharing the same snapshot -//! (composed PromQL shapes routinely do). Building it once per -//! `InstalledPrecomputePlan` hot-reload swap instead of once per query — the same -//! "cheap, but call at swap time not per query, if it shows up in -//! profiles" note `InstalledPrecomputePlan::policy_registry`'s own doc comment -//! already flags — is a further, larger change (it means threading a -//! cached derived value through `InstalledPrecomputePlanHandle`'s swap path) -//! and is not done by this type on its own. - -use std::collections::{BTreeSet, HashMap}; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; -use crate::policy_registry::PolicyRegistry; - -/// See module docs. -#[derive(Debug, Clone, Default)] -pub struct RoutingIndex { - registry: PolicyRegistry, - by_metric: HashMap>, -} - -impl RoutingIndex { - /// Build from a `PolicyRegistry` snapshot. Takes ownership rather than - /// borrowing — callers that still need their own `PolicyRegistry` - /// handle after this should `.clone()` it first (cheap-ish, but real; - /// most callers don't need the raw registry once they have the index, - /// since [`Self::get`] delegates straight through). - pub fn build(registry: PolicyRegistry) -> Self { - let mut by_metric: HashMap> = HashMap::new(); - for (fp, cfg) in registry.iter() { - by_metric.entry(cfg.metric.clone()).or_default().push(*fp); - } - Self { - registry, - by_metric, - } - } - - /// Tier 1 — exact fingerprint lookup. Delegates to the underlying - /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.registry.get(fp) - } - - /// Tier 2 — every policy fingerprint registered for `metric`, in - /// registration order. Empty slice (not an `Option`/error) when - /// nothing is registered for this metric — callers already treat "no - /// candidates" as a normal, expected outcome (capability miss → - /// archive fallback), not a failure to report. - pub fn candidates_for_metric(&self, metric: &str) -> &[PolicyFingerprint] { - self.by_metric.get(metric).map(Vec::as_slice).unwrap_or(&[]) - } - - /// Live policy count (same as the underlying registry's). - pub fn len(&self) -> usize { - self.registry.len() - } - - pub fn is_empty(&self) -> bool { - self.registry.is_empty() - } - - /// All fingerprints currently registered. Delegates to the underlying - /// registry — see [`PolicyRegistry::fingerprints`]. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.registry.fingerprints() - } - - /// Resolve an ingest-side sketch shape to exactly one configured policy. - /// Ambiguous or absent matches fail closed. - pub fn find_policy_by_content( - &self, - metric: &str, - group_by_keys: &BTreeSet, - agg_type: crate::AggregationType, - expected_params: &HashMap, - ) -> Option { - let mut hit = None; - for fp in self.candidates_for_metric(metric) { - let cfg = self.get(*fp)?; - let policy_keys: BTreeSet<_> = cfg.grouping_labels.iter().cloned().collect(); - if cfg.aggregation_type != agg_type - || &policy_keys != group_by_keys - || !cfg.spatial_filter_normalized.is_empty() - || !expected_params.iter().all(|(key, value)| { - cfg.parameters.get(key).or_else(|| match key.as_str() { - // The typed physical-plan compiler names this field - // after SummaryParams, while legacy collector YAML - // uses the equivalent runtime-facing name. - "relative_accuracy" => cfg.parameters.get("alpha"), - "alpha" => cfg.parameters.get("relative_accuracy"), - _ => None, - }) == Some(value) - }) - { - continue; - } - if hit.is_some() { - return None; - } - hit = Some(*fp); - } - hit - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(metric: &str) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn buckets_by_metric() { - let a1 = cfg("http_lat"); - let a2 = cfg("cpu_pct"); - let fp_a1 = PolicyFingerprint::from_config(&a1); - let fp_a2 = PolicyFingerprint::from_config(&a2); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a1, a2])); - - assert_eq!(idx.candidates_for_metric("http_lat"), &[fp_a1]); - assert_eq!(idx.candidates_for_metric("cpu_pct"), &[fp_a2]); - } - - #[test] - fn multiple_policies_for_the_same_metric_all_bucket_together() { - // Same metric, distinct group-by shapes -> distinct fingerprints, - // same bucket. - let a = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::new(vec!["zone".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "http_lat".to_string(), - None, - None, - None, - ); - let b = cfg("http_lat"); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a, b])); - assert_eq!(idx.candidates_for_metric("http_lat").len(), 2); - } - - #[test] - fn unknown_metric_returns_empty_slice_not_missing() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("http_lat")])); - assert!(idx.candidates_for_metric("no_such_metric").is_empty()); - } - - #[test] - fn get_delegates_to_underlying_registry() { - let a = cfg("http_lat"); - let fp = PolicyFingerprint::from_config(&a); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a])); - assert_eq!( - idx.get(fp).map(|c| c.metric.clone()), - Some("http_lat".to_string()) - ); - assert!(idx - .get(PolicyFingerprint::from_config(&cfg("nope"))) - .is_none()); - } - - #[test] - fn len_and_is_empty_match_registry() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs( - Vec::::new(), - )); - assert!(idx.is_empty()); - assert_eq!(idx.len(), 0); - - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("m")])); - assert!(!idx.is_empty()); - assert_eq!(idx.len(), 1); - } - - #[test] - fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { - let mut parameters = StdHashMap::new(); - parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = PrecomputeMaterialization::new( - AggregationType::DDSketch, - String::new(), - parameters, - KeyByLabelNames::new(vec!["service".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 5, - 5, - WindowKind::Tumbling, - String::new(), - "latency".to_string(), - None, - None, - None, - ); - let fingerprint = PolicyFingerprint::from_config(&config); - let index = RoutingIndex::build(PolicyRegistry::from_configs(vec![config])); - let expected = - StdHashMap::from([("relative_accuracy".to_string(), serde_json::json!(0.01))]); - - assert_eq!( - index.find_policy_by_content( - "latency", - &BTreeSet::from(["service".to_string()]), - AggregationType::DDSketch, - &expected, - ), - Some(fingerprint) - ); - } -} diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index a6870ea20..1619c6ba0 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -3,7 +3,7 @@ //! belong to the stored summary identified by this metadata. pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; -use crate::{AggregationType, PrecomputeMaterialization}; +use crate::AggregationType; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -492,15 +492,11 @@ pub enum SummaryOperator { agg_type: AggregationType, parameters_canonical: String, }, - /// Complete planner materialization configuration, including heap/Hydra - /// dimensions and readout/update subtype. Never equal to a legacy projection. + /// The Planner-selected state family of a stored output, including + /// heap/Hydra dimensions. Never equal to a legacy projection. Grouping and + /// pane layout live in the data descriptor and summary definition. Configured { - /// Planner-selected semantic family; grouping and pane layout live in - /// the data descriptor and summary definition, respectively. family: planner_types::post_asap::SummaryFamilyType, - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: BTreeMap, }, } @@ -553,32 +549,19 @@ impl FidelityGuarantee { /// Model IDs name parameterized error families/scopes, not certified numeric /// epsilon/confidence values. Heap membership and Hydra cross-cell readouts /// need separate models; point-frequency/per-cell rank does not attest them. - pub fn from_config(config: &PrecomputeMaterialization) -> Self { - if config.aggregation_type == AggregationType::HLL { - let precision = config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(14); - return Self::HllCardinalityError { - precision, - model: "asap.hll.relative-cardinality.v1".into(), - }; - } - match config.accumulator_spec().map(|s| s.family) { - Ok(SummaryFamilyType::ExactAggregate( + pub fn from_family(family: &SummaryFamilyType) -> Self { + match family { + SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate | planner_types::post_asap::ExactKind::IRate, _, - )) => Self::ExactCounter { + ) => Self::ExactCounter { model: "prometheus.extrapolated-rate.v1".into(), full_pane_coverage_required: true, }, - Ok(SummaryFamilyType::ExactAggregate(..)) => Self::Exact, - Ok(SummaryFamilyType::Sketch(kind, _)) => match kind.params() { + SummaryFamilyType::ExactAggregate(..) => Self::Exact, + SummaryFamilyType::Sketch(kind, grouping) => match kind.params() { SketchParams::UnivMon { heap_size, sketch_rows, @@ -592,7 +575,10 @@ impl FidelityGuarantee { }, SketchParams::Kll { k } => Self::KllRankError { k: *k, - model: if config.aggregation_type == AggregationType::HydraKLL { + model: if matches!( + grouping, + planner_types::post_asap::GroupingStrategy::SharedMultiSubpopulation { .. } + ) { "asap.hydra-kll.per-cell-rank.v1" } else { "asap.kll.normalized-rank.v1" @@ -713,41 +699,6 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; - if let SummaryOperator::Configured { - family, - aggregation_type, - .. - } = &operator - { - if let Some(expected) = aggregation_type.planner_exact_family() { - if family != &expected { - return Err(SdsError( - "configured storage type disagrees with Planner family".into(), - )); - } - } else { - use AggregationType as A; - let expected = match aggregation_type { - A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), - A::CountMinSketch => Some(SketchAlgorithm::Cms), - A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), - A::CountSketch => Some(SketchAlgorithm::CountSketch), - A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), - A::DDSketch => Some(SketchAlgorithm::DDSketch), - A::HLL => Some(SketchAlgorithm::Hll), - A::UnivMon => Some(SketchAlgorithm::UnivMon), - _ => None, - }; - if let Some(expected) = expected { - if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) - { - return Err(SdsError( - "configured sketch storage disagrees with Planner family".into(), - )); - } - } - } - } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), @@ -777,12 +728,9 @@ impl SummaryDescriptor { Ok(()) } - /// Preserve every configured state/update parameter. Omitted defaults remain - /// distinct from explicit defaults (conservative identity, never false sharing). - /// Legacy AggKind projections intentionally have different operator variants: - /// they cannot attest heap, Hydra, or aggregation-subtype semantics they lost. - pub fn from_config(config: &PrecomputeMaterialization) -> Result { - let fidelity = FidelityGuarantee::from_config(config); + /// The descriptor of a stored output whose state has the Planner `family`. + pub fn from_family(family: &SummaryFamilyType) -> Result { + let fidelity = FidelityGuarantee::from_family(family); let state_schema_version = if matches!(fidelity, FidelityGuarantee::ExactCounter { .. }) { 2 } else { @@ -790,17 +738,7 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { - family: config - .accumulator_spec() - .map_err(|error| SdsError(error.to_string()))? - .family, - aggregation_type: config.aggregation_type, - aggregation_sub_type: config.aggregation_sub_type.clone(), - parameters: config - .parameters - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), + family: family.clone(), }, fidelity, state_schema_version, @@ -844,11 +782,7 @@ impl FidelityGuarantee { match operator { SummaryOperator::LegacyPartial { .. } => matches!(self, Unknown { .. }), SummaryOperator::ExactAgg { agg_type, .. } => configured(*agg_type), - SummaryOperator::Configured { - aggregation_type, - parameters, - .. - } => configured(*aggregation_type) && self.matches_parameters(parameters), + SummaryOperator::Configured { family } => self == &Self::from_family(family), SummaryOperator::Sketch { algorithm, parameters, @@ -1653,9 +1587,6 @@ mod tests { assert!(SummaryDescriptor::new( SummaryOperator::Configured { family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), }, FidelityGuarantee::KllRankError { k: 200, @@ -1678,70 +1609,67 @@ mod tests { .is_err()); } + // A configured descriptor's fidelity must be the one its family declares. #[test] - fn configured_descriptor_rejects_family_storage_disagreement() { + fn configured_descriptor_rejects_fidelity_family_disagreement() { assert!(SummaryDescriptor::new( SummaryOperator::Configured { family: AggregationType::Rate.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Increase, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), - }, - FidelityGuarantee::ExactCounter { - model: "prometheus.extrapolated-rate.v1".into(), - full_pane_coverage_required: true, }, + FidelityGuarantee::Exact, 2, ) .is_err()); } + // Configured identity preserves every heap/Hydra/state parameter of the + // family and names no population. #[test] - fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { - let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); - let mut config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); - assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, - FidelityGuarantee::DdSketchRelativeError { .. } - )); - for (kind, key) in [ - (AggregationType::CountMinSketchWithHeap, "heap_size"), - (AggregationType::HydraKLL, "row"), - (AggregationType::HydraKLL, "col"), - (AggregationType::HydraKLL, "k"), - ] { - config.aggregation_type = kind; - config.parameters.insert(key.into(), json!(10)); - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.parameters.insert(key.into(), json!(11)); - let after = SummaryDescriptor::from_config(&config).unwrap(); - assert_ne!(before.id, after.id, "{key}"); - config.metric = "other".into(); - config.spatial_filter_normalized = "job=a".into(); - assert_eq!( - after.id, - SummaryDescriptor::from_config(&config).unwrap().id - ); - } - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.aggregation_sub_type = "max".into(); + fn configured_identity_preserves_state_parameters() { + use planner_types::post_asap::{GroupingStrategy, HydraKind, HydraParams, SketchKind}; + let heap = |heap_size| { + SummaryFamilyType::Sketch( + SketchKind::new( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 64, + depth: 4, + heap_size, + }, + ), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let hydra = |k, shared_buckets| { + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraKll, + params: HydraParams::HydraKll { k, shared_buckets }, + }, + ) + }; + let id = |family: &SummaryFamilyType| SummaryDescriptor::from_family(family).unwrap().id; + assert_ne!(id(&heap(10)), id(&heap(11))); + assert_ne!(id(&hydra(10, 10)), id(&hydra(10, 11))); assert_ne!( - before.id, - SummaryDescriptor::from_config(&config).unwrap().id + id(&hydra(10, 10)), + id(&SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 10 }), + GroupingStrategy::PerSubpopulationInstance, + )) ); + assert!(matches!( + SummaryDescriptor::from_family(&hydra(10, 10)).unwrap().fidelity, + FidelityGuarantee::KllRankError { ref model, .. } if model == "asap.hydra-kll.per-cell-rank.v1" + )); } + // Increase state declares the Prometheus counter fidelity model. #[test] - fn increase_config_declares_prometheus_counter_fidelity() { - let yaml: serde_yaml::Value = serde_yaml::from_str( - "aggregationType: Increase\naggregationSubType: ''\nmetric: requests_total\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters: {}\nwindowSize: 60\nwindowType: tumbling\nspatialFilter: ''\n", - ) - .unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); + fn increase_declares_prometheus_counter_fidelity() { assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, + SummaryDescriptor::from_family(&AggregationType::Increase.planner_exact_family().unwrap()) + .unwrap() + .fidelity, FidelityGuarantee::ExactCounter { ref model, full_pane_coverage_required: true @@ -1750,25 +1678,27 @@ mod tests { } #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { - let a = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), + let a = SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([ + ("k".into(), json!(200)), + ("nested".into(), json!({"z":1,"a":2})), + ]), }; - let b = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), + let b = SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([ + ("nested".into(), json!({"a":2,"z":1})), + ("k".into(), json!(200)), + ]), + }; + let fidelity = FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), }; assert_eq!( - SummaryDescriptor::new(a.clone(), FidelityGuarantee::Exact, 1) - .unwrap() - .id, - SummaryDescriptor::new(b, FidelityGuarantee::Exact, 1) - .unwrap() - .id + SummaryDescriptor::new(a, fidelity.clone(), 1).unwrap().id, + SummaryDescriptor::new(b, fidelity, 1).unwrap().id ); let kll = SummaryOperator::Sketch { algorithm: SketchAlgorithm::Kll, diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index ec924e704..c83ef3ded 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -12,7 +12,7 @@ use crate::sds::{ use crate::PolicyFingerprint; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 6; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 7; /// Canonical definition binds operator and population descriptors. Writer /// layout and concrete state belong to installed plans and runtime instances. @@ -98,24 +98,51 @@ impl SummaryCatalog { }) } - pub fn from_materializations( + /// The catalog of every output of `plan`: its schema's state family and + /// its Planner DAG input predicate. + pub fn from_plan( + plan: &crate::precompute_plan::PrecomputePlan, + ) -> Result { + let outputs = plan + .materializations + .iter() + .map(|config| { + let family = plan.state_family(config.stored_output_id).ok_or( + SummaryCatalogError::MissingDescriptor(config.stored_output_id.as_u64()), + )?; + let filter = plan + .population_filter(config) + .map_err(SummaryCatalogError::Descriptor)?; + Ok((config, family, filter)) + }) + .collect::, SummaryCatalogError>>()?; + Self::from_outputs(plan.envelope.plan_id, plan.envelope.plan_version, outputs) + } + + /// Each output is paired with its state family and canonical input predicate. + pub fn from_outputs( plan_id: u64, plan_version: u64, - materializations: &[crate::PrecomputeMaterialization], + outputs: Vec<( + &crate::PrecomputeMaterialization, + &planner_types::post_asap::SummaryFamilyType, + String, + )>, ) -> Result { - let entries = materializations + let entries = outputs .iter() - .map(|config| { - let summary = SummaryDescriptor::from_config(config) + .map(|(config, family, filter)| { + let summary = SummaryDescriptor::from_family(family) .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; let source = config.source_identity(); let value_projection = config.effective_value_projection().clone(); + config + .table_population_canonical() + .map_err(SummaryCatalogError::Descriptor)?; let data = DataDescriptor::new_typed( source, value_projection, - config - .population_filter_canonical() - .map_err(SummaryCatalogError::Descriptor)?, + filter.clone(), config.grouping_labels.names(), if config.table_name.is_some() && !config.grouping_labels.is_empty() { crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS @@ -132,7 +159,7 @@ impl SummaryCatalog { .collect::, SummaryCatalogError>>()?; let mut catalog = Self::build(plan_id, plan_version, entries)?; let mut semantic_bindings = BTreeMap::new(); - for config in materializations { + for (config, _, _) in &outputs { let output = catalog .outputs .get_mut(&StoredOutputId::from(config.policy_fingerprint())) @@ -343,25 +370,51 @@ impl SummaryCatalog { mod tests { use super::*; use crate::sds::ValueProjectionIdentity; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use crate::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; - fn config(metric: &str, filter: &str, window: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + } + + fn allocated(mut config: PrecomputeMaterialization, filter: &str) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&("sum", filter)); + config + } + + /// An allocated output with the given DAG input filter. + fn config(metric: &str, filter: &str, window: u64) -> (PrecomputeMaterialization, String) { + let config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), window, window, WindowKind::Tumbling, - filter.into(), - metric.into(), - None, - None, - None, + ); + (allocated(config, filter), filter.into()) + } + + fn catalog( + plan_id: u64, + plan_version: u64, + outputs: &[(PrecomputeMaterialization, String)], + ) -> Result { + with_family(plan_id, plan_version, outputs, &sum()) + } + + fn with_family( + plan_id: u64, + plan_version: u64, + outputs: &[(PrecomputeMaterialization, String)], + family: &SummaryFamilyType, + ) -> Result { + SummaryCatalog::from_outputs( + plan_id, + plan_version, + outputs + .iter() + .map(|(config, filter)| (config, family, filter.clone())) + .collect(), ) } @@ -370,7 +423,7 @@ mod tests { fn table_populations_have_distinct_materialization_and_data_identities() { use crate::table_population::{TableColumnPredicate, TablePopulation}; use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let mut requests = config("raw_samples.value", "", 60); + let (mut requests, _) = config("raw_samples.value", "", 60); requests.table_name = Some("raw_samples".into()); requests.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -382,15 +435,17 @@ mod tests { value: ScalarValue::Utf8("requests".into()), }], }); + let requests = allocated(requests, ""); let mut errors = requests.clone(); errors.table_population.as_mut().unwrap().predicates[0].value = ScalarValue::Utf8("errors".into()); + let errors = allocated(errors, ""); assert_ne!(requests.policy_fingerprint(), errors.policy_fingerprint()); let mut other_table = requests.clone(); other_table.table_name = Some("other_samples".into()); assert_ne!( requests.policy_fingerprint(), - other_table.policy_fingerprint() + allocated(other_table, "").policy_fingerprint() ); let mut other_value = requests.clone(); other_value.value_projection = Some(ValueProjectionIdentity::Column { @@ -398,23 +453,27 @@ mod tests { }); assert_ne!( requests.policy_fingerprint(), - other_value.policy_fingerprint() + allocated(other_value, "").policy_fingerprint() ); let mut other_time = requests.clone(); other_time.table_timestamp_column = Some("event_time_ms".into()); + let other_time = allocated(other_time, ""); assert_ne!( requests.policy_fingerprint(), other_time.policy_fingerprint() ); - let mut invalid_labels = requests.clone(); - invalid_labels.table_population = None; - invalid_labels.spatial_filter = "job=\"requests\"".into(); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_labels]).is_err()); let mut invalid_time = requests.clone(); invalid_time.table_timestamp_column = Some("time; DROP TABLE samples".into()); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_time]).is_err()); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); + assert!(catalog(1, 1, &[(allocated(invalid_time, ""), String::new())]).is_err()); + let catalog = catalog( + 1, + 1, + &[requests, errors, other_time].map(|config| { + let filter = config.table_population_canonical().unwrap(); + (config, filter) + }), + ) + .unwrap(); assert_eq!(catalog.data_descriptors.len(), 3); assert_eq!(catalog.outputs.len(), 3); } @@ -423,10 +482,10 @@ mod tests { fn snapshot_reference_is_deterministic_and_content_sensitive() { let a = config("requests", "", 60); let b = config("errors", "", 60); - let left = SummaryCatalog::from_materializations(1, 2, &[a.clone(), b.clone()]).unwrap(); - let reordered = SummaryCatalog::from_materializations(1, 2, &[b, a.clone()]).unwrap(); + let left = catalog(1, 2, &[a.clone(), b.clone()]).unwrap(); + let reordered = catalog(1, 2, &[b, a.clone()]).unwrap(); assert_eq!(left.reference().unwrap(), reordered.reference().unwrap()); - let changed = SummaryCatalog::from_materializations(1, 2, &[a]).unwrap(); + let changed = catalog(1, 2, &[a]).unwrap(); assert_ne!(left.reference().unwrap(), changed.reference().unwrap()); left.reference() .unwrap() @@ -444,8 +503,7 @@ mod tests { fn shares_descriptors_across_windows_and_deduplicates_materializations() { let one = config("requests", "", 60); let two = config("requests", "", 120); - let catalog = - SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); + let catalog = catalog(7, 2, &[one.clone(), two, one]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); assert_eq!(catalog.outputs.len(), 2); @@ -455,7 +513,7 @@ mod tests { // Stored pane duration identifies a deployment output, not the summary meaning. #[test] fn semantic_definition_is_shared_across_deployed_pane_outputs() { - let catalog = SummaryCatalog::from_materializations( + let catalog = catalog( 1, 1, &[config("requests", "", 60), config("requests", "", 120)], @@ -469,10 +527,10 @@ mod tests { #[test] fn hot_and_rebuild_have_one_definition_and_two_bound_outputs() { let mut hot = config("latency", "", 60); - hot.stored_output_id = Some(StoredOutputId(41)); + hot.0.stored_output_id = StoredOutputId(41); let mut rebuild = hot.clone(); - rebuild.stored_output_id = Some(StoredOutputId(42)); - let catalog = SummaryCatalog::from_materializations(7, 42, &[hot, rebuild]).unwrap(); + rebuild.0.stored_output_id = StoredOutputId(42); + let catalog = catalog(7, 42, &[hot, rebuild]).unwrap(); assert_eq!(catalog.definitions.len(), 1); let hot = catalog.output_reference(StoredOutputId(41)).unwrap(); let rebuild = catalog.output_reference(StoredOutputId(42)).unwrap(); @@ -499,7 +557,7 @@ mod tests { // Source/population changes never alias, while the operator can be reused. #[test] fn separates_population_and_operator_identity() { - let catalog = SummaryCatalog::from_materializations( + let catalog = catalog( 1, 1, &[ @@ -516,12 +574,28 @@ mod tests { // Operator changes share population metadata without sharing state identity. #[test] fn changing_operator_parameters_creates_a_new_summary_descriptor() { - let mut a = config("requests", "", 60); - a.aggregation_type = AggregationType::DatasketchesKLL; - a.parameters.insert("K".into(), serde_json::json!(100)); - let mut b = a.clone(); - b.parameters.insert("K".into(), serde_json::json!(200)); - let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + }; + let kll = |k| { + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let (small, large) = (kll(100), kll(200)); + let output = |family: &SummaryFamilyType| { + let (mut config, _) = config("requests", "", 60); + config.allocate_stored_output_id(family); + config + }; + let (a, b) = (output(&small), output(&large)); + let catalog = SummaryCatalog::from_outputs( + 1, + 1, + vec![(&a, &small, String::new()), (&b, &large, String::new())], + ) + .unwrap(); assert_eq!(catalog.summary_descriptors.len(), 2); assert_eq!(catalog.data_descriptors.len(), 1); assert_eq!(catalog.outputs.len(), 2); @@ -532,8 +606,8 @@ mod tests { fn snapshot_is_deterministic_and_round_trips() { let a = config("requests", "", 60); let b = config("errors", "", 60); - let forward = SummaryCatalog::from_materializations(7, 2, &[a.clone(), b.clone()]).unwrap(); - let backward = SummaryCatalog::from_materializations(7, 2, &[b, a]).unwrap(); + let forward = catalog(7, 2, &[a.clone(), b.clone()]).unwrap(); + let backward = catalog(7, 2, &[b, a]).unwrap(); let bytes = serde_json::to_vec(&forward).unwrap(); assert_eq!(bytes, serde_json::to_vec(&backward).unwrap()); let decoded: SummaryCatalog = serde_json::from_slice(&bytes).unwrap(); @@ -543,10 +617,11 @@ mod tests { #[test] fn catalog_definitions_do_not_store_writer_layout() { - let mut materialization = config("requests", "", 60); + let (mut materialization, _) = config("requests", "", 60); materialization.pane_origin_ms = Some(7_000); + let materialization = allocated(materialization, ""); let id = StoredOutputId::from(materialization.policy_fingerprint()); - let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); + let catalog = catalog(7, 2, &[(materialization, String::new())]).unwrap(); assert!(catalog.outputs.contains_key(&id)); assert!( !serde_json::to_value(&catalog).unwrap()["outputs"][id.as_u64().to_string()] @@ -565,8 +640,7 @@ mod tests { fn conflicting_materialization_is_rejected() { let first = config("requests", "", 60); let second = config("errors", "", 60); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[first.clone(), second]).unwrap(); + let catalog = catalog(1, 1, &[first.clone(), second]).unwrap(); let summary = catalog.summary_descriptors.values().next().unwrap().clone(); let data = catalog .data_descriptors @@ -577,8 +651,12 @@ mod tests { 1, 1, [ - (first.policy_fingerprint(), summary.clone(), data[0].clone()), - (first.policy_fingerprint(), summary, data[1].clone()), + ( + first.0.policy_fingerprint(), + summary.clone(), + data[0].clone(), + ), + (first.0.policy_fingerprint(), summary, data[1].clone()), ], ) .unwrap_err(); @@ -591,8 +669,7 @@ mod tests { // Imported catalogs must resolve every foreign key and content identity. #[test] fn rejects_dangling_refs_tampered_keys_and_unknown_schema() { - let catalog = - SummaryCatalog::from_materializations(1, 1, &[config("requests", "", 60)]).unwrap(); + let catalog = catalog(1, 1, &[config("requests", "", 60)]).unwrap(); let mut broken = catalog.clone(); broken.data_descriptors.clear(); assert!(matches!( @@ -631,7 +708,7 @@ mod tests { // Native exact-only plans have a valid empty catalog, not dummy state. #[test] fn empty_catalog_is_valid() { - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); + let catalog = catalog(1, 1, &[]).unwrap(); assert!(catalog.outputs.is_empty()); catalog.validate().unwrap(); } diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs index 757a7e15b..b8ef77da5 100644 --- a/crates/asap_types/src/summary_semantics.rs +++ b/crates/asap_types/src/summary_semantics.rs @@ -24,8 +24,6 @@ pub enum SummarySemantics { Configured { computation: Box, value_source_column: Option, - aggregated_labels: crate::KeyByLabelNames, - rollup_labels: crate::KeyByLabelNames, }, } @@ -57,8 +55,6 @@ impl SummaryDefinition { semantic_format_version: 1, semantics: SummarySemantics::Configured { value_source_column: None, - aggregated_labels: crate::KeyByLabelNames::empty(), - rollup_labels: crate::KeyByLabelNames::empty(), computation: Box::new(SummaryComputation { operator: summary.operator.clone(), source: data.source.clone(), @@ -80,14 +76,10 @@ impl SummaryDefinition { }; } else if let SummarySemantics::Configured { value_source_column, - aggregated_labels, - rollup_labels, .. } = &mut self.semantics { *value_source_column = config.value_source_column.clone(); - *aggregated_labels = config.aggregated_labels.clone(); - *rollup_labels = config.rollup_labels.clone(); } self } diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 48f02c7ed..0d58010f6 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -401,30 +401,20 @@ fn matching_streaming_config( use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; - use std::collections::HashMap; - let cfg = PrecomputeMaterialization::new( - AT::Sum, - String::new(), - HashMap::new(), + let family = AT::Sum.planner_exact_family().unwrap(); + let mut cfg = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, ); + cfg.allocate_stored_output_id(&family); let runtime = data_plane::storage_engines::types::InstalledPrecomputePlan::default(); - let plan = control_plane::physical::compiler::PrecomputePlan::build( + let plan = control_plane::physical::compiler::PrecomputePlan::build_backend_local( runtime.plan().envelope.clone(), - vec![cfg], - &[], + vec![(cfg, family)], ) .unwrap(); data_plane::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan(plan).unwrap() diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 02e2f085e..742d7c490 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -69,7 +69,7 @@ async fn main() { capability_snapshot_id: "sql27-main-eval".into(), }; let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(27, 1, &[]).unwrap(); + asap_types::summary_catalog::SummaryCatalog::from_outputs(27, 1, Vec::new()).unwrap(); let reference = catalog.reference().unwrap(); let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 5a3ef0eb4..72a58e84c 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -684,10 +684,7 @@ async fn route_otlp_to_precompute( let ts_ms = (point.timestamp_nanos / 1_000_000) as i64; let mut matched = false; for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { + if config.metric != point.name { continue; } let group_key = IngestState::extract_group_key_for(&series_key, config); @@ -761,10 +758,7 @@ async fn route_otlp_to_precompute( let sketch_type = identify_sketch_type(&point.payload); let mut matched = false; for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { + if config.metric != point.name { continue; } let group_key = IngestState::extract_group_key_for(&series_key, config); @@ -1399,18 +1393,15 @@ async fn route_modified_otlp_sketches_to_precompute( // into the matched policy's parameters — recorded on the sid // below so the query engine can answer per-item estimate(key) // (the CMS/CountSketch FrequencyEstimate gate consults it). - let item_label_for_sid: Option = { - snap.get_aggregation_config(policy_fp.into()) - .or_else(|| { - snap.materializations() - .values() - .find(|c| c.metric == canonical_name) - }) - .and_then(|c| c.parameters.get("item_label")) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - }; + let item_label_for_sid: Option = snap + .get_aggregation_config(policy_fp.into()) + .or_else(|| { + snap.materializations() + .values() + .find(|c| c.metric == canonical_name) + }) + .and_then(|c| snap.item_label(c.stored_output_id)) + .map(str::to_owned); ingest_state.summary_store.register(SummarySeriesMetadata { storage_handle: sid, metric_name: canonical_name.clone(), @@ -1763,11 +1754,7 @@ async fn route_modified_otlp_sketches_to_precompute( &asap_types::aggregation_config::PrecomputeMaterialization, > = agg_configs .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) + .filter(|config| config.metric == canonical_name) .collect(); let matched_any = !matching_configs.is_empty(); @@ -1887,87 +1874,6 @@ async fn route_modified_otlp_sketches_to_precompute( }) } -/// Map `SketchAlgorithm` to the corresponding wire-format -/// `AggregationType`. Inverse direction is in -/// `sketch_algorithm_for` above. Used by -/// [`derive_sketch_policy_fp`] to find the policy whose -/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested -/// sketch. -/// -/// `Any` is a control-plane analysis-time wildcard — it doesn't -/// appear on the ingest path. Returns `None` so the policy lookup -/// fails the (rare) defensive path explicitly. -fn aggregation_type_for_sketch_algorithm( - handle: crate::storage_engines::sketch_db::index::SketchAlgorithm, -) -> Option { - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - use asap_types::AggregationType; - match handle { - SketchAlgorithm::DDSketch => Some(AggregationType::DDSketch), - SketchAlgorithm::Kll => Some(AggregationType::DatasketchesKLL), - SketchAlgorithm::Hll => Some(AggregationType::HLL), - SketchAlgorithm::UnivMon => Some(AggregationType::UnivMon), - SketchAlgorithm::CountSketch => Some(AggregationType::CountSketch), - SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), - SketchAlgorithm::Cms => Some(AggregationType::CountMinSketch), - SketchAlgorithm::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => None, - } -} - -/// Render a `SketchConfig` into the param map the streaming-config -/// stores. The control plane authors these as -/// `parameters: {: }` JSON; the data plane has the -/// parameters typed in `SketchConfig`. This function converts. -/// -/// Keys MUST match what the control plane emits (see -/// `crates/asap_types/src/aggregation_config.rs::from_yaml_data` for -/// the canonical names). Drift here surfaces as policy lookups that -/// silently miss. -fn sketch_config_to_params( - cfg: &crate::storage_engines::sketch_db::data::SketchConfig, -) -> std::collections::HashMap { - use crate::storage_engines::sketch_db::data::SketchConfig; - let mut params = std::collections::HashMap::new(); - match cfg { - SketchConfig::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => { - params.insert("heap_size".into(), serde_json::json!(heap_size)); - params.insert("sketch_rows".into(), serde_json::json!(sketch_rows)); - params.insert("sketch_cols".into(), serde_json::json!(sketch_cols)); - params.insert("layers".into(), serde_json::json!(layers)); - } - SketchConfig::DDSketch { relative_accuracy } => { - params.insert( - "relative_accuracy".to_string(), - serde_json::json!(*relative_accuracy), - ); - } - SketchConfig::Kll { k } => { - params.insert("k".to_string(), serde_json::json!(*k)); - } - SketchConfig::Hll { precision } => { - params.insert("precision".to_string(), serde_json::json!(*precision)); - } - SketchConfig::CountSketch { rows, cols } | SketchConfig::CountMin { rows, cols } => { - // Canonical key mapping (matches the controller's - // `sketch_params_to_json` in - // `control_plane::emit::stage_config`): `w` is the - // matrix width (=cols), `d` is the depth (=rows). The - // controller writes `{w, d}` into the streaming-config - // `parameters`, so the policy_fp content match has to - // probe the same keys. - params.insert("w".to_string(), serde_json::json!(*cols)); - params.insert("d".to_string(), serde_json::json!(*rows)); - } - } - params -} - // Snapshot and delta bases are scoped to the producer, never only its semantics. fn bound_sketch_series_key( name: &str, @@ -1997,21 +1903,42 @@ fn derive_sketch_policy_fp( group_by_keys: &std::collections::BTreeSet, bound_output: Option, ) -> asap_types::PolicyFingerprint { - let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { - return asap_types::PolicyFingerprint::UNSET; - }; - let params = sketch_config_to_params(cfg); + use crate::storage_engines::sketch_db::index::AggKind; + // An unfiltered output of this exact sketch state and grouping; ambiguous + // or absent matches fail closed. + let expected = AggKind::Sketch { + algorithm: kind, + config: cfg.clone(), + spatial_filter_canonical: String::new(), + } + .canonical_string(); let snap = ingest_state.config_snapshot(); - let registry = snap.policy_registry(); - let registry = if let Some(output) = bound_output { - asap_types::PolicyRegistry::from_configs(registry.get(output.fingerprint()).cloned()) - } else { - registry - }; - let index = asap_types::RoutingIndex::build(registry); - index - .find_policy_by_content(metric, group_by_keys, agg_type, ¶ms) - .unwrap_or(asap_types::PolicyFingerprint::UNSET) + let mut hit = None; + for config in snap.materializations().values() { + let keys: std::collections::BTreeSet<_> = config.grouping_labels.iter().cloned().collect(); + if bound_output.is_some_and(|output| output != config.stored_output_id) + || config.metric != metric + || &keys != group_by_keys + // A plain sketch envelope never fills a shared (Hydra) state. + || matches!( + snap.state_family(config.stored_output_id), + Some(planner_types::post_asap::SummaryFamilyType::Sketch( + _, + planner_types::post_asap::GroupingStrategy::SharedMultiSubpopulation { .. } + )) + ) + || snap + .agg_kind(config.stored_output_id) + .is_none_or(|installed| installed.canonical_string() != expected) + { + continue; + } + if hit.is_some() { + return asap_types::PolicyFingerprint::UNSET; + } + hit = Some(config.policy_fingerprint()); + } + hit.unwrap_or(asap_types::PolicyFingerprint::UNSET) } /// Phase 5 helper — map a `ModifiedOtlpSketchDp` to the matching @@ -3325,70 +3252,6 @@ mod policy_fp_lookup_tests { use crate::storage_engines::sketch_db::data::SketchConfig; use crate::storage_engines::sketch_db::index::SketchAlgorithm; use asap_types::AggregationType; - - #[test] - fn handle_to_agg_type_round_trips_canonical_kinds() { - // Locks in the data-plane → control-plane name mapping. - // Drift surfaces as policy lookups that silently miss because - // the handle resolves to an `AggregationType` no policy uses. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::DDSketch), - Some(AggregationType::DDSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kll), - Some(AggregationType::DatasketchesKLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Hll), - Some(AggregationType::HLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Cms), - Some(AggregationType::CountMinSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CmsWithHeap), - Some(AggregationType::CountMinSketchWithHeap) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CountSketch), - Some(AggregationType::CountSketch) - ); - // `Any` is a control-plane wildcard, not a real DP shape. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kmv), - None - ); - } - - #[test] - fn sketch_config_to_params_uses_canonical_keys() { - // The param-name vocabulary must match what the control plane - // writes in streaming-config YAML (see - // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). - // Drift surfaces as `find_policy_by_content` missing matches. - let dd = sketch_config_to_params(&SketchConfig::DDSketch { - relative_accuracy: 0.01, - }); - assert_eq!(dd.get("relative_accuracy"), Some(&serde_json::json!(0.01))); - - let kll = sketch_config_to_params(&SketchConfig::Kll { k: 200 }); - assert_eq!(kll.get("k"), Some(&serde_json::json!(200))); - - let hll = sketch_config_to_params(&SketchConfig::Hll { precision: 14 }); - assert_eq!(hll.get("precision"), Some(&serde_json::json!(14))); - - let cs = sketch_config_to_params(&SketchConfig::CountSketch { rows: 4, cols: 256 }); - // Canonical keys: w (=cols, width) and d (=rows, depth) — - // matches `control_plane::emit::stage_config::sketch_params_to_json`. - assert_eq!(cs.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cs.get("d"), Some(&serde_json::json!(4))); - - let cm = sketch_config_to_params(&SketchConfig::CountMin { rows: 4, cols: 256 }); - assert_eq!(cm.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cm.get("d"), Some(&serde_json::json!(4))); - } } #[cfg(test)] @@ -4481,23 +4344,19 @@ mod sid_bucketing_tests { } fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::SingleSubpopulation, - "Sum".to_string(), - HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.allocate_stored_output_id(&sum_family()); + config + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } /// Build a Gauge request with one DataPoint per (zone, value) entry. @@ -4573,9 +4432,7 @@ mod sid_bucketing_tests { let metric = "cpu_seconds"; let cfg = sum_agg_config(metric, &["zone"]); let policy_fp = asap_types::PolicyFingerprint(cfg.policy_fp_u64()); - let mut configs = HashMap::new(); - configs.insert(cfg.policy_fp_u64(), cfg.clone()); - let streaming = InstalledPrecomputePlan::from_raw_ids(configs); + let streaming = InstalledPrecomputePlan::new([(cfg.clone(), sum_family())]); let hot_reload = InstalledPrecomputePlanHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); @@ -4659,8 +4516,10 @@ mod sid_bucketing_tests { // regardless of group_key shape — the test pin is on sid // assignment, not on group_key content), then verify the // sid matches the resolver mint for THAT zone. - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(&cfg); + let agg_kind_canonical = crate::storage_engines::sketch_db::data::materialization_kind( + &cfg, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), ""), + ); for (sid, _, _, samples) in &groups { let mut vals: Vec = samples.iter().map(|(_, _, v)| *v).collect(); vals.sort_by(|a, b| a.partial_cmp(b).unwrap()); @@ -4693,7 +4552,7 @@ mod sid_bucketing_tests { ("asap.frame.plan_version".into(), "3".into()), ( "asap.frame.backend_compat".into(), - "asap-query-backend.v1".into(), + "asap-query-backend.v2".into(), ), ("asap.frame.materialization".into(), "99".into()), ( diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index ff117d7b0..14fe449b4 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -452,11 +452,7 @@ impl PrometheusRemoteWriteReceiver { .zip(group_key.values().labels) .collect(); } - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = snapshot.right_closed_panes((*policy_fp).into()); let affected = crate::precompute_engine::maintenance_runtime::affected_materializations( &physical_plan.precompute_plan, (*policy_fp).into(), @@ -726,7 +722,7 @@ fn route_messages( .values() .filter(|config| config.derived_input.is_none()) .filter_map(|config| { - compile_spatial_filter(&config.spatial_filter_normalized) + compile_spatial_filter(snapshot.population_filter(config.stored_output_id)) .ok() .map(|filter| (config, filter)) }) @@ -754,11 +750,14 @@ fn route_messages( == Some(asap_types::sds::PopulationPartitioning::PerEntity) || (config.partitioning.is_none() && matches!( - config.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::Rate - | asap_types::AggregationType::Min - | asap_types::AggregationType::Max + snapshot.state_family(config.stored_output_id), + Some(planner_types::post_asap::SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate + | planner_types::post_asap::ExactKind::Min + | planner_types::post_asap::ExactKind::Max, + _ + )) )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -986,14 +985,25 @@ mod tests { planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - let configs = streaming - .materializations_by_output - .values() - .cloned() - .collect::>(); let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 3, &configs) - .unwrap(), + asap_types::summary_catalog::SummaryCatalog::from_outputs( + 7, + 3, + streaming + .materializations_by_output + .values() + .map(|config| { + ( + config, + streaming.state_family(config.stored_output_id).unwrap(), + streaming + .population_filter(config.stored_output_id) + .to_owned(), + ) + }) + .collect(), + ) + .unwrap(), ); let reference = catalog.reference().unwrap(); let generation = asap_types::sds::CatalogGeneration { @@ -1066,37 +1076,19 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let aggregation = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(vec!["job".into()]).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 60, - slide_interval: 60, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: "requests_total".into(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let policy_fp = aggregation.policy_fp_u64(); + let mut aggregation = PrecomputeMaterialization::new( + "requests_total", + KeyByLabelNames::new(vec!["job".into()]), + 60, + 60, + WindowKind::Tumbling, + ); + aggregation.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }; let streaming = - InstalledPrecomputePlan::from_raw_ids(HashMap::from([(policy_fp, aggregation)])); + InstalledPrecomputePlan::new([crate::tests::test_utilities::outputs::allocated( + aggregation, + AggregationType::Sum.planner_exact_family().unwrap(), + )]); let (sender, receiver) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1134,10 +1126,13 @@ mod tests { let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let hot = physical_config(InstalledPrecomputePlan::from_raw_ids(HashMap::from([( - config.policy_fp_u64(), - config.clone(), - )]))); + let (config, family) = crate::tests::test_utilities::outputs::allocated( + config, + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap(), + ); + let hot = physical_config(InstalledPrecomputePlan::new([(config.clone(), family)])); let physical = hot.active_physical_plan_snapshot().unwrap(); // A direct routing fixture; public installation remains intentionally gated. let (sender, _worker) = mpsc::channel(8); @@ -1195,67 +1190,42 @@ mod tests { use asap_types::enums::WindowKind; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let config = |aggregation_type, grouping: Vec, aggregated: Vec| { - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + let config = |aggregation_type: AggregationType, grouping: Vec, partitioning| { + let family = crate::tests::test_utilities::outputs::family( aggregation_type, - aggregation_sub_type: String::new(), - parameters: match aggregation_type { - AggregationType::CountMinSketchWithHeap => HashMap::from([ - ("w".into(), serde_json::json!(128)), - ("d".into(), serde_json::json!(5)), - ("heap_size".into(), serde_json::json!(2)), - ]), - AggregationType::DatasketchesKLL => { - HashMap::from([("k".into(), serde_json::json!(200))]) - } - _ => HashMap::new(), - }, - grouping_labels: KeyByLabelNames::new(grouping).into(), - aggregated_labels: KeyByLabelNames::new(aggregated), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 60, - slide_interval: 60, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - pane_origin_ms: Some(0), - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: "cpu_seconds_total".into(), - num_aggregates_to_retain: Some(80), - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + &serde_json::json!({"w": 128, "d": 5, "heap_size": 2, "k": 200}), + ); + let mut config = PrecomputeMaterialization::new( + "cpu_seconds_total", + KeyByLabelNames::new(grouping), + 60, + 60, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }; + config.pane_origin_ms = Some(0); + config.num_aggregates_to_retain = Some(80); + config.partitioning = partitioning; + crate::tests::test_utilities::outputs::allocated(config, family) }; - let cms = config( - AggregationType::CountMinSketchWithHeap, + let cms = config(AggregationType::CountMinSketchWithHeap, vec![], None); + let counter = config(AggregationType::Increase, vec!["job".into()], None); + let kll = config( + AggregationType::DatasketchesKLL, + vec![], + Some(asap_types::sds::PopulationPartitioning::PerEntity), + ); + let kll_fp = kll.0.policy_fingerprint(); + let pooled_kll = config( + AggregationType::DatasketchesKLL, vec![], - vec!["job".into()], + Some(asap_types::sds::PopulationPartitioning::Grouped), ); - let counter = config(AggregationType::Increase, vec!["job".into()], vec![]); - let mut kll = config(AggregationType::DatasketchesKLL, vec![], vec![]); - kll.partitioning = Some(asap_types::sds::PopulationPartitioning::PerEntity); - let kll_fp = kll.policy_fingerprint(); - let mut pooled_kll = kll.clone(); - pooled_kll.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let pooled_kll_fp = pooled_kll.policy_fingerprint(); + let pooled_kll_fp = pooled_kll.0.policy_fingerprint(); assert_ne!(kll_fp, pooled_kll_fp); - let cms_fp = cms.policy_fingerprint(); - let counter_fp = counter.policy_fingerprint(); - let streaming = InstalledPrecomputePlan::from_raw_ids(HashMap::from([ - (cms_fp.0, cms), - (counter_fp.0, counter), - (kll_fp.0, kll), - (pooled_kll_fp.0, pooled_kll), - ])); + let cms_fp = cms.0.policy_fingerprint(); + let counter_fp = counter.0.policy_fingerprint(); + let streaming = InstalledPrecomputePlan::new([cms, counter, kll, pooled_kll]); let hot_reload = physical_config(streaming); let physical_plan = hot_reload.active_physical_plan_snapshot().unwrap(); let (sender, _worker) = mpsc::channel(8); @@ -1824,7 +1794,9 @@ impl PrometheusRemoteWriteReceiver { return Err(crate::precompute_engine::revisions::AdmissionRejected("Remote Write request exceeds the correction horizon or contains future input").into()); } for config in plan.precompute_plan.materializations.iter().filter(|c| c.derived_input.is_none()) { - let filter = compile_spatial_filter(&config.spatial_filter_normalized)?; + let filter = compile_spatial_filter( + plan.installed_precompute_plan.population_filter(config.stored_output_id), + )?; let program = plan.installed_precompute_plan.raw_programs.get(&config.policy_fp_u64()).ok_or("revision raw program missing")?; let mut matching = samples.iter().filter_map(|sample| { let labels = sample.labels.iter().map(|(k,v)|(k.clone(),v.clone())).collect(); @@ -1921,7 +1893,7 @@ impl PrometheusRemoteWriteReceiver { if let Some(previous) = committed.get(&output.0) { let config = plan.precompute_plan.materializations.iter().find(|c| c.policy_fp_u64() == output.0).ok_or("committed raw output missing")?; let mut groups: BTreeMap, BTreeMap<(u64,u64),Arc>> = BTreeMap::new(); - for record in previous { groups.entry(record.group.clone()).or_default().insert((record.start_ms,record.end_ms),crate::precompute_engine::revisions::decode_state(record,config)?); } + for record in previous { groups.entry(record.group.clone()).or_default().insert((record.start_ms,record.end_ms),crate::precompute_engine::revisions::decode_state(record, plan.precompute_plan.state_family(config.stored_output_id).ok_or("committed raw output has no state schema")?)?); } for (group,windows) in groups { frozen.push(crate::storage_engines::sketch_db::index::FrozenExactWindows { stored_output_reference: plan.installed_precompute_plan.stored_output_reference(output).ok_or("committed raw binding missing")?, storage_handle: output.0, definition: output, generation:Arc::new(generation.clone()), group, windows, singleton_population_complete:true, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 610f57a5b..19556793b 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2531,7 +2531,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 1, &[]).unwrap(), + asap_types::summary_catalog::SummaryCatalog::from_outputs(7, 1, Vec::new()).unwrap(), ); let generation = catalog.reference().unwrap(); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -3267,46 +3267,30 @@ mod tests { // retirement) can find them. The matching sid in the catalog // is registered via the canonical ingest path so its content // hash matches what `create_checked` would compute. - let mut agg_map = HashMap::new(); + let mut agg_map = Vec::new(); let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::empty().into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.clone(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; + let mut cfg = PrecomputeMaterialization::new( + metric.clone(), + KeyByLabelNames::empty(), + 1, + 1, + WindowKind::Tumbling, + ); + cfg.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + let cfg = crate::tests::test_utilities::outputs::allocated( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + ); // PR 5: streaming-config is keyed on the policy // fingerprint. Build a marker→fingerprint map so the test // POSTs the right id on the wire. - let fp = cfg.policy_fp_u64(); + let fp = cfg.0.policy_fp_u64(); marker_to_fp.insert(*marker, fp); - agg_map.insert(fp, cfg); + agg_map.push(cfg); } - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::from_raw_ids(agg_map)); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::new(agg_map)); let hot_reload = InstalledPrecomputePlanHandle::from_arc(installed_precompute_plan.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -3687,10 +3671,8 @@ mod tests { }; // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. - let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( - Default::default(), - metric_storage_backend, - ); + let streaming_cfg = + InstalledPrecomputePlan::with_storage_backend(Vec::new(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); @@ -5017,10 +4999,8 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( - Default::default(), - metric_storage_backend, - ); + let streaming_cfg = + InstalledPrecomputePlan::with_storage_backend(Vec::new(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); @@ -6557,6 +6537,25 @@ mod catalog_install_tests { ) } + // A physical plan compiled before materializations became deployment-only + // is refused at decode by its schema version, not by a field-level error. + #[test] + fn install_request_from_the_computation_carrying_schema_is_rejected_by_version() { + let mut encoded = serde_json::to_value(request()).unwrap(); + assert!(serde_json::from_value::(encoded.clone()).is_ok()); + encoded["precompute_plan"]["envelope"]["backend_compat"] = + serde_json::json!("asap-query-backend.v1"); + encoded["precompute_plan"]["materializations"][0]["aggregation_type"] = + serde_json::json!("DDSketch"); + let error = serde_json::from_value::(encoded) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsupported installed precompute plan schema asap-query-backend.v1"), + "{error}" + ); + } + #[test] fn complete_dag_cannot_be_installed_as_maintenance() { let mut request = request(); diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index e97045fe5..c1384acb6 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -56,18 +56,22 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::PrecomputeMaterialization, + program: &crate::precompute_engine::raw_dag::RawDagProgram, timestamp_ms: i64, value: f64, ) { - use asap_types::{AggregationType, SampleUpdateRule}; - let (semantics, sketch, implementation) = match config.aggregation_type { - AggregationType::HLL => ( + use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; + let algorithm = match &program.family { + SummaryFamilyType::Sketch(kind, _) => kind.algorithm(), + _ => return, + }; + let (semantics, sketch, implementation) = match algorithm { + SketchAlgorithm::Hll => ( ErpObservationInputSemantics::ScalarSampleValue, "hll", "asap-sketchlib-hll-regular-v1", ), - AggregationType::UnivMon => ( + SketchAlgorithm::UnivMon => ( ErpObservationInputSemantics::UnitSampleFrequency, "univmon", "asap-sketchlib-univmon-standard-v1", @@ -83,12 +87,10 @@ impl RuntimeErpObserver { if state.invalid.is_some() { return; } - if !value.is_finite() - || !matches!( - config.sample_update_rule(), - SampleUpdateRule::Value { scale: 1.0 } - ) - { + // Both implemented raw update domains observe the raw sample value. + let untransformed = asap_types::accumulator_spec::is_scalar_sample_value(&program.input) + || asap_types::accumulator_spec::is_unit_sample_frequency(&program.input); + if !value.is_finite() || !untransformed { state.invalid = Some("unsupported non-finite or transformed summary input".into()); for population in state.populations.values_mut() { population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); @@ -264,23 +266,20 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { - let config = asap_types::PrecomputeMaterialization::new( - asap_types::AggregationType::HLL, - String::new(), - Default::default(), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - 60, - 60, - asap_types::enums::WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, + fn fixture() -> ( + CatalogGeneration, + crate::precompute_engine::raw_dag::RawDagProgram, + ) { + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryFamilyType, + SummaryUpdate, + }; + let config = crate::precompute_engine::raw_dag::RawDagProgram::fixture( + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + GroupingStrategy::PerSubpopulationInstance, + ), + SummaryUpdate::column(planner_types::pre_asap::ColumnRef::SampleValue), ); ( CatalogGeneration { diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index da79aad47..ba0ba21db 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -198,7 +198,7 @@ mod tests { identity_version: 1, plan_id: 7, plan_version: 3, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), materialization: asap_types::PolicyFingerprint(41).into(), series_identity: "service=checkout,zone=a".into(), schema_id: "schema-41".into(), diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index a60e9fbd2..feba9a10e 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -301,27 +301,26 @@ mod tests { use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { - // `_agg_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Kept as a parameter to - // avoid churning the call sites below. - PrecomputeMaterialization::new( - AggregationType::CountMinSketch, - String::new(), - std::collections::HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + fn make_config( + agg_id: u64, + metric: &str, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.stored_output_id = asap_types::sds::StoredOutputId(agg_id); + let family = crate::tests::test_utilities::outputs::family( + AggregationType::CountMinSketch, + &serde_json::json!({}), + ); + (config, family) } /// Set up an `IngestState` with one Active agg for `metric` and a @@ -334,9 +333,7 @@ mod tests { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); - let mut map = std::collections::HashMap::new(); - map.insert(agg_id, make_config(agg_id, metric)); - let streaming = InstalledPrecomputePlan::from_raw_ids(map); + let streaming = InstalledPrecomputePlan::new([make_config(agg_id, metric)]); let hot_reload = crate::storage_engines::types::InstalledPrecomputePlanHandle::new(streaming.clone()); diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index fe42f2d2f..6eb59d33f 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -501,10 +501,18 @@ pub(crate) fn execute_completed_maintenance_cohort( target.fingerprint(), ); output.catalog_generation = Some(Arc::clone(generation)); + let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { family, .. } = + &target_node.payload + else { + return Err("maintenance target node is not a Planner SummaryAgg".into()); + }; + // Derived outputs read stored states, so their input has no predicate. + let kind = crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""); store .publish_frozen_maintenance_output( target_sid, target_config, + &kind, &output, state.as_ref(), &cohort, @@ -784,7 +792,12 @@ fn execute_finite_complete_populations( batch.schema(), ) { output.population_labels = Some(Population::new()); - store.publish_native_summary_output(resolver, config, &output, batch, max_bytes)?; + let family = plan + .state_family(config.stored_output_id) + .ok_or("physical output has no state schema")?; + store.publish_native_summary_output( + resolver, config, family, &output, batch, max_bytes, + )?; continue; } let derived = config @@ -821,9 +834,13 @@ fn execute_finite_complete_populations( continue; } output.population_labels = Some(group); + let family = plan + .state_family(config.stored_output_id) + .ok_or("complete maintenance target has no state schema")?; store.publish_complete_raw_maintenance_output( target_sid, config, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""), &output, state.as_ref(), &cohort, @@ -1720,6 +1737,45 @@ mod tests { } } + /// Raw sources of these fixtures store exact sums. + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap() + } + + /// The maintained output of these fixtures stores a KLL. + fn kll_family() -> planner_types::post_asap::SummaryFamilyType { + crate::tests::test_utilities::outputs::family( + asap_types::AggregationType::DatasketchesKLL, + &serde_json::json!({"k": 200}), + ) + } + + fn sum_kind() -> crate::storage_engines::sketch_db::index::AggKind { + crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), "") + } + + /// The catalog of raw sum sources followed by the maintained KLL output. + fn fixture_catalog( + plan_id: u64, + plan_version: u64, + configs: &[asap_types::PrecomputeMaterialization], + ) -> asap_types::summary_catalog::SummaryCatalog { + let (sum, kll) = (sum_family(), kll_family()); + let (sources, target) = configs.split_at(configs.len() - 1); + asap_types::summary_catalog::SummaryCatalog::from_outputs( + plan_id, + plan_version, + sources + .iter() + .map(|config| (config, &sum, String::new())) + .chain(target.iter().map(|config| (config, &kll, String::new()))) + .collect(), + ) + .unwrap() + } + fn sum(value: f64) -> SummaryState { let mut accumulator = crate::tests::accumulator_fixture::sum_state(0.0); accumulator.update(None, value, 0); @@ -1754,17 +1810,14 @@ mod tests { source_config.window_type = asap_types::WindowKind::Tumbling; source_config.window_layout = asap_types::aggregation_config::WindowMaterializationLayout::Pane { pane_secs: 1 }; + source_config.allocate_stored_output_id(&sum_family()); let source_definition = source_config.policy_fingerprint().into(); let mut target_config = source_config.clone(); target_config.window_layout = asap_types::aggregation_config::WindowMaterializationLayout::FullWindow; - target_config.aggregation_type = asap_types::AggregationType::DatasketchesKLL; - target_config.aggregation_sub_type = "quantile".into(); - target_config - .parameters - .insert("k".into(), serde_json::json!(200)); + target_config.allocate_stored_output_id(&kll_family()); let target = target_config.policy_fingerprint().into(); - let target_family = target_config.accumulator_spec().unwrap().family; + let target_family = kll_family(); let configs = [source_config, target_config]; let binding = BackendExecutableBinding { nodes: BTreeMap::from([ @@ -1815,7 +1868,7 @@ mod tests { read.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; aggregate.output_schema.fields = vec![SummaryField { name: "state".into(), - dtype: configs[1].accumulator_spec().unwrap().family, + dtype: kll_family(), nullable: false, }]; let mut query = node(4); @@ -1868,6 +1921,7 @@ mod tests { ) .unwrap(), ); + durable_configs[1].allocate_stored_output_id(&kll_family()); document.schema_version = asap_types::executable_plan::PRECOMPUTE_DAG_SCHEMA_VERSION; let mut durable_binding = scheduled_binding.clone(); durable_binding.nodes.insert( @@ -1887,14 +1941,7 @@ mod tests { document, binding: durable_binding, }; - let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 1, - &durable_configs, - ) - .unwrap(), - ); + let catalog = Arc::new(fixture_catalog(1, 1, &durable_configs)); let directory = tempfile::tempdir().unwrap(); let persistence_config = || { let mut config = SketchStorePersistenceConfig::with_memory_limit( @@ -1941,6 +1988,7 @@ mod tests { writer.ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &output, sum(value).as_ref(), ) @@ -1977,6 +2025,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &rejected_output, sum(99.0).as_ref() ) @@ -2066,6 +2115,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &correction, sum(100.0).as_ref() ) @@ -2108,6 +2158,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &correction, sum(100.0).as_ref() ) @@ -2130,14 +2181,7 @@ mod tests { }) .sum::(); assert_eq!(target_entries, 1); - let next_catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 2, - &durable_configs, - ) - .unwrap(), - ); + let next_catalog = Arc::new(fixture_catalog(1, 2, &durable_configs)); restored.install_summary_catalog(next_catalog).unwrap(); let next_generation = restored.active_catalog_generation().unwrap(); assert!(restored @@ -2154,6 +2198,7 @@ mod tests { restored.ingest_precompute_with_series_id( 602, &durable_configs[0], + &sum_kind(), &new_population, sum(20.0).as_ref() ), @@ -2188,8 +2233,10 @@ mod tests { first.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; first.partitioning = Some(asap_types::sds::PopulationPartitioning::PerEntity); } + first.allocate_stored_output_id(&sum_family()); let mut second = first.clone(); second.metric = "second_maintenance_source".into(); + second.allocate_stored_output_id(&sum_family()); let first_id = first.policy_fingerprint().into(); let second_id = second.policy_fingerprint().into(); let mut dag = template.clone(); @@ -2277,10 +2324,7 @@ mod tests { binding, }; let configs = [first, second, target]; - let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(2, 1, &configs) - .unwrap(), - ); + let catalog = Arc::new(fixture_catalog(2, 1, &configs)); let directory = tempfile::tempdir().unwrap(); let persistence_config = || { let mut config = @@ -2316,12 +2360,22 @@ mod tests { // is tested with the real compiler/process integration separately. config.population_key_encoding = asap_types::PopulationKeyEncoding::LegacyDelimited; - config + (config, sum_family()) }) .collect(), ) .unwrap(); plan.materializations = configs.to_vec(); + // The unvalidated fixture target still names its stored state schema. + plan.schemas + .push(asap_types::precompute_plan::StateSchemaContract { + materialization: configs[2].stored_output_id, + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + configs[2].stored_output_id, + ), + family: kll_family(), + ..plan.schemas[0].clone() + }); plan.summary_catalog = Some(generation.as_ref().clone()); plan.executable_dags = BTreeMap::from([("cohort-fixture".into(), installed.clone())]); let resolver_path = directory.path().join("resolver.wal"); @@ -2380,6 +2434,7 @@ mod tests { writer.ingest_precompute_with_series_id( sid, config, + &sum_kind(), &output, sum(value + population_index as f64 * 10.0).as_ref(), ) @@ -2569,10 +2624,7 @@ mod tests { ) .unwrap()); assert_eq!(persistence.manifest.live_parts().len(), parts); - let next_catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(2, 2, &configs) - .unwrap(), - ); + let next_catalog = Arc::new(fixture_catalog(2, 2, &configs)); restored.install_summary_catalog(next_catalog).unwrap(); assert_ne!(generation, restored.active_catalog_generation().unwrap()); // Stale scheduling fails at the captured catalog boundary, before any @@ -2790,7 +2842,7 @@ mod tests { summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Default::default())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Vec::new())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3018,7 +3070,11 @@ pub(crate) fn execute_revision_outputs( for (group, state) in super::native_precompute::population_states(&batch, window.1)? { - let payload = encode_state(state, config.accumulator_spec()?.family)?; + let family = plan + .precompute_plan + .state_family(config.stored_output_id) + .ok_or("physical output has no state schema")?; + let payload = encode_state(state, family.clone())?; if payload.len() > limit { return Err(asap_physical_operators::Error::MemoryLimit.into()); } diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 2a294ed9f..756e0cc6e 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -106,10 +106,8 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via - /// `PolicyRegistry::get(output.policy_fp)`. The legacy - /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` - /// is the only identity handle on `PrecomputedOutput`. Outputs + /// Resolves the stored output from the installed plan by + /// `output.policy_fp`, the only identity handle on `PrecomputedOutput`. Outputs /// emitted with the `PolicyFingerprint::UNSET` sentinel (e.g. /// raw-mode fast-path that has no source config) are skipped /// rather than routed by a parallel id. @@ -122,9 +120,11 @@ impl SketchStoreSink { return false; } let cfg = self.hot_reload.snapshot(); - let registry = cfg.policy_registry(); - let agg_cfg = match registry.get(output.policy_fp) { - Some(c) => c.clone(), + let installed = cfg + .get_aggregation_config(output.policy_fp.into()) + .zip(cfg.agg_kind(output.policy_fp.into())); + let (agg_cfg, kind) = match installed { + Some(installed) => installed, None => { // CQ-6 — policy-miss drop: a content-addressed policy_fp // that the running streaming-config registry doesn't know @@ -138,7 +138,6 @@ impl SketchStoreSink { return false; } }; - let agg_cfg = &agg_cfg; let Some(reference) = cfg.stored_output_reference(output.policy_fp.into()) else { return false; }; @@ -175,6 +174,7 @@ impl SketchStoreSink { &address, &self.series_resolver, agg_cfg, + &kind, output, accumulator, ) @@ -372,50 +372,33 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Callers obtain the id - // via `config.policy_fp_u64()`. - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new( - grouping_keys.iter().map(|s| s.to_string()).collect(), - ) - .into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + fn sum_agg_config( + _id: u64, + metric: &str, + grouping_keys: &[&str], + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + metric, + KeyByLabelNames::new(grouping_keys.iter().map(|s| s.to_string()).collect()), + 1, + 1, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + crate::tests::test_utilities::outputs::allocated( + config, + AggregationType::Sum.planner_exact_family().unwrap(), + ) } #[test] fn sketch_index_sink_writes_to_index() { let cfg = sum_agg_config(7, "cpu_seconds", &["zone"]); - let agg_id = cfg.policy_fp_u64(); - let mut configs = HashMap::new(); - configs.insert(agg_id, cfg); - let streaming = InstalledPrecomputePlan::from_raw_ids(configs); + let agg_id = cfg.0.policy_fp_u64(); + let streaming = InstalledPrecomputePlan::new([cfg]); let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); @@ -469,11 +452,11 @@ mod tests { #[test] fn sink_reactivates_catalog_series_without_reusing_retired_payload() { let cfg = sum_agg_config(7, "cpu_seconds", &[]); - let fingerprint = cfg.policy_fingerprint(); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let fingerprint = cfg.0.policy_fingerprint(); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[cfg.clone()], + vec![(&cfg.0, &cfg.1, String::new())], ) .unwrap(); let store = Arc::new(SketchStore::new()); @@ -485,9 +468,7 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::from_raw_ids( - HashMap::from([(fingerprint.0, cfg)]), - )), + InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new([cfg])), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -572,14 +553,16 @@ mod tests { #[test] fn sketch_policy_is_registered_and_stored_as_sketch_state() { - let mut cfg = sum_agg_config(8, "latency", &[]); - cfg.aggregation_type = AggregationType::DDSketch; - cfg.parameters - .insert("alpha".into(), serde_json::json!(0.01)); - let policy_fp = cfg.policy_fp_u64(); - let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::from_raw_ids( - HashMap::from([(policy_fp, cfg)]), - )); + let (cfg, _) = sum_agg_config(8, "latency", &[]); + let cfg = crate::tests::test_utilities::outputs::allocated( + cfg, + crate::tests::test_utilities::outputs::family( + AggregationType::DDSketch, + &serde_json::json!({"alpha": 0.01}), + ), + ); + let policy_fp = cfg.0.policy_fp_u64(); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new([cfg])); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs index fb46b81b2..15d9999e4 100644 --- a/data_plane/src/precompute_engine/raw_dag.rs +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -89,18 +89,16 @@ impl RawDagProgram { source: Source::TimeSeries { metric }, .. } if metric == &config.metric => { - let (metric, window, filter) = - control_plane::physical::compiler::raw_time_series_input_contract( + let (metric, window, _) = + asap_types::precompute_plan::raw_time_series_input_contract( expression, matches!(family, SummaryFamilyType::ExactAggregate(..)), )?; if metric != config.metric || window.is_some_and(|seconds| seconds != config.window_size) - || asap_types::utils::normalize_spatial_filter(&filter) - != config.spatial_filter_normalized { return Err( - "raw DAG source filter/window differs from physical binding".into(), + "raw DAG source window differs from physical binding".into() ); } } @@ -151,35 +149,6 @@ impl RawDagProgram { return Err("raw exact update differs from stored source projection".into()); } } - if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { - return Err( - "materialization storage family differs from selected Planner node".into(), - ); - } - // The stored descriptor must name the same update semantics; its - // content identity cannot be reused for an unrelated DAG program. - let update_matches = match (&input.weight, config.sample_update_rule()) { - ( - SummaryInputExpr::Column(_), - asap_types::SampleUpdateRule::Value { scale }, - ) => scale == 1.0, - (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { - *value == 1.0 - } - _ => { - asap_types::accumulator_spec::is_unit_sample_frequency(input) - || (matches!( - family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - _ - ) - ) && input.weight == SummaryInputExpr::Constant(1.0)) - } - }; - if !update_matches { - return Err("DAG update differs from stored summary identity".into()); - } let compiled = installed .native_program(node.id)? .ok_or("raw materialization lacks its Planner precompute graph")?; @@ -220,6 +189,21 @@ impl RawDagProgram { selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) } + /// A program with the given node semantics and no executable graph, for + /// tests of consumers that only inspect the node. + #[cfg(test)] + pub(crate) fn fixture(family: SummaryFamilyType, input: SummaryUpdate) -> Self { + Self { + node: PostAsapNodeId(1), + family, + input, + grouping: GroupingStrategy::PerSubpopulationInstance, + reduction: planner_types::pre_asap::Reduction::PerEntity, + program: std::sync::Arc::from(Vec::new()), + source: 0, + } + } + /// Execute the Planner graph over one pane's samples as one typed batch. /// Returns `None` when the graph admits no population from them. pub fn build<'a>( diff --git a/data_plane/src/precompute_engine/revisions.rs b/data_plane/src/precompute_engine/revisions.rs index 9aecb674a..14ef43c4a 100644 --- a/data_plane/src/precompute_engine/revisions.rs +++ b/data_plane/src/precompute_engine/revisions.rs @@ -458,10 +458,13 @@ pub(crate) fn encode_state( pub(crate) fn decode_state( record: &RevisionRecord, - config: &asap_types::PrecomputeMaterialization, + family: &SummaryFamilyType, ) -> Result, RevisionError> { - let family = config.accumulator_spec()?.family; - let batch = native::decode_batch(&record.payload, state_schema(family), record.payload.len())?; + let batch = native::decode_batch( + &record.payload, + state_schema(family.clone()), + record.payload.len(), + )?; match batch.rows() { [row] => match row.as_slice() { [Value::Summary { state, .. }] => { @@ -632,7 +635,12 @@ impl RevisionRuntime { if decode_native_record(&plan, record, self.policy.max_checkpoint_bytes)? .is_none() { - decode_state(record, config)?; + decode_state( + record, + plan.precompute_plan + .state_family(config.stored_output_id) + .ok_or("recovered revision has no state schema")?, + )?; } } } diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index d1b1591bb..a367da198 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -14,7 +14,6 @@ use asap_summary_state::codec::KeyCodec; use asap_summary_state::StoredState; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; -use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; // (PolicyFingerprint is used for both `PolicyFingerprint::from_config(...)` // on the emit path and the `policy_fp` field of GroupState below.) @@ -39,6 +38,9 @@ use tracing::{debug, debug_span, info, warn}; /// `GROUP BY window, key`. struct GroupState { program: Option>, + /// Stored state family for isolated kernel tests without a Planner program. + #[cfg(test)] + fixture_family: Option, series_id: u64, stored_output_reference: Option, catalog_generation: Option>, @@ -60,10 +62,6 @@ struct GroupState { /// Samples admitted to each open pane, keyed by pane_start_ms, in arrival /// order. The installed Planner graph builds the pane's state at close. active_panes: BTreeMap, i64, f64)>>, - /// Last cumulative counter sample per source series for heap membership - /// materializations. This is bounded O(series) derivative state, not a - /// raw-sample history, and deliberately survives pane rotation. - counter_previous: HashMap, /// Active panes for pre-built accumulator inputs (e.g. OTLP-delivered /// sketches), keyed by pane_start_ms. Each entry is the running merge /// of every accumulator that landed in that pane's time range. Kept @@ -90,6 +88,16 @@ struct PaneWallClock { } impl GroupState { + #[cfg(test)] + fn fixture_family(&self) -> Option<&planner_types::post_asap::SummaryFamilyType> { + self.fixture_family.as_ref() + } + + #[cfg(not(test))] + fn fixture_family(&self) -> Option<&planner_types::post_asap::SummaryFamilyType> { + None + } + fn stores_full_windows(&self) -> bool { matches!( self.config.window_layout, @@ -522,6 +530,8 @@ impl Worker { let config = Arc::new(cfg.clone()); let gs = GroupState { program, + #[cfg(test)] + fixture_family: snap.state_family(policy_fp.into()).cloned(), series_id: sid, stored_output_reference: snap.stored_output_reference(policy_fp.into()), catalog_generation: self.current_catalog_generation.clone(), @@ -536,7 +546,6 @@ impl Worker { policy_fp, group_key: Arc::clone(group_key), active_panes: BTreeMap::new(), - counter_previous: HashMap::new(), sketch_panes: BTreeMap::new(), max_event_time_ms: i64::MIN, closure_watermark_ms: i64::MIN, @@ -588,12 +597,7 @@ impl Worker { // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. - let right_closed = state - .config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = state.program.is_some(); let pane_timestamp = |ts: i64| { if right_closed { ts.saturating_sub(1) @@ -635,11 +639,7 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = if legacy_counter_delta(state) { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = Some(*val); for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -686,25 +686,18 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if legacy_counter_delta(state) { - if let Some(input) = state.input_revisions.get_mut(&bucket_start) { - Arc::make_mut(input).first_revision = 0; - } - record_late_input("drop", "counter_delta_membership"); - continue; - } record_late_input("append_correction", "raw_sample"); let Some(correction) = build_pane( state.program.as_deref(), - &state.config, + state.fixture_family(), &[(Arc::from(series_key.as_str()), *ts, *val)], (bucket_start, bucket_end), )? else { continue; }; - if let (Some(observer), Some(revision)) = - (&self.erp_observer, &input_revision) + if let (Some(observer), Some(revision), Some(program)) = + (&self.erp_observer, &input_revision, &state.program) { observer.observe( &revision.generation, @@ -716,7 +709,7 @@ impl Worker { }, group_values: group_key.as_population_labels(), }, - &state.config, + program, *ts, *val, ); @@ -751,7 +744,8 @@ impl Worker { let pane = state.active_panes.entry(bucket_start).or_default(); if let Some(value) = value { pane.push((Arc::clone(&series), *ts, value)); - if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) + if let (Some(observer), Some(revision), Some(program)) = + (&self.erp_observer, &input_revision, &state.program) { observer.observe( &revision.generation, @@ -763,7 +757,7 @@ impl Worker { }, group_values: group_key.as_population_labels(), }, - &state.config, + program, *ts, value, ); @@ -1594,7 +1588,7 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { /// Build a pane's state from its samples with the installed Planner graph. fn build_pane( program: Option<&super::raw_dag::RawDagProgram>, - config: &PrecomputeMaterialization, + fixture_family: Option<&planner_types::post_asap::SummaryFamilyType>, samples: &[(Arc, i64, f64)], pane: (i64, i64), ) -> Result>, String> { @@ -1612,15 +1606,21 @@ fn build_pane( #[cfg(test)] { let _ = pane; - let mut updater = create_fixture_accumulator(config); + let family = fixture_family.ok_or("fixture output has no state family")?; + let mut updater = create_fixture_accumulator( + family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + ); for (series, time, value) in samples { - apply_sample(&mut *updater, series, *value, *time, config); + apply_sample(&mut *updater, series, *value, *time, family); } Ok(Some(updater.take_accumulator())) } #[cfg(not(test))] { - let _ = (config, samples, pane); + let _ = (fixture_family, samples, pane); Err("missing installed Planner producer".into()) } } @@ -1635,7 +1635,7 @@ fn close_pane( }; let built = build_pane( state.program.as_deref(), - &state.config, + state.fixture_family(), samples, state.bucket_bounds(start), )?; @@ -1643,34 +1643,24 @@ fn close_pane( Ok(built) } -/// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. -/// -/// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted -/// from the series' **aggregated_labels** — these are the labels that become -/// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL -/// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +/// Route a single fixture sample to `updater`. Planner's PromQL Top-K item +/// is the series identity; other keyed fixtures project no item labels. #[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, ) { + use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; if updater.is_keyed() { - // Planner's PromQL Top-K item is the series identity. When no - // explicit aggregated labels are projected, retain the canonical - // series key instead of collapsing every series onto an empty item. - let key = if config.aggregated_labels.labels.is_empty() - && matches!( - config.aggregation_type, - crate::storage_engines::types::AggregationType::CountMinSketchWithHeap - | crate::storage_engines::types::AggregationType::CountSketchWithHeap - ) { + let key = if matches!(family, SummaryFamilyType::Sketch(kind, _) + if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + { KeyByLabelValues::new_with_labels(vec![series_key.to_string()]) } else { - extract_aggregated_key_from_series(series_key, config) + KeyByLabelValues::new_with_labels(Vec::new()) }; updater.update_keyed(&key, val, ts); } else { @@ -1678,71 +1668,6 @@ pub(crate) fn apply_sample( } } -/// Convert a cumulative counter sample into a non-negative, reset-aware -/// increment. Only the immediately preceding sample per series is retained; -/// pane rotation therefore cannot lose the boundary increment. -/// Whether a sample must be converted to a counter delta before it reaches the -/// accumulator. -/// -/// Only the legacy configured update rule does this. A selected Planner program -/// never does: rate is an explicit upstream operator in the DAG, so the sample -/// reaches the accumulator unchanged. Both call sites used to ask the program -/// and were always told `false`. -fn legacy_counter_delta(state: &GroupState) -> bool { - state.program.is_none() - && matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } - ) -} - -pub(crate) fn reset_aware_counter_delta( - previous: &mut HashMap, - series_key: &str, - value: f64, - timestamp_ms: i64, -) -> Option { - if !value.is_finite() { - return None; - } - match previous.get(series_key).copied() { - Some((previous_ts, _)) if timestamp_ms <= previous_ts => None, - Some((_, previous_value)) => { - previous.insert(series_key.to_owned(), (timestamp_ms, value)); - Some(if value >= previous_value { - value - previous_value - } else { - value.max(0.0) - }) - } - None => { - previous.insert(series_key.to_owned(), (timestamp_ms, value)); - None - } - } -} - -/// Extract aggregated label values from a series key string. -/// These are the labels that form the key dimension *inside* keyed accumulators -/// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. -fn extract_aggregated_key_from_series( - series_key: &str, - config: &PrecomputeMaterialization, -) -> KeyByLabelValues { - let labels = parse_labels_from_series_key(series_key); - let mut values = Vec::new(); - - for label_name in &config.aggregated_labels.labels { - if let Some(val) = labels.get(label_name.as_str()) { - values.push(val.to_string()); - } else { - values.push(String::new()); - } - } - - KeyByLabelValues::new_with_labels(values) -} - #[cfg(test)] mod tests { use super::*; @@ -1795,30 +1720,6 @@ mod tests { crate::precompute_engine::group_key::intern_pairs([("group", value)]) } - #[test] - fn counter_delta_is_reset_aware_series_local_and_cross_pane_safe() { - let mut previous = HashMap::new(); - assert_eq!(reset_aware_counter_delta(&mut previous, "a", 10.0, 1), None); - assert_eq!(reset_aware_counter_delta(&mut previous, "b", 40.0, 1), None); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 15.0, 2), - Some(5.0) - ); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 3.0, 3), - Some(3.0) - ); - assert_eq!( - reset_aware_counter_delta(&mut previous, "b", 44.0, 4), - Some(4.0) - ); - assert_eq!(reset_aware_counter_delta(&mut previous, "a", 99.0, 2), None); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 5.0, 5), - Some(2.0) - ); - } - use flate2::{write::GzEncoder, Compression}; use serde_json::json; use std::io::Write; @@ -1905,64 +1806,52 @@ mod tests { use asap_types::sds::StoredOutputId; use asap_types::AggregationType; - fn make_agg_config( - id: u64, - metric: &str, - agg_type: AggregationType, - agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, - grouping: Vec<&str>, - ) -> PrecomputeMaterialization { - make_agg_config_full( - id, - metric, - agg_type, - agg_sub_type, - window_secs, - slide_secs, - grouping, - vec![], - ) + thread_local! { + /// Stored state family of each fixture output, keyed by its allocated id. + static FIXTURE_FAMILIES: std::cell::RefCell< + HashMap, + > = Default::default(); } - #[allow(clippy::too_many_arguments)] - fn make_agg_config_full( + /// Give `config` the stored state `family`, allocating its id from it. + fn set_fixture_family( + config: &mut PrecomputeMaterialization, + family: planner_types::post_asap::SummaryFamilyType, + ) { + config.allocate_stored_output_id(&family); + FIXTURE_FAMILIES.with(|families| { + families + .borrow_mut() + .insert(config.stored_output_id, family) + }); + } + + fn make_agg_config( _id: u64, metric: &str, agg_type: AggregationType, - agg_sub_type: &str, + _agg_sub_type: &str, window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - aggregated: Vec<&str>, ) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Callers below build the - // streaming-config map by reading `config.policy_fp_u64()` - // from the returned value. let window_type = if slide_secs == 0 || slide_secs == window_secs { WindowKind::Tumbling } else { WindowKind::Sliding }; - PrecomputeMaterialization::new( - agg_type, - agg_sub_type.to_string(), - HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(aggregated.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), window_secs, slide_secs, window_type, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) + ); + set_fixture_family( + &mut config, + crate::tests::test_utilities::outputs::family(agg_type, &serde_json::json!({})), + ); + config } fn make_worker( @@ -2012,8 +1901,14 @@ mod tests { fn make_hot_reload( configs: HashMap, ) -> crate::storage_engines::types::InstalledPrecomputePlanHandle { + let outputs = configs.into_iter().map(|(id, mut config)| { + let family = FIXTURE_FAMILIES + .with(|families| families.borrow()[&config.stored_output_id].clone()); + config.stored_output_id = asap_types::sds::StoredOutputId(id); + (config, family) + }); crate::storage_engines::types::InstalledPrecomputePlanHandle::new( - crate::storage_engines::types::InstalledPrecomputePlan::from_raw_ids(configs), + crate::storage_engines::types::InstalledPrecomputePlan::new(outputs), ) } @@ -2029,57 +1924,6 @@ mod tests { // Test: raw mode — each sample forwarded as an exact Sum with sum==value // ----------------------------------------------------------------------- - #[test] - fn initial_counter_observation_publishes_an_empty_delta_window() { - let mut config = make_agg_config(1, "counter", AggregationType::Sum, "sum", 1, 1, vec![]); - config - .parameters - .insert("weight_mode".into(), serde_json::json!("counter_delta")); - let fingerprint = config.policy_fingerprint(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker( - HashMap::from([(fingerprint.0, config)]), - sink.clone(), - false, - 0, - LateDataPolicy::Drop, - ); - worker.current_input_revision = Some(Arc::new( - crate::storage_engines::types::SummaryInputRevision { - generation: Arc::new(asap_types::sds::CatalogGeneration { - schema_version: 1, - plan_id: 1, - plan_version: 1, - snapshot_sha256: "test".into(), - }), - first_revision: 1, - revision: 1, - }, - )); - worker - .process_group_samples( - 1, - fingerprint, - &test_group_key(""), - vec![("counter".into(), 1000, 7.0)], - ) - .unwrap(); - worker.force_close_all().unwrap(); - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - assert_eq!(captured[0].0.input_revision.as_ref().unwrap().revision, 1); - assert_eq!( - captured[0] - .1 - .as_any() - .downcast_ref::() - .map(crate::tests::accumulator_fixture::sum_view) - .unwrap() - .sum, - 0.0 - ); - } - #[test] fn test_raw_mode_forwarding() { let sink = Arc::new(CapturingOutputSink::new()); @@ -2120,15 +1964,7 @@ mod tests { #[test] fn test_tumbling_window_correctness() { // 10s tumbling window - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2203,17 +2039,9 @@ mod tests { #[test] fn test_group_by_merges_series() { - // SingleSubpopulation Sum with no grouping labels + // Sum with no grouping labels // Two different series in the same group → both feed same accumulator - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2275,15 +2103,7 @@ mod tests { #[test] fn test_different_groups_separate_outputs() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec!["pattern"], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec!["pattern"]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2364,9 +2184,13 @@ mod tests { 0, vec!["pattern"], ); - config - .parameters - .insert("K".to_string(), serde_json::Value::from(20_u64)); + set_fixture_family( + &mut config, + crate::tests::test_utilities::outputs::family( + AggregationType::DatasketchesKLL, + &serde_json::json!({"K": 20}), + ), + ); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2439,15 +2263,7 @@ mod tests { #[test] fn test_sliding_window_pane_sharing() { // 30s window, 10s slide → W=3 panes per window - let config = make_agg_config( - 2, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 30, - 10, - vec![], - ); + let config = make_agg_config(2, "cpu", AggregationType::Sum, "", 30, 10, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(2, config); @@ -2499,15 +2315,7 @@ mod tests { #[test] fn full_window_layout_materializes_each_overlapping_slide() { - let mut config = make_agg_config( - 2, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 30, - 10, - vec![], - ); + let mut config = make_agg_config(2, "cpu", AggregationType::Sum, "", 30, 10, vec![]); config.window_layout = asap_types::WindowMaterializationLayout::FullWindow; let policy = config.policy_fingerprint(); let sink = Arc::new(CapturingOutputSink::new()); @@ -2564,91 +2372,9 @@ mod tests { // All series go to one group, host is the key dimension INSIDE the sketch // ----------------------------------------------------------------------- - #[test] - fn test_keyed_accumulator_aggregated_labels() { - // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) - let config = make_agg_config_full( - 3, - "cpu", - AggregationType::MultipleSubpopulation, - "Sum", - 10, - 0, - vec![], // grouping: empty — one output group - vec!["host"], // aggregated: host is the key INSIDE the sketch - ); - let mut agg_configs = HashMap::new(); - agg_configs.insert(3, config); - - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - // Both series go to the SAME bucket (group_key="" since grouping is empty). - // The host label is extracted as the aggregated key inside the accumulator. - let pf = PolicyFingerprint(3); - worker - .process_group_samples( - 3, - pf, - &test_group_key(""), - vec![ - ("cpu{host=\"A\"}".to_string(), 1000, 10.0), - ("cpu{host=\"B\"}".to_string(), 2000, 20.0), - ], - ) - .unwrap(); - - // Close the single bucket's window - worker - .process_group_samples( - 3, - pf, - &test_group_key(""), - group_samples("cpu{host=\"A\"}", vec![(10000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!( - captured.len(), - 1, - "one group → one output (both hosts inside)" - ); - - let (_output, acc) = &captured[0]; - let ms_acc = acc - .as_any() - .downcast_ref::() - .expect("should be a keyed exact Sum"); - assert!(ms_acc.is_keyed(), "both host keys inside one accumulator"); - let sum = |host: &str| { - ms_acc - .readout( - asap_types::Statistic::Sum, - None, - Some(&KeyByLabelValues::new_with_labels(vec![host.into()])), - ) - .unwrap() - .unwrap() - }; - let found_a = (sum("A") - 10.0).abs() < 1e-10; - let found_b = (sum("B") - 20.0).abs() < 1e-10; - assert!(found_a, "expected key A inside accumulator"); - assert!(found_b, "expected key B inside accumulator"); - } - #[test] fn test_late_data_drop() { - let config = make_agg_config( - 4, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(4, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(4, config); @@ -2704,15 +2430,7 @@ mod tests { #[test] fn test_late_data_forward_to_store() { - let config = make_agg_config( - 5, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(5, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(5, config); @@ -2804,21 +2522,11 @@ mod tests { #[test] fn test_extract_key_from_series() { let config = PrecomputeMaterialization::new( - AggregationType::SingleSubpopulation, - "Sum".to_string(), - HashMap::new(), + "http_requests_total", asap_types::KeyByLabelNames::new(vec!["method".to_string(), "status".to_string()]), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), 60, 0, WindowKind::Tumbling, - "http_requests_total".to_string(), - "http_requests_total".to_string(), - Some(60), - None, - None, ); let key = extract_key_from_series( @@ -2881,15 +2589,7 @@ mod tests { // Two groups on the same worker. Group A advances to t=100s while // group B remains active at t=5s. Group A is not evidence that group // B's source has completed its earlier window. - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let agg_configs = HashMap::from([(1, config)]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); @@ -2980,15 +2680,7 @@ mod tests { #[test] fn repeated_flushes_do_not_make_fixed_timestamp_input_late() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3033,15 +2725,7 @@ mod tests { #[test] fn allowed_lateness_delays_event_time_window_close() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3091,15 +2775,7 @@ mod tests { #[test] fn first_catch_up_batch_closes_every_complete_window() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 5, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 5, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3138,15 +2814,7 @@ mod tests { #[test] fn test_flush_publishes_worker_watermark() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let agg_configs = HashMap::from([(1, config)]); let sink = Arc::new(CapturingOutputSink::new()); let wm = Arc::new(AtomicI64::new(i64::MIN)); @@ -3590,15 +3258,7 @@ mod tests { #[test] fn wall_clock_fallback_does_not_close_active_raw_ingest() { - let cfg = make_agg_config( - 7, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(7, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(7, cfg)]), @@ -3659,15 +3319,7 @@ mod tests { #[test] fn absolute_wall_clock_deadline_closes_active_raw_ingest() { - let cfg = make_agg_config( - 9, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(9, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(9, cfg)]), @@ -3734,15 +3386,7 @@ mod tests { #[test] fn absolute_deadline_is_disabled_for_drop_policy() { - let cfg = make_agg_config( - 11, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(11, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(11, cfg)]), @@ -3969,15 +3613,7 @@ mod tests { // A pooled Sum is correct only for an explicit cross-entity reduction. #[test] fn pooled_sum_does_not_preserve_per_entity_output_rows() { - let config = make_agg_config( - 1, - "gauge", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "gauge", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker( HashMap::from([(1, config)]), @@ -4026,8 +3662,8 @@ mod tests { let config = make_agg_config( 1, "requests_total", - AggregationType::SingleSubpopulation, - "Increase", + AggregationType::Increase, + "", 10, 0, vec![], @@ -4072,15 +3708,7 @@ mod tests { // Acknowledgement proves FIFO input processing and trailing-window publication. #[tokio::test] async fn finite_input_drain_publishes_before_acknowledging() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker( HashMap::from([(1, config)]), @@ -4137,15 +3765,7 @@ mod tests { Err("deliberate sink failure".into()) } } - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut worker = make_worker( HashMap::from([(1, config)]), Arc::new(CapturingOutputSink::new()), @@ -4207,15 +3827,7 @@ mod tests { Err(Box::new(asap_physical_operators::dag::Error::MemoryLimit)) } } - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut worker = make_worker( HashMap::from([(1, config)]), Arc::new(CapturingOutputSink::new()), @@ -4252,15 +3864,7 @@ mod tests { fn shutdown_force_close_emits_trailing_sample_window() { // 10s tumbling window; make_worker uses grace=0, isolating the // force-close from the wall-clock fallback. - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); let sink = Arc::new(CapturingOutputSink::new()); @@ -4547,6 +4151,12 @@ mod dag_execution_tests { .expect("ASAP producer required") .clone(); let fp = config.policy_fingerprint(); + let agg_type = asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + ) + .unwrap(); let streaming = InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan).unwrap(); let sink = Arc::new(CapturingOutputSink::new()); @@ -4615,7 +4225,7 @@ mod dag_execution_tests { let group = crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { entries: vec![std::rc::Rc::new(states)], - agg_type: config.aggregation_type, + agg_type, }; assert_eq!( group.exact_value_for(readout, &None, 0, 5000), @@ -4715,11 +4325,15 @@ mod dag_execution_tests { == Some(asap_types::sds::PopulationPartitioning::PerEntity) || (config.partitioning.is_none() && matches!( - config.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::Rate - | asap_types::AggregationType::Min - | asap_types::AggregationType::Max + installed + .state_family(config.stored_output_id) + .and_then(asap_types::aggregation_type_for_family), + Some( + asap_types::AggregationType::Increase + | asap_types::AggregationType::Rate + | asap_types::AggregationType::Min + | asap_types::AggregationType::Max + ) )); // Routing identity as Remote Write assigns it. let mut groups = BTreeMap::, Vec<(String, i64, f64)>>::new(); @@ -4776,11 +4390,7 @@ mod dag_execution_tests { config.pane_origin_ms, &config.window_layout, ); - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = installed.right_closed_panes(config.stored_output_id); let mut expected = BTreeMap::new(); let mut backfill = BTreeMap::new(); for (key, samples) in &groups { @@ -4862,7 +4472,8 @@ mod dag_execution_tests { "{}", config.metric ); - assert_eq!(actual, expected, "{:?}", config.aggregation_type); + let family = installed.state_family(config.stored_output_id).unwrap(); + assert_eq!(actual, expected, "{family:?}"); // Backfill builds each window with the same installed graph. let backfilled = backfill .into_iter() @@ -4877,14 +4488,10 @@ mod dag_execution_tests { ((key, start, end), state.serialize_to_bytes()) }) .collect::>(); - assert_eq!( - backfilled, expected, - "backfill {:?}", - config.aggregation_type - ); + assert_eq!(backfilled, expected, "backfill {family:?}"); outputs += 1; families.insert( - format!("{:?}", config.accumulator_spec().unwrap().family) + format!("{family:?}") .split(['(', ' ', ',']) .find(|part| { [ @@ -4989,11 +4596,8 @@ mod dag_execution_tests { config.pane_origin_ms, &config.window_layout, ); - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + // Raw PromQL programs close panes on the right, like `(start, end]` ranges. + let right_closed = true; let mut panes = BTreeMap::<(i64, i64), Vec<(&str, i64, f64)>>::new(); for (time, value) in [(1100, 10.0), (1200, 20.0), (1300, 30.0), (1400, 40.0)] { let pane_time = if right_closed { time - 1 } else { time }; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index ee644c84a..63f5853f0 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -811,25 +811,22 @@ mod tests { "SELECT sum(value) FROM requests".into() }; let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + "requests", KeyByLabelNames::empty(), - String::new(), 1, 1, WindowKind::Tumbling, - String::new(), - "requests".into(), - None, - Some("asap_e2e.samples".into()), - Some("value".into()), ); + config.table_name = Some("asap_e2e.samples".into()); + config.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); config.pane_origin_ms = Some(0); config.table_timestamp_column = Some("timestamp_ms".into()); - let sds = SummaryCatalog::from_materializations(41, 1, &[config.clone()]).unwrap(); + let family = AggregationType::Sum.planner_exact_family().unwrap(); + config.allocate_stored_output_id(&family); + let sds = + SummaryCatalog::from_outputs(41, 1, vec![(&config, &family, String::new())]).unwrap(); let materialization = *sds.outputs.keys().next().unwrap(); let read = QueryNodeId(0); let readout = QueryNodeId(1); @@ -1022,7 +1019,7 @@ mod tests { }; let mut precompute = asap_types::precompute_plan::PrecomputePlan::build( envelope.clone(), - vec![config], + vec![(config, family)], &["fixture".into()], ) .unwrap(); @@ -1271,21 +1268,11 @@ mod tests { assert!(request.send().await.unwrap().status().is_success()); } let mut cfg = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::empty(), + "requests", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 1, 1, WindowKind::Tumbling, - String::new(), - "requests".into(), - None, - None, - None, ); cfg.pane_origin_ms = Some(0); cfg.table_name = Some("asap_e2e.samples".into()); @@ -1293,10 +1280,12 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); + let (cfg, family) = crate::tests::test_utilities::outputs::allocated( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + ); let hot = crate::storage_engines::types::InstalledPrecomputePlanHandle::from_arc(Arc::new( - crate::storage_engines::types::InstalledPrecomputePlan::from_raw_ids(HashMap::from([ - (cfg.policy_fp_u64(), cfg.clone()), - ])), + crate::storage_engines::types::InstalledPrecomputePlan::new([(cfg.clone(), family)]), )); let registry = Arc::new(crate::storage_engines::sketch_db::backfill::BackfillRegistry::new()); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index 1ae3909a4..338a10651 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -629,7 +629,7 @@ mod tests { let result = execute_sql_dag_with_external( &SketchStore::new(), &entry, - &SummaryCatalog::from_materializations(1, 1, &[]).unwrap(), + &SummaryCatalog::from_outputs(1, 1, Vec::new()).unwrap(), &[(QueryNodeId(0), relation)].into(), 0, 1000, @@ -821,7 +821,7 @@ mod tests { (QueryNodeId(2), relation), ]); let index = SketchStore::new(); - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); + let catalog = SummaryCatalog::from_outputs(1, 1, Vec::new()).unwrap(); let run = || execute_sql_dag_with_external(&index, &entry, &catalog, &prepared, 0, 1000, false); assert!(matches!(run(), ClickHouseDagOutcome::Accelerated(_))); diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index c227e5eb0..212e2da3e 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -179,24 +179,16 @@ mod tests { } fn catalog_fixture() -> SummaryCatalog { - let config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + let family = AggregationType::Sum.planner_exact_family().unwrap(); + let mut config = PrecomputeMaterialization::new( + "m", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, ); - SummaryCatalog::from_materializations(1, 1, &[config]).unwrap() + config.allocate_stored_output_id(&family); + SummaryCatalog::from_outputs(1, 1, vec![(&config, &family, String::new())]).unwrap() } // A readout cannot relabel a valid sum materialization as a rate capability. diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 7c079838b..3b6f748e3 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1548,26 +1548,29 @@ mod tests { #[test] fn metricsql_quantile_preserves_catalog_metric_name_only_per_entity() { use control_plane::query_plan::*; - let config: asap_types::PrecomputeMaterialization = + let mut config: asap_types::PrecomputeMaterialization = serde_json::from_value(serde_json::json!({ - "aggregation_type": "DDSketch", "aggregation_sub_type": "", + "stored_output_id": 0, "metric": "latency_ms", "window_size": 1, "slide_interval": 1, "window_type": "tumbling", "num_aggregates_to_retain": 3, - "parameters": {"alpha": 0.01}, "pane_origin_ms": 0, + "pane_origin_ms": 0, "partitioning": "per_entity", "window_layout": {"kind": "pane", "pane_secs": 1}, - "grouping_labels": {"labels": []}, "aggregated_labels": {"labels": []}, - "rollup_labels": {"labels": []}, "spatial_filter": "", - "spatial_filter_normalized": "", "original_yaml": "" + "grouping_labels": {"labels": []}, "table_name": null })) .unwrap(); + let family = crate::tests::test_utilities::outputs::family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"alpha": 0.01}), + ); + config.allocate_stored_output_id(&family); let idx = ddsketch_fixture(); let mut metadata = (*idx.instance(1).unwrap()).clone(); metadata.policy_fp = config.policy_fingerprint(); idx.install_summary_catalog(std::sync::Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( + asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[config.clone()], + vec![(&config, &family, String::new())], ) .unwrap(), )) @@ -1848,8 +1851,11 @@ mod tests { metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config( - config, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + &plan.precompute_plan.population_filter(config).unwrap(), ), accuracy: None, first_seen_unix_ms: 0, @@ -1917,7 +1923,12 @@ mod tests { metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config(config), + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + &plan.precompute_plan.population_filter(config).unwrap(), + ), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index fa3ccfcc4..026a85afe 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -13,6 +13,24 @@ use asap_types::PrecomputeMaterialization; use control_plane::physical::compiler::PLANNER_REVISION; use std::{collections::BTreeMap, sync::Arc}; +thread_local! { + /// Stored state family of each fixture output, keyed by its allocated id. + static FAMILIES: std::cell::RefCell< + std::collections::HashMap< + asap_types::sds::StoredOutputId, + planner_types::post_asap::SummaryFamilyType, + >, + > = Default::default(); +} + +/// The stored state family a fixture output was allocated with. +pub(super) fn family_of( + config: &PrecomputeMaterialization, +) -> planner_types::post_asap::SummaryFamilyType { + FAMILIES.with(|families| families.borrow()[&config.stored_output_id].clone()) +} + +/// A per-entity output of the `aggregation` kernel with these dimensions. pub(super) fn materialization( metric: &str, aggregation: &str, @@ -21,18 +39,26 @@ pub(super) fn materialization( pane_ms: u64, ) -> PrecomputeMaterialization { assert_eq!(pane_ms % 1000, 0); - serde_json::from_value(serde_json::json!({ - "aggregation_type": aggregation, "aggregation_sub_type": "", + let mut config: PrecomputeMaterialization = serde_json::from_value(serde_json::json!({ + "stored_output_id": 0, "metric": metric, "window_size": pane_ms / 1000, "slide_interval": pane_ms / 1000, "window_type": "tumbling", "num_aggregates_to_retain": 100, - "parameters": parameters, "pane_origin_ms": 0, + "pane_origin_ms": 0, "partitioning": "per_entity", "window_layout": {"kind": "pane", "pane_secs": pane_ms / 1000}, - "grouping_labels": {"labels": labels}, "aggregated_labels": {"labels": []}, - "rollup_labels": {"labels": []}, "spatial_filter": "", - "spatial_filter_normalized": "", "original_yaml": "" + "grouping_labels": {"labels": labels}, + "table_name": null })) - .unwrap() + .unwrap(); + let family = + crate::tests::test_utilities::outputs::family(aggregation.parse().unwrap(), ¶meters); + config.allocate_stored_output_id(&family); + FAMILIES.with(|families| { + families + .borrow_mut() + .insert(config.stored_output_id, family) + }); + config } pub(super) fn entry( @@ -62,16 +88,16 @@ pub(super) fn entry( .then_some(config.slide_interval * 1000), materialization: config.policy_fingerprint().into(), stored_output_reference: - asap_types::summary_catalog::SummaryCatalog::from_materializations( + asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[config.clone()], + vec![(config, &family_of(config), String::new())], ) .unwrap() .output_reference(config.policy_fingerprint().into()) .unwrap(), output_grouping: grouping, - item_labels: config.aggregated_labels.labels.clone(), + item_labels: Vec::new(), window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(lookback_ms), @@ -104,12 +130,13 @@ pub(super) fn install( planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "runtime-fixture".into(), }; - let materializations: Vec<_> = configs.iter().map(|(c, _)| c.clone()).collect(); - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &materializations) - .unwrap(); + let outputs: Vec<_> = configs + .iter() + .map(|(c, _)| (c.clone(), family_of(c))) + .collect(); let mut precompute = - PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); + PrecomputePlan::build(envelope.clone(), outputs, &["fixture".into()]).unwrap(); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index 084c87cc6..c432139c5 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -3,19 +3,17 @@ //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. -//! Given the `aggregation_type` + `parameters` pinned on an -//! `AggSchema`, the registry can expose the theoretical accuracy -//! bound of every query answer computed from it — so users and -//! control planes see "this quantile is within ε relative error with -//! probability 1 - δ" as a first-class part of the schema, not a -//! number they have to rederive from the sketch literature. +//! Given the Planner state family of a stored output, this exposes the +//! theoretical accuracy bound of every query answer computed from it — so +//! users and control planes see "this quantile is within ε relative error +//! with probability 1 - δ" as a first-class part of the schema, not a number +//! they have to rederive from the sketch literature. //! //! ## Scope of this module //! -//! Pure derivation: `derive(&PrecomputeMaterialization)` -//! looks at `aggregation_type` and the relevant entries in -//! `config.parameters` and returns an `AccuracyProfile`. No -//! runtime measurement, no sampling — just the textbook bound. +//! Pure derivation: [`derive`] maps a `SummaryFamilyType` to an +//! `AccuracyProfile`. No runtime measurement, no sampling — just the +//! textbook bound. //! //! These bounds are asymptotic / probabilistic worst-case //! guarantees from the original sketch papers. Real error @@ -27,83 +25,29 @@ //! CMS, HLL, KLL, and DDSketch profiles come from ASAPPlanner's //! `DefaultAccuracyModel`. HLL reports relative standard error with unknown //! failure probability (`delta: null`), rather than a deterministic guarantee. -//! CountSketch, heap-retention extensions, and GOS augmentation remain local. +//! CountSketch and heap-retention extensions remain local. use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::PrecomputeMaterialization; -use asap_types::AggregationType; - pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; -use planner_types::post_asap::SketchParams as PlannerParams; - -/// Derive an [`AccuracyProfile`] from a pinned -/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any -/// necessary entries in `parameters`; falls back to exact for -/// unknown / legacy variants (harmless — the caller just gets -/// "0 error" rather than a panic). -/// -/// GOS continuous-query envelope (design-gos-unified-edge-telemetry.md §4, -/// Theorem 1): when the edge gates delta transmission by the GOS relative -/// threshold (`parameters["gos_delta_epsilon"] = ε_st > 0`), the warm -/// sketch answered from delta-applied state carries an extra DETERMINISTIC -/// staleness term of at most `ε_st` (relative) at any query time — it adds -/// linearly to the sketch's own probabilistic bound (`ε_total = ε_sk + -/// ε_st`; the random parts compose in quadrature but the staleness part is -/// adversarial, so linear addition is the honest envelope). δ is -/// unchanged (staleness is not probabilistic). -pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { - let mut profile = derive_sketch_only(config); - let eps_st = config - .parameters - .get("gos_delta_epsilon") - .and_then(|v| v.as_f64()) - .unwrap_or(0.0); - if eps_st > 0.0 && profile.kind != AccuracyKind::Exact { - profile.epsilon += eps_st; - } - profile -} - -/// Source adapter for installed aggregation configs. -pub trait BackendAccuracyProfile { - fn derive(config: &PrecomputeMaterialization) -> Self; - fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; -} -impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &PrecomputeMaterialization) -> Self { - derive(config) - } - fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { - derive_sketch_only(config) - } -} - -/// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { - match config.aggregation_type { - AggregationType::UnivMon => AccuracyProfile { +use planner_types::post_asap::{SketchParams as PlannerParams, SummaryFamilyType}; + +/// Derive the [`AccuracyProfile`] of state with the Planner `family`. +/// Exact families have zero error. +pub fn derive(family: &SummaryFamilyType) -> AccuracyProfile { + let SummaryFamilyType::Sketch(kind, _) = family else { + return AccuracyProfile::exact(); + }; + match kind.params() { + PlannerParams::UnivMon { .. } => AccuracyProfile { epsilon: f64::MAX, delta: Some(1.0), kind: AccuracyKind::Uncalibrated, }, - // Exact aggregates. (The `SetAggregator` / - // `DeltaSetAggregator` exact-set-membership family lived - // here too before its retirement.) - AggregationType::Sum - | AggregationType::Count - | AggregationType::Increase - | AggregationType::Rate - | AggregationType::Min - | AggregationType::Max => AccuracyProfile::exact(), - - AggregationType::CountMinSketch => { - let (rows, cols) = cms_params(config); - shared_profile(PlannerParams::Cms { - depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), - width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), - }) - } + params @ (PlannerParams::Cms { .. } + | PlannerParams::Hll { .. } + | PlannerParams::Kll { .. } + | PlannerParams::DDSketch { .. }) => shared_profile(params.clone()), // CountMinSketchWithHeap: CMS frequency estimator // coupled with a heap of the top-`k` heaviest items @@ -115,36 +59,26 @@ fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { // ≥ N/heap_size is guaranteed to be in the top-K // output; each retained count is within N/heap_size // of the true value (SpaceSaving guarantee). - // We report the **tighter** of the two as the - // user-facing ε — typically the heap bound - // `1/heap_size` dominates when heap_size ≪ w, and the - // CMS bound `e/w` dominates when the heap is generously - // sized. `kind = TopK` signals that ε is the - // combined frequency + retention guarantee. - // δ stays `exp(-d)` from the CMS half; retention itself - // is deterministic given an adversarial-free stream, - // but the count estimate remains probabilistic at - // depth d. + // We report the worse of the two as the user-facing ε. + // `kind = TopK` signals that ε is the combined frequency + + // retention guarantee. δ stays `exp(-d)` from the CMS half. // Sources: // - Cormode & Muthukrishnan 2005 (CMS bound) // - Metwally, Agrawal, El Abbadi. "Efficient // computation of frequent and top-k elements in // data streams." ICDT 2005. (top-K retention) - AggregationType::CountMinSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); + PlannerParams::CmsWithHeap { + width, + depth, + heap_size, + } => { let cms = shared_profile(PlannerParams::Cms { - depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), - width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), + depth: (*depth).max(1), + width: (*width).max(1), }); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - // Worst of the two — a user should expect errors - // no bigger than `ε · N`. - let epsilon = cms.epsilon.max(heap_epsilon); - let delta = cms.delta; AccuracyProfile { - epsilon, - delta, + epsilon: cms.epsilon.max(1.0 / f64::from(*heap_size).max(1.0)), + delta: cms.delta, kind: AccuracyKind::TopK, } } @@ -152,58 +86,29 @@ fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { // CountSketch: ε = 1/√w, δ = 1/2^d (Charikar-Chen- // Farach-Colton). Signed counters → tighter epsilon // than CMS but same confidence ramp with depth. - AggregationType::CountSketch => { - let (rows, cols) = cms_params(config); - let epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let delta = Some(0.5_f64.powi(rows as i32)); - AccuracyProfile { - epsilon, - delta, - kind: AccuracyKind::AdditiveFrequency, - } - } - - // CountSketchWithHeap: CountSketch frequency estimator - // paired with a top-k heap. Mirrors the - // CountMinSketchWithHeap branch above — the CountSketch - // half gives ε_point = 1/√w; the heap half gives - // ε_heap = 1/heap_size for retention. Report the - // tighter (max) of the two. - AggregationType::CountSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); - let cs_epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - let epsilon = cs_epsilon.max(heap_epsilon); - let delta = Some(0.5_f64.powi(rows as i32)); - AccuracyProfile { - epsilon, - delta, - kind: AccuracyKind::TopK, - } - } - - AggregationType::HLL => shared_profile(PlannerParams::Hll { - precision: u8::try_from(hll_precision(config)).unwrap_or(14), - }), - AggregationType::DatasketchesKLL | AggregationType::HydraKLL => { - shared_profile(PlannerParams::Kll { - k: kll_k(config).max(1), - }) - } - AggregationType::DDSketch => shared_profile(PlannerParams::DDSketch { - alpha: ddsketch_alpha(config), - }), + PlannerParams::CountSketch { width, depth } => AccuracyProfile { + epsilon: 1.0 / f64::from(*width).max(1.0).sqrt(), + delta: Some(0.5_f64.powi(*depth as i32)), + kind: AccuracyKind::AdditiveFrequency, + }, - // Legacy / wrapper variants. Return exact — they are - // config-shape placeholders that dispatch to concrete - // aggregator types elsewhere; their accuracy profile - // depends on the sub_type, which the factory resolves - // at updater-construction time. Phase 6.4 v2 can walk - // sub_type to give a tighter answer. - AggregationType::SingleSubpopulation | AggregationType::MultipleSubpopulation => { - AccuracyProfile::exact() - } + // CountSketchWithHeap: the CountSketch half gives ε_point = 1/√w; + // the heap half gives ε_heap = 1/heap_size for retention. + PlannerParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => AccuracyProfile { + epsilon: (1.0 / f64::from(*width).max(1.0).sqrt()) + .max(1.0 / f64::from(*heap_size).max(1.0)), + delta: Some(0.5_f64.powi(*depth as i32)), + kind: AccuracyKind::TopK, + }, + _ => AccuracyProfile { + epsilon: f64::MAX, + delta: Some(1.0), + kind: AccuracyKind::Uncalibrated, + }, } } @@ -211,69 +116,6 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { AccuracyProfile::from_sketch_params(¶ms).expect("shared family has a numeric planner bound") } -// Parameter extraction helpers. Kept file-local (not pub) because -// they duplicate tiny bits of `precompute_engine::accumulator_factory` -// and the backfill-vs-live separation rule (see that module's doc) -// says it's OK for them to drift — this module is the single -// authority on *accuracy*, not on *construction*. - -/// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { - let rows = config - .parameters - .get("d") - .and_then(|v| v.as_u64()) - .unwrap_or(4); - let cols = config - .parameters - .get("w") - .and_then(|v| v.as_u64()) - .unwrap_or(1000); - (rows, cols) -} - -fn hll_precision(config: &PrecomputeMaterialization) -> u32 { - config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(|v| v.as_u64()) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(14) -} - -fn kll_k(config: &PrecomputeMaterialization) -> u32 { - config - .parameters - .get("K") - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(200) -} - -fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { - config - .parameters - .get("alpha") - .and_then(|v| v.as_f64()) - .unwrap_or(0.01) -} - -/// CMS-with-heap heap size (the `k` in "top-k retention"). Read -/// from `parameters["heap_size"]` with a default of 100 — -/// matches the default the control plane's planner uses when the -/// caller didn't override. -fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { - config - .parameters - .get("heap_size") - .or_else(|| config.parameters.get("topk")) - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .unwrap_or(100) -} - /// Per-segment accuracy record. Attached to a multi-segment /// [`AccuracyEnvelope`] so clients can see the error bound for /// each piece of the schema-timeline-crossing query. @@ -366,104 +208,60 @@ impl AccuracyEnvelope { #[cfg(test)] mod tests { use super::*; - use asap_types::enums::WindowKind; - use asap_types::KeyByLabelNames; - use serde_json::{json, Value}; - use std::collections::HashMap; - - fn base_config( - agg_type: AggregationType, - params: HashMap, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - params, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, + use planner_types::post_asap::{ + ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, + }; + + fn sketch(algorithm: SketchAlgorithm, params: PlannerParams) -> SummaryFamilyType { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, ) } - /// Both source adapters must agree for the same normalized family parameters. + fn cms(depth: u32, width: u32) -> SummaryFamilyType { + sketch(SketchAlgorithm::Cms, PlannerParams::Cms { width, depth }) + } + + fn hll(precision: u8) -> SummaryFamilyType { + sketch(SketchAlgorithm::Hll, PlannerParams::Hll { precision }) + } + + /// Planner and backend derivations agree for the same shared family parameters. #[test] fn planner_and_backend_shared_family_parity() { use control_plane::types::SketchParams; let cases = vec![ - ( - SketchParams::CountMinSketch { - rows: 4, - cols: 1000, - metric_name: "m".into(), - }, - AggregationType::CountMinSketch, - json!({"d": 4, "w": 1000}), - ), ( SketchParams::CountMinSketch { rows: 7, cols: 4096, metric_name: "m".into(), }, - AggregationType::CountMinSketch, - json!({"d": 7, "w": 4096}), - ), - ( - SketchParams::HLL { precision: 14 }, - AggregationType::HLL, - json!({"precision": 14}), - ), - ( - SketchParams::HLL { precision: 10 }, - AggregationType::HLL, - json!({"p": 10}), - ), - ( - SketchParams::KLL { - k: 200, - quantiles: vec![], - }, - AggregationType::DatasketchesKLL, - json!({"K": 200}), + cms(7, 4096), ), + (SketchParams::HLL { precision: 10 }, hll(10)), ( SketchParams::KLL { k: 512, quantiles: vec![], }, - AggregationType::HydraKLL, - json!({"k": 512}), - ), - ( - SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![], - }, - AggregationType::DDSketch, - json!({"alpha": 0.01}), + sketch(SketchAlgorithm::Kll, PlannerParams::Kll { k: 512 }), ), ( SketchParams::DDSketch { relative_accuracy: 0.05, quantiles: vec![], }, - AggregationType::DDSketch, - json!({"alpha": 0.05}), + sketch( + SketchAlgorithm::DDSketch, + PlannerParams::DDSketch { alpha: 0.05 }, + ), ), ]; - for (params, family, wire) in cases { - let config = base_config(family, serde_json::from_value(wire).unwrap()); + for (params, family) in cases { let planner = control_plane::accuracy::derive(¶ms); - let backend = derive(&config); + let backend = derive(&family); assert_eq!(planner, backend, "parameters: {params:?}"); assert_eq!( serde_json::to_value(planner).unwrap(), @@ -472,30 +270,10 @@ mod tests { } } - /// GOS widens epsilon without inventing confidence for HLL. - #[test] - fn shared_family_gos_preserves_planner_confidence() { - for family in [ - AggregationType::CountMinSketch, - AggregationType::HLL, - AggregationType::DatasketchesKLL, - AggregationType::DDSketch, - ] { - let base = derive(&base_config(family, HashMap::new())); - let widened = derive(&base_config( - family, - HashMap::from([("gos_delta_epsilon".into(), json!(0.05))]), - )); - assert_eq!(widened.epsilon, base.epsilon + 0.05); - assert_eq!(widened.delta, base.delta); - assert_eq!(widened.kind, base.kind); - } - } - /// Unknown confidence stays unknown regardless of segment order. #[test] fn envelope_preserves_unknown_failure_probability() { - let hll = derive(&base_config(AggregationType::HLL, HashMap::new())); + let hll = derive(&hll(14)); assert_eq!(hll.delta, None); for profiles in [ [hll, AccuracyProfile::exact()], @@ -519,230 +297,78 @@ mod tests { /// Backend-only UnivMon must not acquire a calibrated shared-family bound. #[test] fn univmon_remains_uncalibrated() { - let profile = derive(&base_config(AggregationType::UnivMon, HashMap::new())); - assert_eq!(profile.kind, AccuracyKind::Uncalibrated); - assert_eq!(profile.epsilon, f64::MAX); - assert_eq!(profile.delta, Some(1.0)); - } - - #[test] - fn sum_is_exact() { - let p = derive(&base_config(AggregationType::Sum, HashMap::new())); - assert_eq!(p.kind, AccuracyKind::Exact); - assert_eq!(p.epsilon, 0.0); - assert_eq!(p.delta, Some(0.0)); + let p = derive(&sketch( + SketchAlgorithm::UnivMon, + PlannerParams::UnivMon { + heap_size: 32, + sketch_rows: 5, + sketch_cols: 1024, + layers: 4, + }, + )); + assert_eq!(p.kind, AccuracyKind::Uncalibrated); } + /// Every exact family has zero error. #[test] - fn min_max_increase_are_exact() { - for t in [ - AggregationType::Min, - AggregationType::Max, - AggregationType::Increase, + fn exact_families_are_exact() { + for (kind, params) in [ + (ExactKind::Sum, ExactParams::Sum), + (ExactKind::Min, ExactParams::Min), + (ExactKind::Max, ExactParams::Max), + (ExactKind::Increase, ExactParams::Increase), ] { - let p = derive(&base_config(t, HashMap::new())); + let p = derive(&SummaryFamilyType::ExactAggregate(kind, params)); + assert_eq!(p.epsilon, 0.0); assert_eq!(p.kind, AccuracyKind::Exact); } } - #[test] - fn gos_staleness_widens_epsilon_linearly() { - // A CountSketch (ε_sk = 1/√w) whose edge gates deltas at ε_st carries - // ε_total = ε_sk + ε_st in the continuous-query envelope (Theorem 1); - // δ is unchanged (staleness is deterministic). - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(256)); - let base = derive(&base_config(AggregationType::CountSketch, params.clone())); - params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let widened = derive(&base_config(AggregationType::CountSketch, params)); - assert!((widened.epsilon - (base.epsilon + 0.05)).abs() < 1e-12); - assert_eq!(widened.delta, base.delta); - assert_eq!(widened.kind, base.kind); - } - - #[test] - fn gos_staleness_does_not_touch_exact() { - // Exact aggregates are not GOS-gated (Count-Sketch families only), so a - // stray parameter must not fabricate an ε>0 "exact" answer. - let mut params = HashMap::new(); - params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let p = derive(&base_config(AggregationType::Sum, params)); - assert_eq!(p.epsilon, 0.0); - assert_eq!(p.kind, AccuracyKind::Exact); - } - + /// CMS reports ε = e/w and δ = e^-d. #[test] fn cms_epsilon_is_e_over_w() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(2718)); - let p = derive(&base_config(AggregationType::CountMinSketch, params)); - // e / 2718 ≈ 0.0010001 — very close to 0.001. + let p = derive(&cms(5, 2718)); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - std::f64::consts::E / 2718.0).abs() < 1e-12); - // δ = 1/2^5 = 0.03125 assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); } + /// A heap reports the worse of the CMS point bound and the retention bound. #[test] - fn cms_uses_defaults_when_params_absent() { - let p = derive(&base_config( - AggregationType::CountMinSketch, - HashMap::new(), - )); - // Defaults rows=4, cols=1000 per accumulator_factory. - assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta.unwrap() - (-4.0_f64).exp()).abs() < 1e-12); // 1/16 - } - - #[test] - fn cms_with_heap_carries_top_k_kind_and_heap_bound() { - // Large heap: 1/heap_size (= 1e-4) dominates the e/w CMS - // bound (e/1e6 ≈ 2.72e-6). Expect ε = 1/heap. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(1_000_000)); - params.insert("heap_size".to_string(), json!(10_000)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - 1.0 / 10_000.0).abs() < 1e-12); - assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); // 1/2^5 - } - - #[test] - fn cms_with_heap_cms_bound_dominates_when_heap_is_generous() { - // Generously-sized heap (1e6) + narrow CMS (w=100) → - // 1/heap (1e-6) ≪ e/w (2.7e-2), so the CMS bound dominates. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(100)); - params.insert("heap_size".to_string(), json!(1_000_000)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - std::f64::consts::E / 100.0).abs() < 1e-12); - } - - #[test] - fn cms_with_heap_uses_default_heap_size_100() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(1000)); - // heap_size absent → default 100 → 1/100 = 0.01 dominates - // e/1000 ≈ 0.00272. - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - 0.01).abs() < 1e-12); - } - - #[test] - fn cms_with_heap_accepts_alternative_param_names() { - // Config may name the heap "k" or "topk" instead of - // "heap_size" — all three should work. - for alias in ["heap_size", "topk", "k"] { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(1_000_000)); - params.insert(alias.to_string(), json!(500)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert!( - (p.epsilon - 1.0 / 500.0).abs() < 1e-12, - "alias '{}' should produce heap-driven ε", - alias - ); - } + fn cms_with_heap_reports_the_worse_of_point_and_retention_bounds() { + let heap = |depth, width, heap_size| { + derive(&sketch( + SketchAlgorithm::CmsWithHeap, + PlannerParams::CmsWithHeap { + width, + depth, + heap_size, + }, + )) + }; + let retention = heap(5, 1_000_000, 10_000); + assert_eq!(retention.kind, AccuracyKind::TopK); + assert!((retention.epsilon - 1.0 / 10_000.0).abs() < 1e-12); + assert!((retention.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); + let point = heap(4, 100, 1_000_000); + assert!((point.epsilon - std::f64::consts::E / 100.0).abs() < 1e-12); } + /// CountSketch reports ε = 1/√w, which differs from the CMS bound. #[test] fn countsketch_epsilon_is_one_over_sqrt_w() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(100)); - let p = derive(&base_config(AggregationType::CountSketch, params)); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert!((p.epsilon - 0.1).abs() < 1e-9); // 1/√100 = 0.1 - assert!((p.delta.unwrap() - 0.0625).abs() < 1e-12); // Backend-only CountSketch convention. - } - - #[test] - fn hll_epsilon_matches_flajolet_bound() { - let mut params = HashMap::new(); - params.insert("precision".to_string(), json!(14)); - let p = derive(&base_config(AggregationType::HLL, params)); - assert_eq!(p.kind, AccuracyKind::RelativeCardinality); - // 1.04 / √16384 = 1.04 / 128 = 0.008125 - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn hll_uses_default_precision_14() { - let p = derive(&base_config(AggregationType::HLL, HashMap::new())); - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn kll_epsilon_matches_karnin_lang_liberty_bound() { - let mut params = HashMap::new(); - params.insert("K".to_string(), json!(200)); - let p = derive(&base_config(AggregationType::DatasketchesKLL, params)); - assert_eq!(p.kind, AccuracyKind::RankQuantile); - // Planner uses the DataSketches empirical 99th-percentile rank-error fit. - assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); - assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); - } - - #[test] - fn hydra_kll_follows_the_same_kll_bound() { - let mut params = HashMap::new(); - params.insert("k".to_string(), json!(400)); - let p = derive(&base_config(AggregationType::HydraKLL, params)); - // Planner uses the DataSketches empirical 99th-percentile rank-error fit. - assert!((p.epsilon - 2.296 / 400.0_f64.powf(0.9723)).abs() < 1e-12); - } - - #[test] - fn ddsketch_epsilon_is_alpha_directly() { - let mut params = HashMap::new(); - params.insert("alpha".to_string(), json!(0.02)); - let p = derive(&base_config(AggregationType::DDSketch, params)); - assert_eq!(p.kind, AccuracyKind::RelativeQuantile); - assert_eq!(p.epsilon, 0.02); - assert_eq!(p.delta, Some(0.0)); - } - - #[test] - fn ddsketch_uses_default_alpha_0_01() { - let p = derive(&base_config(AggregationType::DDSketch, HashMap::new())); - assert_eq!(p.epsilon, 0.01); - } - - #[test] - fn legacy_wrapper_types_fall_back_to_exact() { - // `SingleSubpopulation` / `MultipleSubpopulation` are - // config-shape wrappers whose real type is in sub_type. - // Without resolving sub_type we return exact (harmless - // lower bound). Phase 6.4 v2 may tighten this. - for t in [ - AggregationType::SingleSubpopulation, - AggregationType::MultipleSubpopulation, - ] { - let p = derive(&base_config(t, HashMap::new())); - assert_eq!(p.kind, AccuracyKind::Exact); - } + let cs = derive(&sketch( + SketchAlgorithm::CountSketch, + PlannerParams::CountSketch { + width: 10_000, + depth: 4, + }, + )); + assert!((cs.epsilon - 0.01).abs() < 1e-12); + assert!(derive(&cms(4, 10_000)).epsilon < cs.epsilon); } + /// Profiles round-trip through their wire form. #[test] fn accuracy_profile_roundtrips_through_serde() { let input = AccuracyProfile { @@ -755,25 +381,4 @@ mod tests { let back: AccuracyProfile = serde_json::from_str(&json).unwrap(); assert_eq!(back, input); } - - #[test] - fn cms_and_countsketch_differ_by_sqrt_e() { - // Sanity check: for the same (w, d), CountSketch's ε is - // smaller by a factor of √e / √w × 1/√w = 1/(√e) — - // i.e. CountSketch is a factor ~1.65 tighter than CMS on - // epsilon alone. Confirms the bounds are not copy-pasted. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(10000)); - let cms = derive(&base_config( - AggregationType::CountMinSketch, - params.clone(), - )); - let cs = derive(&base_config(AggregationType::CountSketch, params)); - // cms.epsilon = e/10000 ≈ 2.718e-4 - // cs.epsilon = 1/√10000 = 0.01 = 1e-2 - // So cms < cs (for w=10000). They cross at w = e. - assert!(cms.epsilon < cs.epsilon); - assert!(cs.epsilon > 0.0); - } } diff --git a/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs index acea64985..1970da66c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs @@ -242,7 +242,7 @@ pub fn clickhouse_reader_factory(config: ClickHouseReaderConfig) -> ReaderFactor return Err("table materialization has no explicit value projection".into()) } } - materialization.population_filter_canonical()?; + materialization.table_population_canonical()?; let mut reader = ClickHouseReader::new(source_config)?; reader.population = Some(materialization.table_population.clone().unwrap_or_default()); reader.value_projection = Some(materialization.effective_value_projection().clone()); @@ -548,22 +548,16 @@ mod tests { #[test] fn typed_source_enters_clickhouse_backfill_lifecycle() { let mut materialization = asap_types::PrecomputeMaterialization::new( - asap_types::AggregationType::Sum, - String::new(), - Default::default(), + "samples.value", asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 1, 1, asap_types::WindowKind::Tumbling, - String::new(), - "samples.value".into(), - None, - Some("another_table".into()), - Some("value".into()), ); + materialization.table_name = Some("another_table".into()); + materialization.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); let factory = clickhouse_reader_factory(config("samples")); materialization.table_timestamp_column = Some("timestamp_ms".into()); let reader = factory( diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index ee6c355ee..a82e8ecfa 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -216,6 +216,9 @@ impl WindowProcessor for BackfillWindowProcessor { .cloned() .ok_or_else(|| format!("agg_id {agg_id} not in current InstalledPrecomputePlan"))?; let program = snapshot.raw_programs.get(&agg_id).cloned(); + let kind = snapshot + .agg_kind(asap_types::sds::StoredOutputId(agg_id)) + .ok_or_else(|| format!("agg_id {agg_id} has no installed state schema"))?; #[cfg(not(test))] if program.is_none() { return Err("backfill requires a post-ASAP DAG installation".into()); @@ -299,7 +302,15 @@ impl WindowProcessor for BackfillWindowProcessor { } else { #[cfg(test)] { - build_backfilled_accumulator(&config, &samples) + build_backfilled_accumulator( + snapshot + .state_family(config.stored_output_id) + .ok_or("backfill fixture has no state family")?, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &samples, + ) } #[cfg(not(test))] { @@ -372,6 +383,7 @@ impl WindowProcessor for BackfillWindowProcessor { &address, _resolver, &config, + &kind, output, accumulator.as_ref(), ) @@ -428,29 +440,18 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - std::collections::HashMap::new(), - grouping_labels, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + let mut config = + PrecomputeMaterialization::new(metric, grouping_labels, 60, 60, WindowKind::Tumbling); + config.allocate_stored_output_id(&sum_family()); + config + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { - let mut map = std::collections::HashMap::new(); - map.insert(config.policy_fp_u64(), config); - Arc::new(InstalledPrecomputePlan::from_raw_ids(map)) + Arc::new(InstalledPrecomputePlan::new([(config, sum_family())])) } #[tokio::test] @@ -640,7 +641,12 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_fixture_accumulator(&cfg); + let mut updater = create_fixture_accumulator( + &sum_family(), + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + ); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -650,7 +656,13 @@ mod tests { // Backfill path: build_backfilled_accumulator. let backfill_bytes = { - let acc = build_backfilled_accumulator(&cfg, &samples); + let acc = build_backfilled_accumulator( + &sum_family(), + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &samples, + ); acc.serialize_to_bytes() }; @@ -843,10 +855,10 @@ mod tests { let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let summary_store = Arc::new(SketchStore::new()); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[cfg.clone()], + vec![(&cfg, &sum_family(), String::new())], ) .unwrap(); summary_store @@ -1000,6 +1012,7 @@ mod tests { .ok() }, &cfg, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), ""), &output, &acc, ) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index 8e5639424..fb6674010 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -352,29 +352,22 @@ mod tests { fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - std::collections::HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.allocate_stored_output_id(&AggregationType::Sum.planner_exact_family().unwrap()); + config } fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { - let mut m = std::collections::HashMap::new(); - m.insert(cfg.policy_fp_u64(), cfg); - Arc::new(InstalledPrecomputePlan::from_raw_ids(m)) + Arc::new(InstalledPrecomputePlan::new([( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + )])) } async fn wait_for_status( diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index d419cb1fc..3b9705028 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -12,8 +12,6 @@ use crate::storage_engines::types::AggregateCore; use crate::tests::accumulator_fixture::create_fixture_accumulator; #[cfg(test)] use asap_physical_operators::factory::AccumulatorUpdater; -#[cfg(test)] -use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -30,17 +28,18 @@ use asap_types::aggregation_config::PrecomputeMaterialization; /// implementation without worrying about the async runtime. #[cfg(test)] pub fn build_backfilled_accumulator( - config: &PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, + update: &planner_types::post_asap::SummaryUpdate, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_fixture_accumulator(config); + let mut updater: Box = create_fixture_accumulator(family, update); for sample in samples { apply_sample( &mut *updater, &sample.labels, sample.value, sample.timestamp_ms, - config, + family, ); } updater.take_accumulator() @@ -49,30 +48,16 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::PrecomputeMaterialization; - use asap_types::enums::WindowKind; use asap_types::AggregationType; - use asap_types::KeyByLabelNames; - use std::collections::HashMap; + use planner_types::post_asap::{SummaryInputExpr, SummaryUpdate}; + use planner_types::pre_asap::ColumnRef; - fn sum_config() -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) + fn value_update() -> SummaryUpdate { + SummaryUpdate::column(ColumnRef::SampleValue) + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } fn raw(labels: &str, ts: i64, v: f64) -> RawSample { @@ -94,18 +79,20 @@ mod tests { AggregationType::CountSketchWithHeap, ] { for mode in ["count", "value"] { - let mut config = sum_config(); - config.aggregation_type = kind; - config.parameters = serde_json::from_value(serde_json::json!({ - "d": 4, "w": 1024, "heap_size": 10, "weight_mode": mode - })) - .unwrap(); + let family = crate::tests::test_utilities::outputs::family( + kind, + &serde_json::json!({"d": 4, "w": 1024, "heap_size": 10}), + ); + let mut update = value_update(); + if mode == "count" { + update.weight = SummaryInputExpr::Constant(1.0); + } let samples = vec![ raw("m{svc=\"a\"}", 10, 100.0), raw("m{svc=\"b\"}", 20, 2.0), raw("m{svc=\"b\"}", 30, 3.0), ]; - let acc = build_backfilled_accumulator(&config, &samples); + let acc = build_backfilled_accumulator(&family, &update, &samples); let mut ranked: Vec<(String, f64)> = if let Some(heap) = acc.as_any() .downcast_ref::() @@ -138,21 +125,18 @@ mod tests { #[test] fn sum_accumulator_sums_all_samples_in_order() { - let config = sum_config(); let samples = vec![ raw("m{svc=\"a\"}", 10, 1.0), raw("m{svc=\"a\"}", 20, 2.0), raw("m{svc=\"a\"}", 30, 3.0), ]; - let acc = build_backfilled_accumulator(&config, &samples); + let acc = build_backfilled_accumulator(&sum_family(), &value_update(), &samples); assert_eq!(crate::tests::accumulator_fixture::sum_of(acc.as_ref()), 6.0); } #[test] fn empty_samples_produce_empty_accumulator() { - let config = sum_config(); - let acc = build_backfilled_accumulator(&config, &[]); - // An empty exact Sum is the additive identity. + let acc = build_backfilled_accumulator(&sum_family(), &value_update(), &[]); assert_eq!(crate::tests::accumulator_fixture::sum_of(acc.as_ref()), 0.0); } } diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index f515c9e14..f12ac1c55 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -33,9 +33,6 @@ //! (in `drivers::ingest::series_resolver`) is the single mint //! authority for every sid in the system. `AggKind::canonical_string()` //! here produces the third element of the resolver's cache key. -//! - [`canonical_parameters`] — helper that renders a parameters -//! `HashMap` into the canonical string form -//! `AggKind::ExactAgg::parameters_canonical` expects. //! //! ## Re-exports for callers //! @@ -122,11 +119,9 @@ pub enum AggKind { /// accumulator serializer. ExactAgg { agg_type: AggregationType, - /// Stable canonical encoding of the agg's - /// `parameters: HashMap` — sorted keys, each - /// value rendered via serde_json. Held as a single owned - /// string so equality / hashing stay cheap and independent of - /// the original HashMap's iteration order. + /// Canonical tuning parameters of the exact state. Planner exact + /// families have none, so installed outputs register it empty; + /// recovered legacy records may carry a non-empty encoding. parameters_canonical: String, /// Canonical form of the policy's spatial-filter predicate; /// see the `Sketch` variant's field doc. @@ -135,103 +130,84 @@ pub enum AggKind { } /// Complete resolver identity for a configured materialization. All live and -/// replay paths must include policy semantics, not just the sketch family. -pub(crate) fn materialization_kind_for_config( +/// replay paths must include the stored output, not just the sketch family. +pub(crate) fn materialization_kind( config: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, ) -> String { format!( "{}|{}", - agg_kind_for_config(config).canonical_string(), + kind.canonical_string(), config.policy_fingerprint() ) } -/// Resolve the physical state family produced by a precompute policy. This is -/// shared by SID minting and store registration so a sketch policy can never -/// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config( - config: &asap_types::aggregation_config::PrecomputeMaterialization, +/// The physical state kind of a stored output of the Planner `family` whose +/// input has the canonical predicate `filter`. This is shared by SID minting +/// and store registration so a sketch output can never be minted as `ExactAgg` +/// and later registered as `Sketch` (or vice versa). +pub fn agg_kind_for_family( + family: &planner_types::post_asap::SummaryFamilyType, + filter: &str, ) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; - - // HLL is intentionally absent from raw-value accumulator dispatch because - // it arrives through SketchEnvelope ingest. It is still a sketch for SID - // identity and store registration, so classify it before consulting the - // accumulator factory contract. - let sketch = (config.aggregation_type == AggregationType::HLL) - .then(|| { - let precision = config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(serde_json::Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) - .unwrap_or(14); - AggKind::Sketch { - algorithm: SketchAlgorithm::Hll, - config: SketchConfig::Hll { precision }, - spatial_filter_canonical: config.spatial_filter_normalized.clone(), - } - }) - .or_else(|| { - config.accumulator_spec().ok().and_then(|spec| { - let SummaryFamilyType::Sketch(kind, _) = spec.family else { - return None; - }; - let algorithm = kind.algorithm().clone(); - let physical = match (kind.algorithm(), kind.params()) { - ( - Algorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - }, - ) => SketchConfig::UnivMon { - heap_size: *heap_size, - sketch_rows: *sketch_rows, - sketch_cols: *sketch_cols, - layers: *layers, + let sketch = match family { + SummaryFamilyType::Sketch(kind, _) => { + let physical = match (kind.algorithm(), kind.params()) { + ( + Algorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, }, - (Algorithm::DDSketch, SketchParams::DDSketch { alpha }) => { - SketchConfig::DDSketch { - relative_accuracy: *alpha, - } - } - (Algorithm::Kll, SketchParams::Kll { k }) => SketchConfig::Kll { k: *k }, - (Algorithm::Hll, SketchParams::Hll { precision }) => SketchConfig::Hll { - precision: (*precision).into(), - }, - (Algorithm::Cms, SketchParams::Cms { width, depth }) - | (Algorithm::CmsWithHeap, SketchParams::CmsWithHeap { width, depth, .. }) => { - SketchConfig::CountMin { - rows: *depth as i32, - cols: *width as i32, - } - } - (Algorithm::CountSketch, SketchParams::CountSketch { width, depth }) - | ( - Algorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { width, depth, .. }, - ) => SketchConfig::CountSketch { + ) => Some(SketchConfig::UnivMon { + heap_size: *heap_size, + sketch_rows: *sketch_rows, + sketch_cols: *sketch_cols, + layers: *layers, + }), + (Algorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Some(SketchConfig::DDSketch { + relative_accuracy: *alpha, + }) + } + (Algorithm::Kll, SketchParams::Kll { k }) => Some(SketchConfig::Kll { k: *k }), + (Algorithm::Hll, SketchParams::Hll { precision }) => Some(SketchConfig::Hll { + precision: (*precision).into(), + }), + (Algorithm::Cms, SketchParams::Cms { width, depth }) + | (Algorithm::CmsWithHeap, SketchParams::CmsWithHeap { width, depth, .. }) => { + Some(SketchConfig::CountMin { rows: *depth as i32, cols: *width as i32, - }, - _ => return None, - }; - Some(AggKind::Sketch { - algorithm, - config: physical, - spatial_filter_canonical: config.spatial_filter_normalized.clone(), - }) + }) + } + (Algorithm::CountSketch, SketchParams::CountSketch { width, depth }) + | ( + Algorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width, depth, .. }, + ) => Some(SketchConfig::CountSketch { + rows: *depth as i32, + cols: *width as i32, + }), + _ => None, + }; + physical.map(|config| AggKind::Sketch { + algorithm: kind.algorithm().clone(), + config, + spatial_filter_canonical: filter.to_owned(), }) - }); - + } + _ => None, + }; sketch.unwrap_or_else(|| AggKind::ExactAgg { - agg_type: config.aggregation_type, - parameters_canonical: canonical_parameters(&config.parameters), - spatial_filter_canonical: config.spatial_filter_normalized.clone(), + agg_type: asap_types::aggregation_type_for_family(family) + .expect("installed plans only carry families with a stored-state kernel"), + // Exact families have no tuning parameters. + parameters_canonical: String::new(), + spatial_filter_canonical: filter.to_owned(), }) } @@ -296,24 +272,6 @@ impl AggKind { } } -/// Render a `HashMap` of parameters into the canonical -/// string form `AggKind::ExactAgg::parameters_canonical` expects. -/// Keys sorted lexicographically; each value via `serde_json`. -pub fn canonical_parameters( - parameters: &std::collections::HashMap, -) -> String { - let sorted: std::collections::BTreeMap<&String, &serde_json::Value> = - parameters.iter().collect(); - let mut buf = String::new(); - for (k, v) in sorted { - buf.push_str(k); - buf.push('='); - buf.push_str(&serde_json::to_string(v).unwrap_or_default()); - buf.push(';'); - } - buf -} - impl AggKind { /// Stable string form of this `AggKind`, used as the third element /// of the `SeriesIdResolver` cache key and as the `agg_kind_canonical` @@ -355,9 +313,7 @@ impl AggKind { spatial_filter_canonical, } => { // `AggregationType`'s `Display` impl is stable - // (matches the snake-case form on the wire) and - // `parameters_canonical` is already canonicalized - // upstream (see [`canonical_parameters`]). + // (matches the snake-case form on the wire). format!( "exact_agg:{}:{}:filter={}", agg_type, parameters_canonical, spatial_filter_canonical, @@ -579,31 +535,19 @@ impl AggPayload { #[cfg(test)] mod tests { use super::*; - use asap_types::{enums::WindowKind, KeyByLabelNames}; - use std::collections::HashMap; + // HLL state registers as a sketch kind with its precision. #[test] - fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::PrecomputeMaterialization::new( - AggregationType::HLL, - String::new(), - HashMap::from([("precision".to_string(), serde_json::json!(12))]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "unique_users".to_string(), - None, - None, - None, + fn hll_family_is_registered_as_a_sketch() { + let family = planner_types::post_asap::SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + planner_types::post_asap::SketchParams::Hll { precision: 12 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, ); - assert!(matches!( - agg_kind_for_config(&config), + agg_kind_for_family(&family, ""), AggKind::Sketch { algorithm: SketchAlgorithm::Hll, config: SketchConfig::Hll { precision: 12 }, diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index b57e679de..9a3b3002c 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -638,6 +638,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, sources: &[FrozenExactWindows], @@ -646,6 +647,7 @@ impl SketchStore { self.publish_frozen_output( sid, config, + kind, output, state, FrozenPublicationInputs::Requested(sources), @@ -657,6 +659,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, cohort: &CompleteRawMaintenanceCohort, @@ -665,6 +668,7 @@ impl SketchStore { self.publish_frozen_output( sid, config, + kind, output, state, FrozenPublicationInputs::Complete(cohort), @@ -676,6 +680,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, proof: FrozenPublicationInputs<'_>, @@ -711,7 +716,7 @@ impl SketchStore { return Err("derived publication differs from its installed input identity".into()); } let labels = self - .register_precompute_output_with_instances(sid, config, output, &mut instances) + .register_precompute_output_with_instances(sid, config, kind, output, &mut instances) .ok_or("derived output registration failed")?; let _mutation = self.begin_state_mutation(); let binding = instances.get(&sid).ok_or("derived output was removed")?; @@ -799,6 +804,32 @@ mod tests { use super::*; use crate::storage_engines::types::PrecomputedOutput; + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap() + } + + fn sum_kind() -> AggKind { + crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), "") + } + + fn sum_catalog( + plan_version: u64, + configs: &[asap_types::PrecomputeMaterialization], + ) -> asap_types::summary_catalog::SummaryCatalog { + let family = sum_family(); + asap_types::summary_catalog::SummaryCatalog::from_outputs( + 1, + plan_version, + configs + .iter() + .map(|c| (c, &family, String::new())) + .collect(), + ) + .unwrap() + } + #[test] fn complete_population_keeps_every_sid_and_rejects_missing_live_binding() { // Multiple SIDs are inventory entries, never an implicit singleton; @@ -812,18 +843,16 @@ mod tests { .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); - first.aggregation_type = asap_types::AggregationType::Sum; - first.aggregation_sub_type = "sum".into(); first.grouping_labels = ["instance".to_string()].into_iter().collect(); + first.allocate_stored_output_id(&"sum"); let mut target = first.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([first.policy_fingerprint().into()]), program_sha256: "0".repeat(64), }); + target.allocate_stored_output_id(&"sum"); let configs = [first.clone(), first, target]; - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &configs) - .unwrap(); + let catalog = sum_catalog(1, &configs); let store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(catalog.clone())) @@ -864,7 +893,15 @@ mod tests { revision, revision, 2000, - |writer| writer.ingest_precompute_with_series_id(sid, config, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + sid, + config, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); } @@ -923,11 +960,11 @@ mod tests { .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); - first.aggregation_type = asap_types::AggregationType::Sum; - first.aggregation_sub_type = "sum".into(); first.grouping_labels = std::iter::empty::().collect(); + first.allocate_stored_output_id(&"sum"); let mut second = first.clone(); second.metric = "cohort_second".into(); + second.allocate_stored_output_id(&"sum"); let mut target = first.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([ @@ -936,10 +973,9 @@ mod tests { ]), program_sha256: "0".repeat(64), }); + target.allocate_stored_output_id(&"sum"); let configs = [first, second, target]; - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &configs) - .unwrap(); + let catalog = sum_catalog(1, &configs); let mut store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(catalog.clone())) @@ -979,7 +1015,15 @@ mod tests { revision, revision, 2000, - |writer| writer.ingest_precompute_with_series_id(sid, config, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + sid, + config, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); requests.push(( @@ -1018,6 +1062,7 @@ mod tests { writer.ingest_precompute_with_series_id( 900, &configs[0], + &sum_kind(), &extra_window, &extra_sum, ) @@ -1056,7 +1101,13 @@ mod tests { sum.update(None, 11.0, 0); assert!(store .publish_complete_raw_maintenance_output( - 902, target, &output, &sum, &complete, [42; 32] + 902, + target, + &sum_kind(), + &output, + &sum, + &complete, + [42; 32] ) .unwrap()); assert!(store @@ -1093,11 +1144,17 @@ mod tests { let mut extra = PrecomputedOutput::new(0, 1000, None, configs[0].policy_fingerprint()); extra.catalog_generation = Some(Arc::clone(&generation)); assert!(store - .register_precompute_output(904, &configs[0], &extra) + .register_precompute_output(904, &configs[0], &sum_kind(), &extra) .is_some()); assert!(store .publish_complete_raw_maintenance_output( - 905, target, &output, &sum, &complete, [42; 32] + 905, + target, + &sum_kind(), + &output, + &sum, + &complete, + [42; 32] ) .is_err()); assert!(!store.instances.read().unwrap().contains_key(&905)); @@ -1111,7 +1168,15 @@ mod tests { .any(|record| record.storage_handle == 905)); store.force_expire(901).unwrap(); assert!(store - .publish_frozen_maintenance_output(903, target, &output, &sum, &cohort, [42; 32]) + .publish_frozen_maintenance_output( + 903, + target, + &sum_kind(), + &output, + &sum, + &cohort, + [42; 32] + ) .is_err()); assert!(store .recover_frozen_maintenance_output(902, target, &cohort, [42; 32], (0, 1000)) @@ -1204,7 +1269,15 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(700, source, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + 700, + source, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); } @@ -1409,7 +1482,9 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(702, source, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id(702, source, &sum_kind(), &output, &sum) + }, ) .unwrap(); let deadline = @@ -1479,21 +1554,15 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let mut source_config = plan.precompute_plan.materializations[0].clone(); - source_config.aggregation_type = asap_types::AggregationType::Sum; - source_config.aggregation_sub_type = "sum".into(); + let source_config = plan.precompute_plan.materializations[0].clone(); let source_id = source_config.policy_fingerprint().into(); let mut target = source_config.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([source_id]), program_sha256: "0".repeat(64), }); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 1, - &[source_config.clone(), target.clone()], - ) - .unwrap(); + target.allocate_stored_output_id(&"sum"); + let catalog = sum_catalog(1, &[source_config.clone(), target.clone()]); let store = Arc::new(SketchStore::new()); store.install_summary_catalog(Arc::new(catalog)).unwrap(); let directory = tempfile::tempdir().unwrap(); @@ -1510,10 +1579,10 @@ mod tests { let mut source_output = output.clone(); source_output.policy_fp = source_config.policy_fingerprint(); store - .register_precompute_output(600, &source_config, &source_output) + .register_precompute_output(600, &source_config, &sum_kind(), &source_output) .unwrap(); store - .register_precompute_output(601, &target, &output) + .register_precompute_output(601, &target, &sum_kind(), &output) .unwrap(); let record = store .metadata_record(&store.instances.read().unwrap()[&601]) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index bb47a012d..abccdb791 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -40,8 +40,8 @@ use crate::storage_engines::sketch_db::sds::{ }; pub use crate::storage_engines::sketch_db::data::{ - canonical_parameters, AccuracyBound, AggKind, AggPayload, AggregationType, Capability, - SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchTimeSeries, + AccuracyBound, AggKind, AggPayload, AggregationType, Capability, SketchAlgorithm, SketchConfig, + SketchEncoding, SketchSampleState, SketchTimeSeries, }; fn now_ms() -> u64 { @@ -764,6 +764,7 @@ impl SummaryPublicationWriter<'_> { address: &asap_types::sds::StoredSummaryKey, resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -813,7 +814,7 @@ impl SummaryPublicationWriter<'_> { return None; } self.0 - .ingest_precompute_with_admission(handle, config, output, state) + .ingest_precompute_with_admission(handle, config, kind, output, state) } #[cfg(test)] @@ -821,11 +822,12 @@ impl SummaryPublicationWriter<'_> { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { self.0 - .ingest_precompute_with_admission(sid, config, output, state) + .ingest_precompute_with_admission(sid, config, kind, output, state) } #[cfg(test)] @@ -3180,6 +3182,7 @@ impl SketchStore { &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3187,7 +3190,7 @@ impl SketchStore { if admission.is_finite_closed() { return None; } - self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, output, accumulator) + self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, kind, output, accumulator) } #[cfg(test)] @@ -3195,6 +3198,7 @@ impl SketchStore { &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3206,31 +3210,33 @@ impl SketchStore { // invoking the sid-direct sibling. let (attrs_fp, _label_values_map) = build_attrs_fp_and_label_map(agg_cfg, output).ok()?; let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(agg_cfg); + crate::storage_engines::sketch_db::data::materialization_kind(agg_cfg, kind); let sid = mint_sid(&agg_cfg.metric, &attrs_fp, &agg_kind_canonical).into()?; - self.ingest_precompute_with_admission(sid, agg_cfg, output, accumulator) + self.ingest_precompute_with_admission(sid, agg_cfg, kind, output, accumulator) } fn register_precompute_output( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Option> { let mut instances = self.instances.write().ok()?; - self.register_precompute_output_with_instances(sid, agg_cfg, output, &mut instances) + self.register_precompute_output_with_instances(sid, agg_cfg, kind, output, &mut instances) } fn register_precompute_output_with_instances( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, instances: &mut HashMap, ) -> Option> { let (_attrs_fp, label_values_map) = build_attrs_fp_and_label_map(agg_cfg, output).ok()?; let key_names = &agg_cfg.grouping_labels.names(); - let agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg); + let agg_kind = kind.clone(); let (capability, accuracy) = agg_kind.capability_and_accuracy(); match instances.get(&sid) { @@ -3314,6 +3320,7 @@ impl SketchStore { &self, sid: u64, agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3321,13 +3328,14 @@ impl SketchStore { if admission.is_finite_closed() { return None; } - self.ingest_precompute_with_admission(sid, agg_cfg, output, accumulator) + self.ingest_precompute_with_admission(sid, agg_cfg, kind, output, accumulator) } fn ingest_precompute_with_admission( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3335,19 +3343,14 @@ impl SketchStore { tracing::warn!(sid, %error, "rejecting summary state without a stored codec"); return None; } - let expected = agg_cfg.accumulator_spec().ok()?.family; - if matches!( - expected, - planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) - ) && accumulator - .get_accumulator_type() - .planner_exact_family() - .as_ref() - != Some(&expected) - { - return None; + if let AggKind::ExactAgg { agg_type, .. } = kind { + if accumulator.get_accumulator_type().planner_exact_family() + != agg_type.planner_exact_family() + { + return None; + } } - let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; + let label_values_map = self.register_precompute_output(sid, agg_cfg, kind, output)?; // Keep the physical lifetime alive through publication. Removal takes // this same lock exclusively, so it cannot race metadata validation and @@ -3398,7 +3401,7 @@ impl SketchStore { } let window = (output.start_timestamp, output.end_timestamp); - let accepted = match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { + let accepted = match kind { AggKind::Sketch { .. } => self.append_sample_with_binding( sid, label_values_map, @@ -3933,9 +3936,13 @@ mod tests { fn meta_for_config( sid: u64, config: &asap_types::PrecomputeMaterialization, + plan: &asap_types::precompute_plan::PrecomputePlan, ) -> SummarySeriesMetadata { let mut metadata = meta_with_policy(sid, config.policy_fingerprint()); - metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(config); + metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.state_family(config.stored_output_id).unwrap(), + &plan.population_filter(config).unwrap(), + ); metadata } @@ -4049,7 +4056,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -4060,7 +4070,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(41, &state_config)); + store.register(meta_for_config(41, &state_config, &plan.precompute_plan)); store.append_sample( 41, BTreeMap::from([("job".to_string(), "api".to_string())]), @@ -4143,7 +4153,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -4154,7 +4167,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - store.register(meta_for_config(42, &state_config)); + store.register(meta_for_config(42, &state_config, &plan.precompute_plan)); let producers = BTreeMap::from([( StoredOutputId::from(fingerprint), ( @@ -4234,7 +4247,7 @@ mod tests { identity_version: 1, plan_id: 7, plan_version: 2, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), materialization: PolicyFingerprint(41).into(), series_identity: "service=checkout,zone=a".into(), schema_id: "schema-41".into(), @@ -4627,17 +4640,6 @@ mod tests { // canonical strings → different resolver cache keys → different // sids by construction. - #[test] - fn canonical_parameters_is_insertion_order_independent() { - let mut p_ab = std::collections::HashMap::new(); - p_ab.insert("alpha".to_string(), serde_json::json!(1)); - p_ab.insert("beta".to_string(), serde_json::json!(2)); - let mut p_ba = std::collections::HashMap::new(); - p_ba.insert("beta".to_string(), serde_json::json!(2)); - p_ba.insert("alpha".to_string(), serde_json::json!(1)); - assert_eq!(canonical_parameters(&p_ab), canonical_parameters(&p_ba)); - } - // Unkeyed Planner exact MIN panes feed the MIN rollup. #[test] fn planner_exact_min_panes_feed_the_min_rollup() { @@ -5349,14 +5351,17 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) .unwrap() .clone(); let fingerprint = state_config.policy_fingerprint(); - let metadata = meta_for_config(507, &state_config); + let metadata = meta_for_config(507, &state_config, &plan.precompute_plan); let record = StoredOutputMetadataRecord::new( metadata.storage_handle, metadata.metric_name.clone(), @@ -5403,7 +5408,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5414,7 +5422,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(509, &state_config)); + store.register(meta_for_config(509, &state_config, &plan.precompute_plan)); store.append_sample(509, BTreeMap::new(), (0, 10_000), sample(1)); let mut next = plan.summary_catalog; next.plan_version += 1; @@ -5445,13 +5453,13 @@ mod tests { ) .unwrap(); assert!(inventory.instances.is_empty()); - store.register(meta_for_config(510, &state_config)); + store.register(meta_for_config(510, &state_config, &plan.precompute_plan)); store.append_sample(510, BTreeMap::new(), (0, 10_000), sample(2)); assert_eq!(store.series_ids_for_policy(fingerprint), vec![510]); - let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let incompatible = asap_types::summary_catalog::SummaryCatalog::from_outputs( plan.precompute_plan.envelope.plan_id, plan.precompute_plan.envelope.plan_version + 2, - &[], + Vec::new(), ) .unwrap(); store @@ -5477,7 +5485,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5500,7 +5511,11 @@ mod tests { .unwrap(); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); old_sid = resolver.resolve("metric", "group", "family"); - store.register(meta_for_config(old_sid, &state_config)); + store.register(meta_for_config( + old_sid, + &state_config, + &plan.precompute_plan, + )); for pane in 0..4 { store.append_sample( old_sid, @@ -5527,7 +5542,11 @@ mod tests { }) .unwrap(); assert_ne!(new_sid, old_sid); - store.register(meta_for_config(new_sid, &state_config)); + store.register(meta_for_config( + new_sid, + &state_config, + &plan.precompute_plan, + )); for pane in 0..4 { store.append_sample( new_sid, @@ -5602,7 +5621,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5614,7 +5636,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(850, &state_config)); + store.register(meta_for_config(850, &state_config, &plan.precompute_plan)); let generation = store.active_catalog_generation().unwrap(); let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), @@ -5711,7 +5733,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5724,7 +5749,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(851, &state_config)); + store.register(meta_for_config(851, &state_config, &plan.precompute_plan)); let mut config = durable_cfg(directory.path().to_path_buf()); config.hot_window_ms = None; config.seal_window_count = 100; @@ -5833,7 +5858,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5849,7 +5877,7 @@ mod tests { .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); for sid in [801, 802, 803] { - store.register(meta_for_config(sid, &state_config)); + store.register(meta_for_config(sid, &state_config, &plan.precompute_plan)); } let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for sid in [801, 802, 803] { @@ -5876,7 +5904,7 @@ mod tests { expected_retirement = store.force_retire(801, Duration::from_secs(3600)).unwrap(); assert!(store.force_expire(802).is_some()); assert!(store.remove_instance(803).is_some()); - store.register(meta_for_config(803, &state_config)); + store.register(meta_for_config(803, &state_config, &plan.precompute_plan)); assert!( store.instance(803).is_none(), "removed SID reused before restart" @@ -5906,7 +5934,7 @@ mod tests { recovered.instance(803).is_none(), "removed state resurrected" ); - recovered.register(meta_for_config(803, &state_config)); + recovered.register(meta_for_config(803, &state_config, &plan.precompute_plan)); assert!( recovered.instance(803).is_none(), "removed SID reused after restart" @@ -5929,7 +5957,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5946,7 +5977,7 @@ mod tests { )]); let tmp = tempfile::TempDir::new().unwrap(); let disk = tmp.path().to_path_buf(); - let mut metadata = meta_for_config(506, &state_config); + let mut metadata = meta_for_config(506, &state_config, &plan.precompute_plan); metadata.group_by_keys = ["job".to_string()].into_iter().collect(); { @@ -6715,7 +6746,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -6756,7 +6790,7 @@ mod tests { ) .unwrap(); assert_ne!(handle, other_population); - store.register(meta_for_config(handle, &config)); + store.register(meta_for_config(handle, &config, &plan.precompute_plan)); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for pane in 0..4 { store.append_sample( @@ -6843,6 +6877,7 @@ mod tests { .iter() .find(|config| config.policy_fingerprint() == output.stored_output_id.fingerprint()) .unwrap(), + &plan.precompute_plan, )); assert_eq!(store.storage_handles_for_output(&output), vec![100]); let mut next = plan.summary_catalog; @@ -6942,31 +6977,38 @@ impl SketchStore { output.population_labels = Some(record.group.clone()); output.catalog_generation = Some(Arc::clone(&generation)); output.stored_output_reference = Some(record.reference.clone()); - let state = crate::precompute_engine::revisions::decode_state(record, config)?; + let family = plan + .precompute_plan + .state_family(definition) + .ok_or("revision view lacks output state schema")?; + let kind = plan + .installed_precompute_plan + .agg_kind(definition) + .ok_or("revision view lacks output state schema")?; + let state = crate::precompute_engine::revisions::decode_state(record, family)?; let labels = view - .register_precompute_output(sid, config, &output) + .register_precompute_output(sid, config, &kind, &output) .ok_or("revision view rejected output metadata")?; // This is an immutable read view of already committed revisions, // not additive producer admission (which correctly rejects derived // writes). Every payload was decoded against its installed family. - let accepted = - match crate::storage_engines::sketch_db::data::agg_kind_for_config(config) { - AggKind::Sketch { .. } => view.append_sample_with_binding( - sid, - labels, - (record.start_ms, record.end_ms), - SketchSampleState { - bytes: state.serialize_to_bytes(), - encoding: SketchEncoding::full_frame_for(state.as_ref()), - }, - ), - AggKind::ExactAgg { .. } => view.append_precompute_with_binding( - sid, - labels, - (record.start_ms, record.end_ms), - state.clone_boxed_core(), - ), - }; + let accepted = match kind { + AggKind::Sketch { .. } => view.append_sample_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + SketchSampleState { + bytes: state.serialize_to_bytes(), + encoding: SketchEncoding::full_frame_for(state.as_ref()), + }, + ), + AggKind::ExactAgg { .. } => view.append_precompute_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + state.clone_boxed_core(), + ), + }; if !accepted { return Err("revision view rejected committed payload".into()); } diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index 636be0961..bbf70853b 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -39,6 +39,7 @@ impl SketchStore { &self, resolver: &SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, output: &crate::storage_engines::types::PrecomputedOutput, batch: Batch, max_bytes: usize, @@ -57,7 +58,6 @@ impl SketchStore { self.read_complete_raw_maintenance_cohort(generation, &program.inputs, window)?; let (population_key, group) = build_attrs_fp_and_label_map(config, output)?; let state = NativeSummaryOutput::new(batch, max_bytes)?; - let family = config.accumulator_spec().map_err(|e| e.to_string())?.family; if state .batch() .schema() @@ -69,13 +69,13 @@ impl SketchStore { planner_types::post_asap::SummaryFamilyType::Plain(_) ) }) - .any(|field| field.dtype != family) + .any(|field| &field.dtype != family) { return Err("native output schema differs from installed definition".into()); } for value in state.batch().rows().iter().flatten() { if let Value::Summary { family: actual, .. } = value { - if actual != &family { + if actual != family { return Err("native output family differs from installed definition".into()); } } @@ -104,6 +104,8 @@ impl SketchStore { self.publish_complete_raw_maintenance_output( sid, config, + // Derived outputs read stored states, so their input has no predicate. + &crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""), output, &state, &cohort, @@ -378,6 +380,14 @@ mod tests { .iter() .find(|c| c.derived_input.is_some()) .unwrap(); + let source_family = plan + .precompute_plan + .state_family(source.stored_output_id) + .unwrap(); + let target_family = plan + .precompute_plan + .state_family(target.stored_output_id) + .unwrap(); let directory = tempfile::tempdir().unwrap(); let resolver_path = directory.path().join("resolver.jsonl"); let resolver = SeriesIdResolver::open(resolver_path.clone()).unwrap(); @@ -421,7 +431,18 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(700, source, &output, &state), + |writer| { + writer.ingest_precompute_with_series_id( + 700, + source, + &crate::storage_engines::sketch_db::data::agg_kind_for_family( + source_family, + "", + ), + &output, + &state, + ) + }, ) .unwrap(); } @@ -456,14 +477,8 @@ mod tests { time_index: None, }); let input = Batch::try_new(raw_schema.clone(), values).unwrap(); - let build = Operator::summary_build( - raw_schema, - target.accumulator_spec().unwrap().family, - 0, - None, - vec![], - ) - .unwrap(); + let build = + Operator::summary_build(raw_schema, target_family.clone(), 0, None, vec![]).unwrap(); let output_batch = run(input, build); let expected_schema = output_batch.schema().clone(); let mut output = PrecomputedOutput::new(0, 60_000, None, target.policy_fingerprint()); @@ -473,13 +488,21 @@ mod tests { .publish_native_summary_output( &resolver, target, + target_family, &output, output_batch.clone(), 1 << 20 ) .unwrap()); store - .publish_native_summary_output(&resolver, target, &output, output_batch, 1 << 20) + .publish_native_summary_output( + &resolver, + target, + target_family, + &output, + output_batch, + 1 << 20, + ) .unwrap(); assert_eq!(persistence.manifest.live_parts().len(), before + 1); let reference = plan diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 93b0ed5c8..176086452 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -199,35 +199,24 @@ mod tests { use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::empty().into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: format!("metric_{id}"), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + fn sum_agg_config( + id: u64, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + format!("metric_{id}"), + KeyByLabelNames::empty(), + 1, + 1, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + crate::tests::test_utilities::outputs::allocated( + config, + AggregationType::Sum.planner_exact_family().unwrap(), + ) } /// Build a streaming-config keyed on the policy-fingerprint u64 @@ -235,18 +224,14 @@ mod tests { /// marker_id → fingerprint mapping so callers can look up the /// right key. fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { - let mut map = HashMap::new(); + let mut outputs = Vec::new(); let mut id_to_fp = HashMap::new(); for &id in ids { let cfg = sum_agg_config(id); - let fp = cfg.policy_fp_u64(); - id_to_fp.insert(id, fp); - map.insert(fp, cfg); + id_to_fp.insert(id, cfg.0.policy_fp_u64()); + outputs.push(cfg); } - ( - Arc::new(InstalledPrecomputePlan::from_raw_ids(map)), - id_to_fp, - ) + (Arc::new(InstalledPrecomputePlan::new(outputs)), id_to_fp) } fn write_one( @@ -280,6 +265,9 @@ mod tests { .ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, + &installed_precompute_plan + .agg_kind(agg_cfg.stored_output_id) + .unwrap(), &output, &acc, ) diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index d6310b46f..e28518f81 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -13,7 +13,7 @@ use std::time::Duration; use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; -use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; +use crate::storage_engines::sketch_db::data::AggKind; use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::lifecycle::AggStatus; @@ -166,21 +166,19 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { - let agg_kind = AggKind::ExactAgg { - agg_type: cfg.aggregation_type, - parameters_canonical: canonical_parameters(&cfg.parameters), - spatial_filter_canonical: cfg.spatial_filter_normalized.clone(), - }; +fn signature_from_agg_config(cfg: &PrecomputeMaterialization, agg_kind: &AggKind) -> Vec { let group_by_keys: BTreeSet = cfg.grouping_labels.iter().cloned().collect(); - signature_bytes(&cfg.metric, &agg_kind, &group_by_keys) + signature_bytes(&cfg.metric, agg_kind, &group_by_keys) } fn build_live_signature_set(config: &InstalledPrecomputePlan) -> HashSet> { config .materializations() .values() - .map(signature_from_agg_config) + .filter_map(|cfg| { + let kind = config.agg_kind(cfg.stored_output_id)?; + Some(signature_from_agg_config(cfg, &kind)) + }) .collect() } @@ -276,23 +274,15 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - HashMap::new(), - KeyByLabelNames::new(group_by.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 1, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + crate::tests::test_utilities::outputs::output( + metric, + agg_type.planner_exact_family().unwrap(), + group_by, 1, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, ) } @@ -321,12 +311,13 @@ mod tests { } } - fn streaming(configs: Vec) -> InstalledPrecomputePlan { - let mut map = HashMap::new(); - for (i, c) in configs.into_iter().enumerate() { - map.insert(i as u64 + 1, c); - } - InstalledPrecomputePlan::from_raw_ids(map) + fn streaming( + configs: Vec<( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + )>, + ) -> InstalledPrecomputePlan { + InstalledPrecomputePlan::new(configs) } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 58ea8eb62..3ffe43dcb 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -78,30 +78,8 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { // A valid deployment reference does not authorize bytes in another state format. fn operator_matches(summary: &SummaryDescriptor, actual: &AggKind) -> bool { match &summary.operator { - SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - parameters, - .. - } => { - let config = asap_types::PrecomputeMaterialization::new( - *aggregation_type, - aggregation_sub_type.clone(), - parameters.clone().into_iter().collect(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), - 1, - 1, - asap_types::WindowKind::Tumbling, - String::new(), - String::new(), - None, - None, - None, - ); - super::data::agg_kind_for_config(&config).operator_canonical_string() + SummaryOperator::Configured { family } => { + super::data::agg_kind_for_family(family, "").operator_canonical_string() == actual.operator_canonical_string() } SummaryOperator::ExactAgg { diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index b624d8164..c66e1b6b5 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -702,7 +702,7 @@ mod tests { generated_at_unix_ms: activation_unix_ms, activation_unix_ms, expiry_unix_ms, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; @@ -735,7 +735,7 @@ mod tests { generated_at_unix_ms: activation_unix_ms, activation_unix_ms, expiry_unix_ms, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }, @@ -830,23 +830,17 @@ mod tests { ); } - fn dummy_agg(id: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), + fn dummy_agg( + id: u64, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + crate::tests::test_utilities::outputs::output( + &format!("metric_{id}"), + AggregationType::Sum.planner_exact_family().unwrap(), + vec![], 60, - 60, - WindowKind::Tumbling, - String::new(), - format!("metric_{id}"), - None, - None, - None, ) } @@ -856,15 +850,14 @@ mod tests { /// dummy-id→fingerprint mapping so the assertions below can look /// up entries. fn cfg_with_ids(ids: &[u64]) -> (InstalledPrecomputePlan, std::collections::HashMap) { - let mut map = HashMap::new(); + let mut outputs = Vec::new(); let mut id_to_fp = std::collections::HashMap::new(); for &id in ids { let cfg = dummy_agg(id); - let fp = cfg.policy_fp_u64(); - id_to_fp.insert(id, fp); - map.insert(fp, cfg); + id_to_fp.insert(id, cfg.0.policy_fp_u64()); + outputs.push(cfg); } - (InstalledPrecomputePlan::from_raw_ids(map), id_to_fp) + (InstalledPrecomputePlan::new(outputs), id_to_fp) } #[test] diff --git a/data_plane/src/storage_engines/types/installed_precompute_plan.rs b/data_plane/src/storage_engines/types/installed_precompute_plan.rs index 1ec253024..d616a0307 100644 --- a/data_plane/src/storage_engines/types/installed_precompute_plan.rs +++ b/data_plane/src/storage_engines/types/installed_precompute_plan.rs @@ -5,7 +5,8 @@ use std::ops::Index; use super::storage_backend::StorageBackend; use asap_types::sds::StoredOutputId; -use asap_types::{PolicyRegistry, PrecomputeMaterialization}; +use asap_types::PrecomputeMaterialization; +use planner_types::post_asap::SummaryFamilyType; #[derive(Debug, Clone)] pub struct InstalledPrecomputePlan { @@ -14,16 +15,33 @@ pub struct InstalledPrecomputePlan { HashMap>, pub(crate) precompute_plan: Option, pub(crate) materializations_by_output: HashMap, + /// Each output's stored state family, from its schema contract. + state_families: HashMap, + /// Each output's canonical input predicate, from its Planner DAG scan. + population_filters: HashMap, + /// The per-item dimension of outputs whose Planner update keys items by a label. + item_labels: HashMap, pub(crate) storage_backend: StorageBackend, } impl InstalledPrecomputePlan { - fn derived_view(materializations: HashMap) -> Self { + fn derived_view( + outputs: impl IntoIterator, + ) -> Self { + let mut materializations = HashMap::new(); + let mut state_families = HashMap::new(); + for (config, family) in outputs { + state_families.insert(config.stored_output_id, family); + materializations.insert(config.stored_output_id, config); + } Self { partitioning: Default::default(), raw_programs: HashMap::new(), precompute_plan: None, materializations_by_output: materializations, + state_families, + population_filters: HashMap::new(), + item_labels: HashMap::new(), storage_backend: StorageBackend::default(), } } @@ -31,6 +49,53 @@ impl InstalledPrecomputePlan { /// Production construction always validates the executable DAG and bindings. pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { let materializations = plan.runtime_materializations()?; + let outputs = materializations + .into_values() + .map(|config| { + let family = plan + .state_family(config.stored_output_id) + .cloned() + .ok_or_else(|| anyhow::anyhow!("stored output has no state schema"))?; + Ok((config, family)) + }) + .collect::>>()?; + let population_filters = outputs + .iter() + .map(|(config, _)| { + plan.population_filter(config) + .map(|filter| (config.stored_output_id, filter)) + .map_err(anyhow::Error::msg) + }) + .collect::>>()?; + let mut item_labels = HashMap::new(); + for (config, _) in &outputs { + use planner_types::post_asap::{PostAsapOperatorPayload, SummaryInputExpr}; + use planner_types::pre_asap::ColumnRef; + let Some((node, _)) = plan + .summary_producer(config.stored_output_id) + .map_err(anyhow::Error::msg)? + else { + continue; + }; + if let PostAsapOperatorPayload::SummaryAgg { + input: + planner_types::post_asap::SummaryUpdate { + item: + Some(SummaryInputExpr::Column( + ColumnRef::Named(label) | ColumnRef::Qualified { name: label, .. }, + )), + .. + }, + .. + } = node.payload + { + item_labels.insert(config.stored_output_id, label); + } + } + let materializations: HashMap<_, _> = outputs + .iter() + .map(|(config, _)| (config.stored_output_id, config.clone())) + .collect(); let mut programs = HashMap::new(); for config in materializations.values().filter(|config| { config.derived_input.is_none() @@ -42,7 +107,9 @@ impl InstalledPrecomputePlan { .map_err(anyhow::Error::msg)?; programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); } - let mut view = Self::derived_view(materializations); + let mut view = Self::derived_view(outputs); + view.population_filters = population_filters; + view.item_labels = item_labels; view.partitioning = crate::precompute_engine::partitioning::DagPartitioning::from_plan(&plan); view.precompute_plan = Some(plan); @@ -52,30 +119,20 @@ impl InstalledPrecomputePlan { // Isolated kernel/storage fixtures can omit a physical installation. This // constructor is absent from the production library and binary. + /// Each output is paired with its stored state family. #[cfg(test)] - pub fn new(materializations: HashMap) -> Self { - Self::derived_view(materializations) - } - - /// Fixtures hold raw output ids, so say that once here rather than - /// wrapping every literal. Production construction goes through - /// `from_precompute_plan`, which takes the ids from the plan itself. - #[cfg(test)] - pub fn from_raw_ids(materializations: HashMap) -> Self { - Self::new( - materializations - .into_iter() - .map(|(id, value)| (StoredOutputId(id), value)) - .collect(), - ) + pub fn new( + outputs: impl IntoIterator, + ) -> Self { + Self::derived_view(outputs) } #[cfg(test)] pub fn with_storage_backend( - materializations: HashMap, + outputs: impl IntoIterator, storage_backend: StorageBackend, ) -> Self { - let mut view = Self::derived_view(materializations); + let mut view = Self::derived_view(outputs); view.storage_backend = storage_backend; view } @@ -92,19 +149,21 @@ impl InstalledPrecomputePlan { }); #[cfg(test)] let selected = selected.or_else(|| { - let materializations = self + let outputs = self .materializations_by_output .values() - .cloned() - .collect::>(); - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 0, - 0, - &materializations, - ) - .ok()? - .output_reference(definition) - .ok() + .map(|config| { + Some(( + config, + self.state_family(config.stored_output_id)?, + self.population_filter(config.stored_output_id).to_owned(), + )) + }) + .collect::>>()?; + asap_types::summary_catalog::SummaryCatalog::from_outputs(0, 0, outputs) + .ok()? + .output_reference(definition) + .ok() }); selected } @@ -134,8 +193,41 @@ impl InstalledPrecomputePlan { self.materializations_by_output.contains_key(&output) } - pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs(self.materializations_by_output.values().cloned()) + /// The stored state family of `output`. + pub fn state_family(&self, output: StoredOutputId) -> Option<&SummaryFamilyType> { + self.state_families.get(&output) + } + + /// The physical stored-state kind of `output`. + pub fn agg_kind( + &self, + output: StoredOutputId, + ) -> Option { + self.state_family(output).map(|family| { + crate::storage_engines::sketch_db::data::agg_kind_for_family( + family, + self.population_filter(output), + ) + }) + } + + /// Whether `output`'s panes follow PromQL's `(start, end]` range + /// convention. Raw Planner programs read only time-series scans, whose + /// ranges are PromQL ranges. + pub fn right_closed_panes(&self, output: StoredOutputId) -> bool { + self.raw_programs.contains_key(&output.as_u64()) + } + + /// The label that keys `output`'s items, when its Planner update has one. + pub fn item_label(&self, output: StoredOutputId) -> Option<&str> { + self.item_labels.get(&output).map(String::as_str) + } + + /// The canonical input predicate of `output`; empty when unfiltered. + pub fn population_filter(&self, output: StoredOutputId) -> &str { + self.population_filters + .get(&output) + .map_or("", String::as_str) } } @@ -170,6 +262,111 @@ impl Default for InstalledPrecomputePlan { #[cfg(test)] mod tests { + fn filtered_plan(job: &str) -> control_plane::physical::compiler::PrecomputePlan { + let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + snapshot["query_workload"]["repeating_queries"][0]["query"] = + serde_json::json!(format!("quantile_over_time(0.99, m{{job=\"{job}\"}}[1m])")); + crate::tests::test_utilities::planning::quoted_snapshot( + serde_json::from_value(snapshot).unwrap(), + false, + ) + .compile_promql() + .unwrap() + .precompute_plan + } + + /// Rebind every materialization node of `plan`'s DAGs to `output`. + fn bind_all(plan: &mut control_plane::physical::compiler::PrecomputePlan, output: u64) { + for installed in plan.executable_dags.values_mut() { + for binding in installed.binding.nodes.values_mut() { + if let asap_types::executable_plan::BackendNodeBinding::Materialization { + stored_output, + } = binding + { + *stored_output = asap_types::sds::StoredOutputId(output); + } + } + } + } + + // Two DAGs that read different populations cannot share one stored output. + #[test] + fn producers_disagreeing_on_population_are_rejected() { + let mut plan = filtered_plan("a"); + let output = plan.materializations[0].stored_output_id.as_u64(); + let mut other = filtered_plan("b"); + bind_all(&mut other, output); + let (_, mut dag) = other.executable_dags.into_iter().next().unwrap(); + dag.document.query_id = "other-population".into(); + plan.executable_dags.insert("other-population".into(), dag); + let error = plan.validate().unwrap_err().to_string(); + assert!(error.contains("disagree on its computation"), "{error}"); + } + + // A raw output with no Planner producer is not read as unfiltered. + #[test] + fn unbound_raw_output_is_rejected() { + let mut plan = filtered_plan("a"); + let mut unbound = plan.materializations[0].clone(); + unbound.metric = "unbound".into(); + unbound.stored_output_id = asap_types::sds::StoredOutputId(1); + let mut schema = plan.schemas[0].clone(); + schema.materialization = unbound.stored_output_id; + schema.schema_id = schema.schema_id.replace( + &plan.materializations[0] + .stored_output_id + .as_u64() + .to_string(), + "1", + ); + schema.stored_output_reference = + asap_types::sds::StoredOutputReference::for_output(unbound.stored_output_id); + plan.materializations.push(unbound); + plan.schemas.push(schema); + let error = plan.validate().unwrap_err().to_string(); + assert!( + error.contains("raw output has no Planner producer"), + "{error}" + ); + } + + // A raw output's input predicate, pane convention and stored state kind + // come from its installed Planner producer, not from the materialization. + #[test] + fn raw_output_computation_comes_from_its_planner_producer() { + let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + snapshot["query_workload"]["repeating_queries"][0]["query"] = + serde_json::json!("quantile_over_time(0.99, m{job=\"api\"}[1m])"); + let plan = crate::tests::test_utilities::planning::quoted_snapshot( + serde_json::from_value(snapshot).unwrap(), + false, + ) + .compile_promql() + .unwrap(); + let installed = + super::InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan.clone()) + .unwrap(); + let output = plan.precompute_plan.materializations[0].stored_output_id; + assert_eq!(installed.population_filter(output), "{job=\"api\"}"); + assert!(installed.right_closed_panes(output)); + let family = installed.state_family(output).unwrap(); + assert_eq!(Some(family), plan.precompute_plan.state_family(output)); + assert_eq!( + installed.agg_kind(output).unwrap().canonical_string(), + crate::storage_engines::sketch_db::data::agg_kind_for_family(family, "{job=\"api\"}") + .canonical_string() + ); + let data = &plan.summary_catalog.data_descriptors + [&plan.summary_catalog.outputs[&output].data_descriptor_id]; + assert_eq!(data.population_filter_canonical, "{job=\"api\"}"); + } + // Flat lists cannot enter through the authoritative physical-plan document. #[test] fn rejects_flat_aggregation_documents() { diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 8644f37da..fa6a147f3 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -78,7 +78,7 @@ impl PrecomputedOutput { /// /// `policy_fp` is the content-addressed handle on the source /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the - /// config via `PolicyRegistry::get(policy_fp)`. Construction sites + /// installed output by it. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. pub fn new( diff --git a/data_plane/src/tests/accumulator_fixture.rs b/data_plane/src/tests/accumulator_fixture.rs index 4df1d3018..d5e9a9c3e 100644 --- a/data_plane/src/tests/accumulator_fixture.rs +++ b/data_plane/src/tests/accumulator_fixture.rs @@ -1,34 +1,40 @@ -//! Config fixtures for backend integration tests; production binds Planner payloads. +//! Kernel fixtures for backend unit tests: a stored state family and the +//! Planner update it ingests. Production executes the bound Planner DAG. use asap_physical_operators::factory::*; use asap_physical_operators::summary_kernels::exact::ExactAccumulator; -use asap_summary_state::{AggregateCore, KeyByLabelValues}; -use asap_types::{accumulator_spec::cms_params, PrecomputeMaterialization}; -use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; -#[cfg(test)] -/// Return `true` if `config` produces a keyed (MultipleSubpopulation) updater, -/// without allocating an updater object. +use asap_summary_state::{AggregateCore, AggregationType, KeyByLabelValues}; +use planner_types::post_asap::{ + ExactKind, GroupingStrategy, HydraParams, SketchAlgorithm, SketchParams, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; + +/// Whether `family` stores a keyed (multi-population) accumulator. /// /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { - config - .accumulator_spec() - .expect("valid fixture") - .grouping - .is_some() +#[cfg(test)] +pub fn family_is_keyed(family: &SummaryFamilyType) -> bool { + match family { + SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }) => true, + SummaryFamilyType::Sketch(kind, _) => matches!( + kind.algorithm(), + SketchAlgorithm::Cms + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketch + | SketchAlgorithm::CountSketchWithHeap + ), + _ => false, + } } -/// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. -/// -/// * `value` / `sum`: sum values per key (default). -/// * `count` / `frequency` / `freq`: count occurrences per key. +/// Top-k ranking quantity: a unit weight counts occurrences per key, +/// otherwise values are summed. #[cfg(test)] -fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { - match config.sample_update_rule() { - asap_types::SampleUpdateRule::Count => TopkWeight::Count, - asap_types::SampleUpdateRule::Value { .. } - | asap_types::SampleUpdateRule::CounterDelta { .. } => TopkWeight::Value, +fn topk_weight(update: &SummaryUpdate) -> TopkWeight { + match update.weight { + SummaryInputExpr::Constant(1.0) => TopkWeight::Count, + _ => TopkWeight::Value, } } @@ -36,20 +42,15 @@ fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { // Factory function // --------------------------------------------------------------------------- -/// Read the KLL `k` out of `SketchParams::Kll`. `accumulator_spec()` -/// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with -/// `SketchParams::Kll`, so the -/// other arm is unreachable from a `spec` this module builds itself. +/// Read the KLL `k` out of `SketchParams::Kll`; a valid `SketchKind` +/// pairs `SketchAlgorithm::Kll` with no other params. #[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { - // Lossless: `accumulator_spec()` only ever stores a value that - // already fit in `u16` (via `kll_k_param`'s own `u16::try_from` - // fallback) widened to `u32`. SketchParams::Kll { k } => *k as u16, - other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::Kll with non-Kll params: {other:?}" - ), + other => { + unreachable!("SketchKind paired SketchAlgorithm::Kll with non-Kll params: {other:?}") + } } } @@ -61,7 +62,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { (*depth as usize, *width as usize) } other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" + "SketchKind paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" ), } } @@ -81,7 +82,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { heap_size, } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( - "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" + "SketchKind paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), } } @@ -92,7 +93,7 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::DDSketch with non-DDSketch params: {other:?}" + "SketchKind paired SketchAlgorithm::DDSketch with non-DDSketch params: {other:?}" ), } } @@ -101,17 +102,14 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { /// Production execution requires a validated Planner DAG program. #[cfg(test)] pub fn create_fixture_accumulator( - config: &PrecomputeMaterialization, + family: &SummaryFamilyType, + update: &SummaryUpdate, ) -> Box { - let spec = config - .accumulator_spec() - .expect("invalid isolated kernel fixture"); - - let keyed = spec.grouping.is_some(); + let keyed = family_is_keyed(family); - match (&spec.family, keyed) { + match (family, keyed) { (SummaryFamilyType::ExactAggregate(..), keyed) => Box::new(ExactUpdater { - acc: ExactAccumulator::new(spec.family.clone(), keyed).expect("exact fixture family"), + acc: ExactAccumulator::new(family.clone(), keyed).expect("exact fixture family"), }), (SummaryFamilyType::Sketch(kind, _), false) @@ -119,18 +117,20 @@ pub fn create_fixture_accumulator( { Box::new(KllAccumulatorUpdater::new(kll_k(kind.params()))) } - // HydraKLL: `k` comes off the typed params like the unkeyed case, - // but the `(row, col)` tiling grid has no `SketchParams::Kll` - // field to live in (see `asap_types::accumulator_spec`'s module - // doc) — read it the same way bare CMS does, via `cms_params`. - (SummaryFamilyType::Sketch(kind, _), true) if kind.algorithm() == &SketchAlgorithm::Kll => { - let (row_num, col_num) = cms_params(config); - Box::new(HydraKllAccumulatorUpdater::new( - row_num, - col_num, - kll_k(kind.params()), - )) - } + // HydraKLL: `k` comes off the typed params like the unkeyed case; + // the shared bucket count is the tiling width over four rows. + ( + SummaryFamilyType::Sketch( + kind, + GroupingStrategy::SharedMultiSubpopulation { + params: HydraParams::HydraKll { shared_buckets, .. }, + .. + }, + ), + true, + ) if kind.algorithm() == &SketchAlgorithm::Kll => Box::new( + HydraKllAccumulatorUpdater::new(4, *shared_buckets as usize, kll_k(kind.params())), + ), // Bare CMS: point-frequency only, min-of-rows estimator. `keyed=false` // can't actually arise here today (no `AggregationType` resolves to @@ -152,9 +152,8 @@ pub fn create_fixture_accumulator( // Heap-bearing top-k variant (raw-input ingest path): route to the // real `CmsHeapAccumulatorUpdater` so the per-policy top-k heap is // BUILT (heap-less CMS could not answer `topk(...)` — recall 0). - // Keyed by the configured group-by `aggregated_labels` (e.g. `host`), - // ranked by Σ value per key by default (`weight_mode: value`), or Σ - // count for genuine frequency-top-k (`weight_mode: count`). The OTLP + // Keyed by the update's item, ranked by Σ value per key, or Σ count + // when the update weight is the constant 1. The OTLP // modified-sketch path builds the heap agent-side and uses // `SketchEnvelope` ingest, not this raw arm. (SummaryFamilyType::Sketch(kind, _), _) @@ -165,7 +164,7 @@ pub fn create_fixture_accumulator( row_num, col_num, heap_size, - topk_weight_param(config), + topk_weight(update), )) } @@ -178,7 +177,7 @@ pub fn create_fixture_accumulator( row_num, col_num, heap_size, - topk_weight_param(config), + topk_weight(update), )) } @@ -203,7 +202,7 @@ pub fn create_fixture_accumulator( unreachable!("validated UnivMon family parameters") }; create_planner_accumulator( - &spec.family, + family, &planner_types::post_asap::SummaryUpdate::column( planner_types::pre_asap::ColumnRef::SampleValue, ), @@ -219,7 +218,7 @@ pub fn create_fixture_accumulator( unreachable!("validated HLL family parameters") }; create_planner_accumulator( - &spec.family, + family, &planner_types::post_asap::SummaryUpdate::column( planner_types::pre_asap::ColumnRef::SampleValue, ), diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index 21cd2b115..a278e28a8 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,32 +27,17 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::PrecomputeMaterialization; -use asap_types::enums::WindowKind; use asap_types::AggregationType; -use asap_types::KeyByLabelNames; use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - params, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) +/// The state family a kernel fixture with these dimensions stores. +fn cfg( + agg_type: AggregationType, + params: HashMap, +) -> planner_types::post_asap::SummaryFamilyType { + crate::tests::test_utilities::outputs::family(agg_type, &serde_json::to_value(params).unwrap()) } // -------- published-constant parity ---------- diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 62a7621ec..720efc483 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -1,538 +1,158 @@ //! Engine factory helpers for integration tests //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore -//! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with -//! the correct aggregation_type string. +//! populated with stored states of a given kernel family. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; -use crate::query_engines::query_result::InstantVectorElement; +use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::types::{ - AggregationType, InstalledPrecomputePlan, KeyByLabelValues, PrecomputeMaterialization, - PrecomputedOutput, QueryLanguage, WindowKind, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, WindowKind, }; use crate::AggregateCore; -use asap_types::KeyByLabelNames; -use std::collections::HashMap; - -/// Helper for test factories — wraps the closure-mint call with a -/// fresh resolver and forwards to `SketchStore::ingest_precompute_for_agg_config`. -/// Each factory gets its own resolver instance; tests are isolated so -/// the `next_sid = 1, 2, ...` counter doesn't bleed between fixtures. -fn ingest_with_fresh_resolver( - summary_store: &crate::storage_engines::sketch_db::index::SketchStore, - resolver: &std::sync::Arc, - agg_cfg: &PrecomputeMaterialization, - output: &PrecomputedOutput, - accumulator: &dyn AggregateCore, -) -> Option { - let resolver = resolver.clone(); - summary_store.ingest_precompute_for_agg_config( - |m, fp, ak| resolver.resolve(m, fp, ak), - agg_cfg, - output, - accumulator, - ) -} use std::sync::Arc; /// Data to insert into a store: (label_values, accumulator) pub type AccumulatorData = Vec<(Option>, Box)>; -/// Creates a ASAPQueryEngine with a single aggregation populated with given data. -/// -/// # Arguments -/// * `metric` - Metric name -/// * `aggregation_type` - Accumulator type string (e.g. "SumAccumulator", "DatasketchesKLLAccumulator") -/// * `grouping_labels` - Label names for GROUP BY -/// * `data` - Vec of (label_values, accumulator) pairs to insert -/// * `promql_query` - The PromQL query string -pub fn create_engine_single_pop( +/// One stored output of `kind` state over `metric`, with its storage kind. +fn stored_output( metric: &str, - aggregation_type: AggregationType, + kind: AggregationType, grouping_labels: Vec<&str>, - data: AccumulatorData, - promql_query: &str, -) -> ASAPQueryEngine { - create_engine_single_pop_with_aggregated( + window_size: u64, + window_type: WindowKind, +) -> ( + PrecomputeMaterialization, + crate::storage_engines::sketch_db::index::AggKind, +) { + let family = super::outputs::family(kind, &serde_json::json!({})); + let mut config = PrecomputeMaterialization::new( metric, - aggregation_type, - grouping_labels, - vec![], - data, - promql_query, - ) + asap_types::KeyByLabelNames::new(grouping_labels.into_iter().map(str::to_owned).collect()), + window_size, + 1, + window_type, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + config.allocate_stored_output_id(&family); + let agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_family(&family, ""); + (config, agg_kind) } -/// Creates a ASAPQueryEngine with aggregated labels (sub-key labels within the accumulator). -/// -/// Use for self-keyed multi-population accumulators (Multiple* types) where -/// `aggregated_labels` are the labels that key the accumulator internally -/// (e.g. "endpoint" within a MultipleIncrease accumulator). -pub fn create_engine_single_pop_with_aggregated( - metric: &str, - aggregation_type: AggregationType, - grouping_labels: Vec<&str>, - aggregated_labels: Vec<&str>, - data: AccumulatorData, - promql_query: &str, -) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - let aggregated_label_strings: Vec = - aggregated_labels.iter().map(|s| s.to_string()).collect(); - let all_schema_labels: Vec = grouping_label_strings - .iter() - .chain(aggregated_label_strings.iter()) - .cloned() - .collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - - // Insert data into SketchStore via the canonical helper (M2.3.6e). - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg config must be in installed_precompute_plan"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); +/// Ingest `(end_timestamp, labels, state)` rows of one output, each covering +/// `[end - span, end]`, through a fresh resolver. +fn ingest( + store: &SketchStore, + output: &( + PrecomputeMaterialization, + crate::storage_engines::sketch_db::index::AggKind, + ), + span: u64, + rows: impl IntoIterator>, Box)>, +) { + let resolver = Arc::new(SeriesIdResolver::new()); + let (config, kind) = output; + for (timestamp, labels, state) in rows { + let key = labels.map(|labels| KeyByLabelValues { labels }); let output = PrecomputedOutput::new( - timestamp, + timestamp - span, timestamp, key, - asap_types::PolicyFingerprint(agg_id), + config.policy_fingerprint(), + ); + store.ingest_precompute_for_agg_config( + |m, fp, ak| resolver.resolve(m, fp, ak), + config, + kind, + &output, + state.as_ref(), ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } +} - ASAPQueryEngine::new(1).with_sketch_index(summary_store) +fn at_one_second(data: AccumulatorData) -> Vec<(u64, Option>, Box)> { + data.into_iter() + .map(|(labels, state)| (1_000_000, labels, state)) + .collect() } -/// Build a test engine with separate value and key aggregations. Each input -/// carries its own accumulator type, labels, and samples. -#[allow(clippy::too_many_arguments)] -#[allow(dead_code)] -pub fn create_engine_dual_input( +/// Creates a ASAPQueryEngine with a single aggregation populated with given data. +pub fn create_engine_single_pop( metric: &str, - value_agg_type: AggregationType, - key_agg_type: AggregationType, + aggregation_type: AggregationType, grouping_labels: Vec<&str>, - aggregated_labels: Vec<&str>, - value_data: AccumulatorData, - keys_data: AccumulatorData, - promql_query: &str, + data: AccumulatorData, + _promql_query: &str, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - let aggregated_label_strings: Vec = - aggregated_labels.iter().map(|s| s.to_string()).collect(); - let all_labels: Vec = grouping_label_strings - .iter() - .chain(aggregated_label_strings.iter()) - .cloned() - .collect(); - - let mut materializations_by_output = HashMap::new(); - - // Value aggregation - let value_agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: value_agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let value_id = value_agg_config.policy_fp_u64(); - materializations_by_output.insert(value_id, value_agg_config); - - // Keys aggregation - let keys_agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: key_agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let keys_id = keys_agg_config.policy_fp_u64(); - materializations_by_output.insert(keys_id, keys_agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - - let agg_cfg_1 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(value_id)) - .cloned() - .expect("value agg config"); - let agg_cfg_2 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(keys_id)) - .cloned() - .expect("keys agg config"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in value_data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(value_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); - } - for (label_values_opt, acc) in keys_data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(keys_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); - } - - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + let store = Arc::new(SketchStore::new()); + let output = stored_output( + metric, + aggregation_type, + grouping_labels, + 1, + WindowKind::Tumbling, + ); + ingest(&store, &output, 0, at_one_second(data)); + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a ASAPQueryEngine with two independent metrics, each with their own -/// aggregation config and query_config. -/// -/// agg_id=1 → metric_a, agg_id=2 → metric_b. -/// Both are registered as separate query_configs in the inference config. +/// stored output. #[allow(clippy::too_many_arguments)] pub fn create_engine_two_metrics( metric_a: &str, aggregation_type_a: AggregationType, grouping_labels_a: Vec<&str>, data_a: AccumulatorData, - query_a: &str, + _query_a: &str, metric_b: &str, aggregation_type_b: AggregationType, grouping_labels_b: Vec<&str>, data_b: AccumulatorData, - query_b: &str, + _query_b: &str, ) -> ASAPQueryEngine { - let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); - let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - - let agg_config_a = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: aggregation_type_a, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels_a.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric_a.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id_a = agg_config_a.policy_fp_u64(); - materializations_by_output.insert(id_a, agg_config_a); - - let agg_config_b = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: aggregation_type_b, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels_b.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric_b.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id_b = agg_config_b.policy_fp_u64(); - materializations_by_output.insert(id_b, agg_config_b); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(id_a)) - .cloned() - .expect("agg a"); - let agg_cfg_2 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(id_b)) - .cloned() - .expect("agg b"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in data_a { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(id_a), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); - } - for (label_values_opt, acc) in data_b { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(id_b), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); + let store = Arc::new(SketchStore::new()); + for (metric, kind, labels, data) in [ + (metric_a, aggregation_type_a, grouping_labels_a, data_a), + (metric_b, aggregation_type_b, grouping_labels_b, data_b), + ] { + let output = stored_output(metric, kind, labels, 1, WindowKind::Tumbling); + ingest(&store, &output, 0, at_one_second(data)); } - let _ = (query_a, query_b); - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a ASAPQueryEngine with three independent metrics, each with their own -/// aggregation config and query_config. -/// -/// agg_id=1 → metric_a, agg_id=2 → metric_b, agg_id=3 → metric_c. +/// stored output. #[allow(clippy::too_many_arguments)] pub fn create_engine_three_metrics( metric_a: &str, aggregation_type_a: AggregationType, grouping_labels_a: Vec<&str>, data_a: AccumulatorData, - query_a: &str, + _query_a: &str, metric_b: &str, aggregation_type_b: AggregationType, grouping_labels_b: Vec<&str>, data_b: AccumulatorData, - query_b: &str, + _query_b: &str, metric_c: &str, aggregation_type_c: AggregationType, grouping_labels_c: Vec<&str>, data_c: AccumulatorData, - query_c: &str, + _query_c: &str, ) -> ASAPQueryEngine { - let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); - let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - let labels_c: Vec = grouping_labels_c.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let mut ids: Vec = Vec::new(); - - for (agg_type, labels, metric) in [ - (aggregation_type_a, &labels_a, metric_a), - (aggregation_type_b, &labels_b, metric_b), - (aggregation_type_c, &labels_c, metric_c), + let store = Arc::new(SketchStore::new()); + for (metric, kind, labels, data) in [ + (metric_a, aggregation_type_a, grouping_labels_a, data_a), + (metric_b, aggregation_type_b, grouping_labels_b, data_b), + (metric_c, aggregation_type_c, grouping_labels_c, data_c), ] { - let cfg = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id = cfg.policy_fp_u64(); - ids.push(id); - materializations_by_output.insert(id, cfg); + let output = stored_output(metric, kind, labels, 1, WindowKind::Tumbling); + ingest(&store, &output, 0, at_one_second(data)); } - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfgs: Vec<_> = ids - .iter() - .map(|id| { - installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(*id)) - .cloned() - .expect("agg present") - }) - .collect(); - let timestamp = 1_000_000_u64; - for (idx, data) in [(0, data_a), (1, data_b), (2, data_c)] { - let agg_cfg = &agg_cfgs[idx]; - let agg_id = ids[idx]; - for (label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, agg_cfg, &output, acc.as_ref()); - } - } - - let _ = (labels_a, labels_b, labels_c, query_a, query_b, query_c); - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a single-pop engine with data at multiple timestamps for testing merge. @@ -544,76 +164,21 @@ pub fn create_engine_multi_timestamp( data: Vec<(u64, Option>, Box)>, promql_query: &str, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + create_engine_multi_timestamp_with_window( + metric, aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg"); - for (timestamp, label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp - 1000, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); - } - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + grouping_labels, + data, + promql_query, + 1, + WindowKind::Tumbling, + ) } /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the stored output (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -621,72 +186,18 @@ pub fn create_engine_multi_timestamp_with_window( aggregation_type: AggregationType, grouping_labels: Vec<&str>, data: Vec<(u64, Option>, Box)>, - promql_query: &str, + _promql_query: &str, window_size: u64, window_type: WindowKind, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + let store = Arc::new(SketchStore::new()); + let output = stored_output( + metric, aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), + grouping_labels, window_size, - slide_interval: 1, window_type, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg"); - for (timestamp, label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp - 1000, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); - } - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ); + ingest(&store, &output, 1000, data); + ASAPQueryEngine::new(1).with_sketch_index(store) } diff --git a/data_plane/src/tests/test_utilities/mod.rs b/data_plane/src/tests/test_utilities/mod.rs index 824a72884..2dffa3274 100644 --- a/data_plane/src/tests/test_utilities/mod.rs +++ b/data_plane/src/tests/test_utilities/mod.rs @@ -6,3 +6,5 @@ pub mod timing; pub use engine_factories::*; pub mod planning; + +pub mod outputs; diff --git a/data_plane/src/tests/test_utilities/outputs.rs b/data_plane/src/tests/test_utilities/outputs.rs new file mode 100644 index 000000000..917b7e36a --- /dev/null +++ b/data_plane/src/tests/test_utilities/outputs.rs @@ -0,0 +1,125 @@ +//! Stored-output fixtures: deployment fields paired with the stored state +//! family an installed plan's schema contract would carry. +use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; +use planner_types::post_asap::{ + GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, SketchParams, + SummaryFamilyType, +}; +use serde_json::Value; + +/// A tumbling output over `metric` grouped by `grouping`, with its id +/// allocated from `family`. +pub fn output( + metric: &str, + family: SummaryFamilyType, + grouping: Vec<&str>, + window: u64, +) -> (PrecomputeMaterialization, SummaryFamilyType) { + let config = PrecomputeMaterialization::new( + metric, + KeyByLabelNames::new(grouping.into_iter().map(str::to_owned).collect()), + window, + window, + WindowKind::Tumbling, + ); + allocated(config, family) +} + +/// Allocate `config`'s id from `family` after its deployment fields are final. +pub fn allocated( + mut config: PrecomputeMaterialization, + family: SummaryFamilyType, +) -> (PrecomputeMaterialization, SummaryFamilyType) { + config.allocate_stored_output_id(&family); + (config, family) +} + +/// The Planner family of a kernel fixture spelled as a backend kernel tag and +/// its sketch dimensions, with the kernels' default dimensions. +pub fn family(kind: AggregationType, parameters: &Value) -> SummaryFamilyType { + let u32_parameter = |names: &[&str], default: u32| { + names + .iter() + .find_map(|name| parameters.get(*name).and_then(Value::as_u64)) + .map_or(default, |value| value as u32) + }; + let sketch = |algorithm, params| { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let (width, depth) = (u32_parameter(&["w"], 1000), u32_parameter(&["d"], 4)); + let heap_size = u32_parameter(&["heap_size", "k", "K"], 20); + match kind { + AggregationType::DatasketchesKLL => sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: u32_parameter(&["K", "k"], 200), + }, + ), + AggregationType::HydraKLL => { + let k = u32_parameter(&["K", "k"], 200); + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraKll, + params: HydraParams::HydraKll { + k, + shared_buckets: k, + }, + }, + ) + } + AggregationType::CountMinSketch => { + sketch(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }) + } + AggregationType::CountMinSketchWithHeap => sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + ), + AggregationType::CountSketch => sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width, depth }, + ), + AggregationType::CountSketchWithHeap => sketch( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + ), + AggregationType::DDSketch => sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { + alpha: ["relativeAccuracy", "relative_accuracy", "alpha"] + .iter() + .find_map(|name| parameters.get(*name).and_then(Value::as_f64)) + .unwrap_or(0.01), + }, + ), + AggregationType::HLL => sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: u32_parameter(&["precision", "p"], 14) as u8, + }, + ), + AggregationType::UnivMon => sketch( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: u32_parameter(&["heap_size"], 32), + sketch_rows: u32_parameter(&["sketch_rows"], 5), + sketch_cols: u32_parameter(&["sketch_cols"], 1024), + layers: u32_parameter(&["layers"], 4) as u8, + }, + ), + exact => exact + .planner_exact_family() + .unwrap_or_else(|| panic!("{exact:?} names no Planner family")), + } +} diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index 4340e6154..41619306c 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -107,7 +107,12 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) -> Backend { +async fn start_backend( + materialization: &( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ), +) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); @@ -124,11 +129,12 @@ async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) .find(|s| s.materialization == rule.materialization) .unwrap() .family, - control_plane::physical::compiler::StateFamilyContract::Sketch { - algorithm: planner_types::post_asap::SketchAlgorithm::Cms - | planner_types::post_asap::SketchAlgorithm::CountSketch, - .. - } + planner_types::post_asap::SummaryFamilyType::Sketch(ref kind, _) + if matches!( + kind.algorithm(), + planner_types::post_asap::SketchAlgorithm::Cms + | planner_types::post_asap::SketchAlgorithm::CountSketch + ) ) { rule.encoding = control_plane::physical::compiler::StateEncoding::SketchCoreMsgpackV1; } @@ -281,11 +287,18 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { .collect() } -fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { +fn config( + metric: &str, + kind: &str, + parameters: &str, +) -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + let parameters: serde_json::Value = serde_yaml::from_str(parameters).unwrap(); physical_fixture::materialization( metric, - kind.parse().unwrap(), - serde_yaml::from_str(parameters).unwrap(), + physical_fixture::sketch_family(kind.parse().unwrap(), ¶meters), ) } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 8e231adfd..05cdb4511 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -379,9 +379,16 @@ async fn certified_kll_state_to_query_oracle() { .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); - let k = plan.precompute_plan.materializations[0].parameters["k"] - .as_u64() - .unwrap() as u32; + let family = plan + .precompute_plan + .state_family(plan.precompute_plan.materializations[0].stored_output_id) + .unwrap(); + let planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) = family else { + panic!("quantile output stores a sketch") + }; + let planner_types::post_asap::SketchParams::Kll { k } = *kind.params() else { + panic!("quantile output stores a KLL") + }; let guarantee = asap_aware_mapping::DefaultAccuracyModel::sketch_guarantee( &planner_types::post_asap::SketchAlgorithm::Kll, &planner_types::post_asap::SketchParams::Kll { k }, @@ -769,14 +776,24 @@ async fn registered_temporal_topk( } _ => panic!("fixture requires a heap implementation"), }; + let output = plan.precompute_plan.materializations[0].stored_output_id; assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - expected_type - ); - assert_eq!( - plan.precompute_plan.materializations[0].parameters["weight_mode"], - "count" + plan.precompute_plan + .state_family(output) + .and_then(asap_types::aggregation_type_for_family), + Some(expected_type) ); + // The Planner update counts occurrences. + let (producer, _) = plan + .precompute_plan + .summary_producer(output) + .unwrap() + .unwrap(); + assert!(matches!( + producer.payload, + planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { ref input, .. } + if input.weight == planner_types::post_asap::SummaryInputExpr::Constant(1.0) + )); let artifact = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -1072,7 +1089,15 @@ async fn run_shared_dashboard(multi_pane: bool) { .precompute_plan .materializations .iter() - .map(|m| m.aggregation_type.as_str()) + .map(|m| { + asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(m.stored_output_id) + .unwrap(), + ) + .unwrap() + .as_str() + }) .collect::>(); assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 8a6d59269..198e88259 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -127,8 +127,10 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let install = physical_fixture::artifact_from_materializations(vec![physical_fixture::materialization( "component_process_e2e_latency_ms", - asap_types::AggregationType::DDSketch, - [("relative_accuracy".into(), serde_json::json!(0.01))].into(), + physical_fixture::sketch_family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"relative_accuracy": 0.01}), + ), )]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index ecc9d51aa..ce8e22b59 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -45,6 +45,7 @@ //! metric. use asap_types::PrecomputeMaterialization; +use planner_types::post_asap::SummaryFamilyType; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -72,23 +73,27 @@ fn phase_aligned_now_ns() -> u64 { now - now % 5_000_000_000 + 3_000_000_000 } +/// A stored output with its state family. +type Output = (PrecomputeMaterialization, SummaryFamilyType); + async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[PrecomputeMaterialization], + materializations: &[Output], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the // physical layout to match them. - for config in &mut configs { + for (config, family) in &mut configs { config.window_size = 1; config.slide_interval = 1; config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + config.allocate_stored_output_id(family); } let mut artifact = physical_fixture::artifact_from_materializations(configs.clone()); if configs .iter() - .any(|c| c.metric == "http_requests_total_latency_ms") + .any(|(c, _)| c.metric == "http_requests_total_latency_ms") { for rule in &mut artifact.transmission_plan.rules { rule.mode = asap_types::producer_plan::TransmissionMode::Delta; @@ -170,7 +175,7 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( @@ -204,32 +209,39 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec Vec { - vec![PrecomputeMaterialization::new( - asap_types::AggregationType::CountMinSketch, - String::new(), - std::collections::HashMap::from([ - ("w".into(), serde_json::json!(512)), - ("d".into(), serde_json::json!(5)), - ]), +fn imported_cms_materializations(metric: &str) -> Vec { + let family = SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + planner_types::post_asap::SketchAlgorithm::Cms, + planner_types::post_asap::SketchParams::Cms { + width: 512, + depth: 5, + }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let mut config = PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(vec!["service".into()]), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 5, 5, asap_types::enums::WindowKind::Tumbling, - String::new(), - metric.into(), - Some(12), - None, - None, - )] + ); + config.num_aggregates_to_retain = Some(12); + config.allocate_stored_output_id(&family); + vec![(config, family)] } /// Epsilon-delta accuracy target in the shape `QueryRequirements` expects. @@ -654,7 +666,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { 1, "expected exactly one materialization: {materializations:#?}" ); - let agg = &materializations[0]; + let agg = &materializations[0].0; assert_eq!(agg.metric, "http_latency_ms"); assert!(agg.window_size > 0, "window size must be > 0: {agg:#?}"); @@ -681,7 +693,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // The planner must thread the query's grouping into the materialization // the backend keys its per-population state by. - let agg = &materializations[0]; + let agg = &materializations[0].0; assert!( agg.grouping_labels .names() @@ -698,6 +710,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // endpoint that reports only plan phase. assert!( materializations[0] + .0 .grouping_labels .names() .iter() @@ -905,9 +918,12 @@ async fn controller_plan_to_query_full_roundtrip_kll() { ); post_full_config(&client, &stack, &materializations).await; - let alpha = materializations[0].parameters["alpha"] - .as_f64() - .expect("planner sized a relative-accuracy quantile summary"); + let SummaryFamilyType::Sketch(kind, _) = &materializations[0].1 else { + panic!("planner selected a quantile sketch") + }; + let planner_types::post_asap::SketchParams::DDSketch { alpha } = kind.params().clone() else { + panic!("planner sized a relative-accuracy quantile summary") + }; let dd_state = build_dd_sketch_state(alpha, vec![5u64, 10, 15, 20], -1); let sketch_bytes = encode_dd_full_state(dd_state); @@ -1230,14 +1246,14 @@ async fn imported_cms_frequency_wire_roundtrip() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { - let w = agg.parameters["w"] - .as_u64() - .expect("materialization must carry parameters.w") as u32; - let d = agg.parameters["d"] - .as_u64() - .expect("materialization must carry parameters.d") as u32; - (w, d) +fn extract_w_d(agg: &Output) -> (u32, u32) { + let SummaryFamilyType::Sketch(kind, _) = &agg.1 else { + panic!("materialization stores a sketch") + }; + let planner_types::post_asap::SketchParams::Cms { width, depth } = kind.params() else { + panic!("materialization stores a CMS") + }; + (*width, *depth) } // Heap TopK serving acceptance lives in asapquery_compatibility_process_e2e: diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index a3c6359c5..50a4a18d2 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -335,24 +335,16 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { } } -fn ddsketch_config() -> asap_types::PrecomputeMaterialization { - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - PrecomputeMaterialization::new( - AggregationType::DDSketch, - String::new(), - std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(ALPHA))]), - KeyByLabelNames::new(vec!["service".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 1, - 1, - WindowKind::Tumbling, - String::new(), - METRIC.into(), - None, - None, - None, +fn ddsketch_config() -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + physical_fixture::materialization( + METRIC, + physical_fixture::sketch_family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"relative_accuracy": ALPHA}), + ), ) } @@ -362,9 +354,9 @@ async fn bound_sds_keeps_hot_and_rebuild_outputs_and_groups_isolated() { use asap_types::query_plan::QueryPlanNode; use asap_types::sds::StoredOutputId; let mut hot = ddsketch_config(); - hot.stored_output_id = Some(StoredOutputId(1001)); + hot.0.stored_output_id = StoredOutputId(1001); let mut rebuild = hot.clone(); - rebuild.stored_output_id = Some(StoredOutputId(1002)); + rebuild.0.stored_output_id = StoredOutputId(1002); let mut install = physical_fixture::artifact_from_materializations(vec![hot, rebuild]); assert_eq!(install.summary_catalog.definitions.len(), 1); assert_eq!(install.summary_catalog.outputs.len(), 2); diff --git a/data_plane/tests/support/empty_physical_plan.rs b/data_plane/tests/support/empty_physical_plan.rs index 0e453d85b..339bd56a4 100644 --- a/data_plane/tests/support/empty_physical_plan.rs +++ b/data_plane/tests/support/empty_physical_plan.rs @@ -12,7 +12,7 @@ pub fn empty() -> asap_types::plan_publication::PhysicalPlanInstallRequest { capability_snapshot_id: "empty-bootstrap".into(), }; let catalog = - control_plane::physical::summary_catalog::SummaryCatalog::from_materializations(0, 0, &[]) + control_plane::physical::summary_catalog::SummaryCatalog::from_outputs(0, 0, Vec::new()) .unwrap(); let mut precompute_plan = PrecomputePlan::build(envelope.clone(), vec![], &[]).unwrap(); precompute_plan.summary_catalog = Some(catalog.reference().unwrap()); diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index 1aa1fefc5..d6b3d516e 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -66,10 +66,17 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { ); // The production cost model may choose DDSketch or KLL. Preserve that // choice and use its actual value contract for this singleton oracle. - let max_relative_error = match derived.aggregation_type { - asap_types::AggregationType::DDSketch => derived.parameters["alpha"].as_f64().unwrap(), - asap_types::AggregationType::DatasketchesKLL => 0.0, - ref other => panic!("singleton quantile oracle missing for {other:?}"), + let family = plan + .precompute_plan + .state_family(derived.stored_output_id) + .unwrap(); + let max_relative_error = match family { + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::DDSketch { alpha } => *alpha, + planner_types::post_asap::SketchParams::Kll { .. } => 0.0, + other => panic!("singleton oracle missing for {other:?}"), + }, + other => panic!("singleton quantile oracle missing for {other:?}"), }; eprintln!( "IMMUTABLE_SELECTED {}", diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index e86a100e9..80d6c6d39 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -8,40 +8,97 @@ use data_plane::{ }; use std::{collections::BTreeMap, sync::Arc}; -/// Imported-state fixtures bind explicit storage metadata, never a flat runtime config. +/// An imported-state output with its stored state family. Imported-state +/// fixtures bind explicit storage metadata, never a flat runtime config. #[allow(dead_code)] pub fn materialization( metric: &str, - family: asap_types::AggregationType, - parameters: std::collections::HashMap, -) -> asap_types::PrecomputeMaterialization { - asap_types::PrecomputeMaterialization::new( - family, - String::new(), - parameters, + family: planner_types::post_asap::SummaryFamilyType, +) -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + let mut config = asap_types::PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(vec!["service".into()]), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 1, 1, asap_types::enums::WindowKind::Tumbling, - String::new(), - metric.into(), - None, - None, - None, + ); + config.allocate_stored_output_id(&family); + (config, family) +} + +/// The sketch family an imported-state fixture's collector produces, from +/// the sketch kind and the dimensions it was spelled with. +#[allow(dead_code)] +pub fn sketch_family( + kind: asap_types::AggregationType, + parameters: &serde_json::Value, +) -> planner_types::post_asap::SummaryFamilyType { + use asap_types::AggregationType as A; + use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + let get = |names: &[&str]| { + names + .iter() + .find_map(|name| parameters.get(*name).and_then(serde_json::Value::as_f64)) + }; + let (width, depth) = ( + get(&["w"]).unwrap_or(1000.0) as u32, + get(&["d"]).unwrap_or(4.0) as u32, + ); + let (algorithm, params) = match kind { + A::DatasketchesKLL => ( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: get(&["k", "K"]).unwrap_or(200.0) as u32, + }, + ), + A::HLL => ( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: get(&["precision", "p"]).unwrap_or(14.0) as u8, + }, + ), + A::CountMinSketch => (SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), + A::CountSketch => ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width, depth }, + ), + A::DDSketch => ( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { + alpha: get(&["alpha", "relative_accuracy"]).unwrap_or(0.01), + }, + ), + other => panic!("no imported-state fixture for {other:?}"), + }; + planner_types::post_asap::SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + Default::default(), ) } pub fn artifact(config: &InstalledPrecomputePlan) -> PhysicalPlanInstallRequest { - artifact_from_materializations(config.materializations().values().cloned().collect()) + artifact_from_materializations( + config + .materializations() + .values() + .map(|c| { + let family = config.state_family(c.stored_output_id).unwrap().clone(); + (c.clone(), family) + }) + .collect(), + ) } /// Same as [`artifact`], but from materializations the planner produced /// directly — no legacy `InstalledPrecomputePlan` document in between. pub fn artifact_from_materializations( - mut configs: Vec, + mut configs: Vec<( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + )>, ) -> PhysicalPlanInstallRequest { let envelope = PlanEnvelope { plan_id: 1, @@ -56,7 +113,7 @@ pub fn artifact_from_materializations( // Installed physical plans always carry an explicit pane phase. Inputs that // predate that contract bind to the Unix epoch grid before deriving catalog // identities and bindings. - for config in &mut configs { + for (config, _) in &mut configs { config.pane_origin_ms.get_or_insert(0); // These fixtures import synthetic states and replace the producer's // window layout. They do not install the original Planner DAG, so @@ -68,12 +125,10 @@ pub fn artifact_from_materializations( ); config.semantic_fragment = None; } - let catalog = control_plane::physical::summary_catalog::SummaryCatalog::from_materializations( - 1, 1, &configs, - ) - .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); + let catalog = + control_plane::physical::summary_catalog::SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, @@ -93,7 +148,8 @@ pub fn artifact_from_materializations( use data_plane::storage_engines::types::AggregationType; let metric = &config.metric; let mut queries = Vec::new(); - match config.aggregation_type { + let family = precompute.state_family(config.stored_output_id).unwrap(); + match asap_types::aggregation_type_for_family(family).unwrap() { AggregationType::DatasketchesKLL | AggregationType::DDSketch => { for (window, ms) in [ ("1s", 1_000), @@ -165,7 +221,7 @@ pub fn artifact_from_materializations( config.policy_fingerprint().into(), ), output_grouping, - item_labels: config.aggregated_labels.labels.clone(), + item_labels: Vec::new(), window_ms: config.slide_interval * 1000, pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(lookback_ms), diff --git a/data_plane/tests/support/revisable_maintenance_process.rs b/data_plane/tests/support/revisable_maintenance_process.rs index eac0f8ca9..fef0e85d2 100644 --- a/data_plane/tests/support/revisable_maintenance_process.rs +++ b/data_plane/tests/support/revisable_maintenance_process.rs @@ -58,10 +58,17 @@ async fn run_revisions(multi_source: bool) { .iter() .find(|m| m.derived_input.is_some()) .unwrap(); - let relative_error = match derived.aggregation_type { - asap_types::AggregationType::DDSketch => derived.parameters["alpha"].as_f64().unwrap(), - asap_types::AggregationType::DatasketchesKLL => 0.0, - ref other => panic!("singleton revision oracle missing for {other:?}"), + let family = installed + .precompute_plan + .state_family(derived.stored_output_id) + .unwrap(); + let relative_error = match family { + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::DDSketch { alpha } => *alpha, + planner_types::post_asap::SketchParams::Kll { .. } => 0.0, + other => panic!("singleton oracle missing for {other:?}"), + }, + other => panic!("singleton revision oracle missing for {other:?}"), }; std::fs::write(&artifact, serde_json::to_vec(&installed).unwrap()).unwrap(); let spawn = |port: u16| { diff --git a/docs/examples/asapquery-compatibility-demo-snapshot.json b/docs/examples/asapquery-compatibility-demo-snapshot.json index c24e976c4..ed36d55f3 100644 --- a/docs/examples/asapquery-compatibility-demo-snapshot.json +++ b/docs/examples/asapquery-compatibility-demo-snapshot.json @@ -120,6 +120,6 @@ "plan_version": 1, "activation_unix_ms": 10000, "expiry_unix_ms": null, - "backend_compat": "asap-query-backend.v1" + "backend_compat": "asap-query-backend.v2" } } diff --git a/docs/examples/asapquery-planning-snapshot.json b/docs/examples/asapquery-planning-snapshot.json index 736857c49..655aed5a1 100644 --- a/docs/examples/asapquery-planning-snapshot.json +++ b/docs/examples/asapquery-planning-snapshot.json @@ -71,6 +71,6 @@ "plan_version": 1, "activation_unix_ms": 10000, "expiry_unix_ms": null, - "backend_compat": "asap-query-backend.v1" + "backend_compat": "asap-query-backend.v2" } }