From 2eded4fb5408598c17a209e5df8d0284203cba0e Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 13:38:32 +0000 Subject: [PATCH 1/3] refactor!: keep only deployment fields on precompute materializations An installed PrecomputeMaterialization now carries its stored output id, source binding, grouping layout, cadence and window layout, retention and semantic fragment. What an output computes is defined only by the Planner DAG node bound to it. Removed: aggregation_type, aggregation_sub_type, parameters, spatial_filter, spatial_filter_normalized, aggregated_labels, rollup_labels, original_yaml, and the derived accumulator_spec() and sample_update_rule() projections. - The stored state family lives in the existing StateSchemaContract (now the lossless Planner SummaryFamilyType); validation requires it to equal the family of any SummaryAgg node that produces the output. - Input predicates, item labels and the PromQL right-closed pane rule are read from the bound DAG producer; the catalog's data descriptors use the same predicate. - The compiler allocates stored_output_id explicitly from the deployment fields and the producer's computation identity. - PolicyRegistry and RoutingIndex are removed; OTLP content matching and the sketch sink read the installed plan directly. - BACKEND_COMPAT is asap-query-backend.v2 and SummaryCatalog schema 7; older plans are rejected on decode by version. Co-Authored-By: Claude Opus 5.5 --- .../examples/audit_clickhouse_corpus.rs | 27 +- control_plane/src/clickhouse.rs | 330 +++----- control_plane/src/opamp/mod.rs | 4 +- control_plane/src/physical/compiler.rs | 399 +++++---- control_plane/src/physical/erp.rs | 38 +- control_plane/src/physical/pane_reuse.rs | 17 +- control_plane/src/query_plan.rs | 46 +- crates/asap_types/src/accumulator_spec.rs | 795 +----------------- crates/asap_types/src/aggregation_config.rs | 669 +++++---------- crates/asap_types/src/aggregation_type.rs | 34 + crates/asap_types/src/derived_input.rs | 76 +- crates/asap_types/src/lib.rs | 7 +- crates/asap_types/src/policy_fingerprint.rs | 473 ++--------- crates/asap_types/src/policy_registry.rs | 169 ---- crates/asap_types/src/precompute_plan.rs | 457 +++++++--- .../asap_types/src/precompute_plan/catalog.rs | 28 +- crates/asap_types/src/producer_plan.rs | 4 +- crates/asap_types/src/routing_index.rs | 270 ------ crates/asap_types/src/sds.rs | 236 ++---- crates/asap_types/src/summary_catalog.rs | 195 +++-- crates/asap_types/src/summary_semantics.rs | 8 - data_plane/benches/sketch_db.rs | 22 +- .../examples/audit_clickhouse_fallback.rs | 2 +- data_plane/src/drivers/ingest/otel.rs | 257 ++---- .../drivers/ingest/prometheus_remote_write.rs | 192 ++--- data_plane/src/drivers/query/servers/http.rs | 81 +- .../src/precompute_engine/erp_observer.rs | 55 +- .../src/precompute_engine/frame_lineage.rs | 2 +- .../src/precompute_engine/ingest_handler.rs | 37 +- .../precompute_engine/maintenance_runtime.rs | 126 ++- .../src/precompute_engine/output_sink.rs | 105 +-- data_plane/src/precompute_engine/raw_dag.rs | 52 +- data_plane/src/precompute_engine/revisions.rs | 16 +- data_plane/src/precompute_engine/worker.rs | 660 +++------------ .../accelerator.rs | 43 +- .../asap_clickhouse_query_engine/execution.rs | 4 +- .../asap_query_engine/catalog_resolver.rs | 18 +- .../asap_query_engine/post_asap_readout.rs | 33 +- .../asap_query_engine/test_plan.rs | 57 +- .../src/storage_engines/sketch_db/accuracy.rs | 663 +++------------ .../sketch_db/backfill/clickhouse_reader.rs | 18 +- .../sketch_db/backfill/processor.rs | 63 +- .../sketch_db/backfill/service.rs | 25 +- .../sketch_db/backfill/window_builder.rs | 62 +- .../src/storage_engines/sketch_db/data/mod.rs | 208 ++--- .../sketch_db/index/maintenance.rs | 129 ++- .../storage_engines/sketch_db/index/mod.rs | 205 +++-- .../storage_engines/sketch_db/index/native.rs | 49 +- .../sketch_db/lifecycle/eviction.rs | 62 +- .../sketch_db/lifecycle/reconcile.rs | 53 +- .../src/storage_engines/sketch_db/sds.rs | 26 +- .../types/hot_reload_config.rs | 39 +- .../types/installed_precompute_plan.rs | 192 ++++- data_plane/src/tests/accumulator_fixture.rs | 97 ++- .../accuracy_empirical_validation_tests.rs | 27 +- .../tests/test_utilities/engine_factories.rs | 703 +++------------- data_plane/src/tests/test_utilities/mod.rs | 2 + .../src/tests/test_utilities/outputs.rs | 125 +++ .../tests/all_sketches_process_oracle_e2e.rs | 31 +- .../asapquery_compatibility_process_e2e.rs | 45 +- data_plane/tests/component_process_e2e.rs | 6 +- ...e2e_controller_plans_and_backend_serves.rs | 86 +- .../tests/promql_differential_process_e2e.rs | 32 +- .../tests/support/empty_physical_plan.rs | 2 +- .../support/immutable_maintenance_process.rs | 15 +- data_plane/tests/support/physical_fixture.rs | 106 ++- .../support/revisable_maintenance_process.rs | 15 +- ...asapquery-compatibility-demo-snapshot.json | 2 +- .../examples/asapquery-planning-snapshot.json | 2 +- 69 files changed, 3255 insertions(+), 5879 deletions(-) delete mode 100644 crates/asap_types/src/policy_registry.rs delete mode 100644 crates/asap_types/src/routing_index.rs create mode 100644 data_plane/src/tests/test_utilities/outputs.rs diff --git a/control_plane/examples/audit_clickhouse_corpus.rs b/control_plane/examples/audit_clickhouse_corpus.rs index d7488a927..557f198f5 100644 --- a/control_plane/examples/audit_clickhouse_corpus.rs +++ b/control_plane/examples/audit_clickhouse_corpus.rs @@ -25,24 +25,20 @@ struct Row { } fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { + let family = AggregationType::Max.planner_exact_family().unwrap(); let mut materialization = PrecomputeMaterialization::new( - AggregationType::Max, - String::new(), - std::collections::HashMap::from([("variant".into(), json!(2))]), + "raw_samples.value", KeyByLabelNames::new(vec!["labels".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - "raw_samples.value".into(), - None, - Some("raw_samples".into()), - Some("value".into()), ); + materialization.table_name = Some("raw_samples".into()); + materialization.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); materialization.pane_origin_ms = Some(0); + materialization.allocate_stored_output_id(&family); let envelope = PlanEnvelope { plan_id: 27, plan_version: 1, @@ -54,7 +50,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { capability_snapshot_id: "sql27-main-eval".into(), }; let mut precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![materialization.clone()]) + PrecomputePlan::build_backend_local(envelope.clone(), vec![(materialization, family)]) .unwrap(); let mut transmission_plan = control_plane::physical::compiler::build_transmission_plan( envelope, @@ -62,12 +58,7 @@ fn publication_inputs(schema: &Schema, sql: String) -> ClickHouseSqlWorkload { &Default::default(), ) .unwrap(); - let sds = asap_types::summary_catalog::SummaryCatalog::from_materializations( - 27, - 1, - &[materialization], - ) - .unwrap(); + let sds = asap_types::summary_catalog::SummaryCatalog::from_plan(&precompute_plan).unwrap(); let reference = sds.reference().unwrap(); precompute_plan.bind_catalog(&sds).unwrap(); transmission_plan.summary_catalog = Some(reference); diff --git a/control_plane/src/clickhouse.rs b/control_plane/src/clickhouse.rs index dc421e778..289dc9fe8 100644 --- a/control_plane/src/clickhouse.rs +++ b/control_plane/src/clickhouse.rs @@ -260,7 +260,7 @@ pub async fn compile_automatic_clickhouse_workload( let selection_trace = std::mem::take(&mut planned.selection_trace); let template = planned.canonical_sql.clone(); let (entry, installed) = compile_selected_sql(query, planned, |node, family| { - let config = materialize_selected_sql(node, family, query) + let (config, family) = materialize_selected_sql(node, family, query) .map_err(crate::query_plan::QueryPlanError::Invalid)?; let binding = MaterializationBinding { full_window_slide_ms: matches!( @@ -276,11 +276,12 @@ pub async fn compile_automatic_clickhouse_workload( window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(query.end_ms - query.start_ms), - item_labels: config.aggregated_labels.labels.clone(), + // SQL table summaries project no per-item dimension. + item_labels: Vec::new(), }; materializations .entry(config.policy_fingerprint()) - .or_insert(config); + .or_insert((config, family)); Ok(binding) })?; index_sql_template(&mut window_templates, template, &entry); @@ -300,15 +301,11 @@ pub async fn compile_automatic_clickhouse_workload( ); } let configs: Vec<_> = materializations.into_values().collect(); - let sds = SummaryCatalog::from_materializations( - request.envelope.plan_id, - request.envelope.plan_version, - &configs, - ) - .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; let mut precompute = PrecomputePlan::build_backend_local(request.envelope.clone(), configs) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; precompute.executable_dags = installed_dags; + let sds = SummaryCatalog::from_plan(&precompute) + .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; precompute .bind_catalog(&sds) .map_err(|error| ClickHousePlanningError::Lower(error.to_string()))?; @@ -350,7 +347,13 @@ fn materialize_selected_sql( node: &planner_types::post_asap::SummaryNode, family: &planner_types::post_asap::SummaryFamilyType, query: &ClickHouseSqlWorkloadEntry, -) -> Result { +) -> Result< + ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ), + String, +> { use crate::physical::backend_stage::{AggregationInput, BackendAggregation}; use planner_types::{post_asap::SummaryExpr, pre_asap::Reduction}; let SummaryExpr::SummaryAgg { @@ -401,11 +404,12 @@ fn materialize_selected_sql( heap_update_mode: None, aggregation_input: AggregationInput::Raw, }; - let mut config = crate::physical::compiler::aggregation_config_for_materialization( - &aggregation, - asap_types::QueryLanguage::ClickHouseSql, - ) - .map_err(|error| error.to_string())?; + let (mut config, computation) = + crate::physical::compiler::aggregation_config_for_materialization( + &aggregation, + asap_types::QueryLanguage::ClickHouseSql, + ) + .map_err(|error| error.to_string())?; config.table_name = Some(table); config.grouping_labels = grouping; config.value_projection = Some(value); @@ -418,7 +422,8 @@ fn materialize_selected_sql( .map_err(|_| "SQL evaluation timestamp exceeds runtime range")?, ); config.num_aggregates_to_retain = Some(2); - Ok(config) + config.allocate_stored_output_id(&computation); + Ok((config, computation.0)) } pub async fn compile_clickhouse_workload( @@ -620,7 +625,7 @@ fn bind_selected_node( .map_err(crate::query_plan::QueryPlanError::Invalid)?; let expected = family.clone(); let selected = select_materialization( - &request.precompute_plan.materializations, + &request.precompute_plan, &table_ref, &value_column, &spatial_filter.canonical(), @@ -646,7 +651,7 @@ fn bind_selected_node( window_ms: selected.stored_window_ms(), pane_origin_ms: selected.pane_origin_ms, readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1000)), - item_labels: selected.aggregated_labels.labels.clone(), + item_labels: Vec::new(), }) } @@ -920,21 +925,18 @@ fn clickhouse_materialization_leaf_contract( } fn select_materialization<'a>( - materializations: &'a [asap_types::PrecomputeMaterialization], + plan: &'a asap_types::precompute_plan::PrecomputePlan, table_ref: &str, value_projection: &asap_types::sds::ValueProjectionIdentity, spatial_filter: &str, expected: &planner_types::post_asap::SummaryFamilyType, semantic_window_seconds: u64, ) -> Result<&'a asap_types::PrecomputeMaterialization, crate::query_plan::QueryPlanError> { - let mut matches = materializations.iter().filter(|candidate| { + let mut matches = plan.materializations.iter().filter(|candidate| { candidate.table_name.as_deref() == Some(table_ref) && candidate.effective_value_projection() == value_projection - && candidate.population_filter_canonical().ok().as_deref() == Some(spatial_filter) - && candidate - .accumulator_spec() - .ok() - .is_some_and(|spec| spec.family == *expected) + && candidate.table_population_canonical().ok().as_deref() == Some(spatial_filter) + && plan.state_family(candidate.stored_output_id) == Some(expected) && semantic_window_seconds .checked_mul(1000) .is_some_and(|window| window % candidate.window_size.saturating_mul(1000) == 0) @@ -956,6 +958,7 @@ fn select_materialization<'a>( mod tests { use super::*; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::SummaryFamilyType; use planner_types::pre_asap::{Column, DataType, Schema}; // A dashboard refresh must reuse the installed identity without treating @@ -1042,33 +1045,54 @@ mod tests { } } + fn exact(agg: AggregationType) -> SummaryFamilyType { + agg.planner_exact_family().unwrap() + } + + fn ddsketch(alpha: f64) -> SummaryFamilyType { + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + }; + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::DDSketch, SketchParams::DDSketch { alpha }), + GroupingStrategy::PerSubpopulationInstance, + ) + } + fn materialization( - agg: AggregationType, + family: SummaryFamilyType, value_column: &str, window: u64, slide: u64, - parameter: (&str, serde_json::Value), - ) -> PrecomputeMaterialization { + ) -> (PrecomputeMaterialization, SummaryFamilyType) { let mut value = PrecomputeMaterialization::new( - agg, - String::new(), - std::collections::HashMap::from([(parameter.0.into(), parameter.1)]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + format!("telemetry.{value_column}"), KeyByLabelNames::empty(), - String::new(), window, slide, WindowKind::Tumbling, - String::new(), - format!("telemetry.{value_column}"), - None, - Some("telemetry".into()), - Some(value_column.into()), ); + value.table_name = Some("telemetry".into()); + value.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: value_column.into(), + }); value.pane_origin_ms = Some(0); value.table_timestamp_column = Some("timestamp_ms".into()); - value + value.allocate_stored_output_id(&family); + (value, family) + } + + fn envelope(plan_id: u64) -> crate::physical::compiler::PlanEnvelope { + crate::physical::compiler::PlanEnvelope { + plan_id, + plan_version: 1, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: crate::physical::compiler::BACKEND_COMPAT.into(), + planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), + capability_snapshot_id: "clickhouse-test".into(), + } } /// A table leaf whose value column has the given producer typing. @@ -1108,16 +1132,7 @@ mod tests { ], schema, }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; + let family = exact(AggregationType::Sum); let summary_schema = SummarySchema { fields: vec![], time_index: None, @@ -1208,16 +1223,7 @@ mod tests { fields: vec![], time_index: None, }; - let family = materialization( - AggregationType::Sum, - "value", - 60, - 60, - ("variant", serde_json::json!(1)), - ) - .accumulator_spec() - .unwrap() - .family; + let family = exact(AggregationType::Sum); let node = SummaryNode { expr: SummaryExpr::SummaryAgg { child: std::rc::Rc::new(SummaryNode { @@ -1246,130 +1252,55 @@ mod tests { #[test] fn selected_nodes_bind_unique_family_parameters_source_and_window() { - let sum_60 = materialization( - AggregationType::Sum, - "requests", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let count_60 = materialization( - AggregationType::Max, - "requests", - 60, - 10, - ("variant", serde_json::json!(2)), - ); - let sum_300 = materialization( - AggregationType::Sum, - "requests", - 300, - 30, - ("variant", serde_json::json!(3)), - ); - let other = materialization( - AggregationType::Sum, - "latency", - 60, - 10, - ("variant", serde_json::json!(1)), - ); - let dd_2 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.02)), - ); - let dd_5 = materialization( - AggregationType::DDSketch, - "requests", - 60, - 10, - ("relativeAccuracy", serde_json::json!(0.05)), - ); - let configs = vec![ - sum_60.clone(), - count_60.clone(), - sum_300, - other, - dd_2.clone(), - dd_5, - ]; - let sum_family = sum_60.accumulator_spec().unwrap().family; - let count_family = count_60.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - sum_60.policy_fingerprint() - ); - let dd_family = dd_2.accumulator_spec().unwrap().family; - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &dd_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - dd_2.policy_fingerprint() - ); - assert_eq!( - select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &count_family, - 60 - ) - .unwrap() - .policy_fingerprint(), - count_60.policy_fingerprint() - ); - assert!(select_materialization( - &configs, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "missing".into() - }, - "", - &sum_family, - 60 - ) - .is_err()); - let mut ambiguous = configs.clone(); - ambiguous.push(sum_60); - assert!(select_materialization( - &ambiguous, - "telemetry", - &asap_types::sds::ValueProjectionIdentity::Column { - name: "requests".into() - }, - "", - &sum_family, - 60 + let sum_60 = materialization(exact(AggregationType::Sum), "requests", 60, 60); + let count_60 = materialization(exact(AggregationType::Max), "requests", 60, 60); + let sum_300 = materialization(exact(AggregationType::Sum), "requests", 300, 300); + let other = materialization(exact(AggregationType::Sum), "latency", 60, 60); + let dd_2 = materialization(ddsketch(0.02), "requests", 60, 60); + let dd_5 = materialization(ddsketch(0.05), "requests", 60, 60); + let plan = PrecomputePlan::build_backend_local( + envelope(1), + vec![ + sum_60.clone(), + count_60.clone(), + sum_300, + other, + dd_2.clone(), + dd_5, + ], ) - .unwrap_err() - .to_string() - .contains("ambiguous")); + .unwrap(); + let requests = asap_types::sds::ValueProjectionIdentity::Column { + name: "requests".into(), + }; + let select = |plan, projection, family| { + select_materialization(plan, "telemetry", projection, "", family, 60) + .map(|selected| selected.policy_fingerprint()) + }; + for expected in [&sum_60, &count_60, &dd_2] { + assert_eq!( + select(&plan, &requests, &expected.1).unwrap(), + expected.0.policy_fingerprint() + ); + } + let missing = asap_types::sds::ValueProjectionIdentity::Column { + name: "missing".into(), + }; + assert!(select(&plan, &missing, &sum_60.1).is_err()); + let mut ambiguous = plan.clone(); + let mut duplicate = sum_60.0.clone(); + duplicate.stored_output_id = asap_types::sds::StoredOutputId(1); + ambiguous.materializations.push(duplicate.clone()); + ambiguous + .schemas + .push(asap_types::precompute_plan::StateSchemaContract { + materialization: duplicate.stored_output_id, + ..plan.schemas[0].clone() + }); + assert!(select(&ambiguous, &requests, &sum_60.1) + .unwrap_err() + .to_string() + .contains("ambiguous")); } #[test] @@ -1414,27 +1345,11 @@ mod tests { #[tokio::test] async fn rejects_mixed_snapshots_and_preserves_local_sql_contracts() { - let config = materialization( - AggregationType::Sum, - "value", - 2, - 2, - ("variant", serde_json::json!(1)), - ); - let sds = - SummaryCatalog::from_materializations(71, 1, std::slice::from_ref(&config)).unwrap(); - let envelope = crate::physical::compiler::PlanEnvelope { - plan_id: 71, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: crate::physical::compiler::BACKEND_COMPAT.into(), - planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), - capability_snapshot_id: "clickhouse-mixed-compile-test".into(), - }; + let config = materialization(exact(AggregationType::Sum), "value", 2, 2); + let envelope = envelope(71); let mut precompute = PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); + let sds = SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&sds).unwrap(); let mut transmission = crate::physical::compiler::build_transmission_plan( envelope, @@ -1734,11 +1649,14 @@ mod tests { value: planner_types::pre_asap::ScalarValue::Utf8("requests".into()), }], }); - request.summary_catalog = - SummaryCatalog::from_materializations(71, 1, &[config.clone()]).unwrap(); + config.allocate_stored_output_id(&exact(AggregationType::Sum)); let envelope = request.precompute_plan.envelope.clone(); - request.precompute_plan = - PrecomputePlan::build_backend_local(envelope.clone(), vec![config]).unwrap(); + request.precompute_plan = PrecomputePlan::build_backend_local( + envelope.clone(), + vec![(config, exact(AggregationType::Sum))], + ) + .unwrap(); + request.summary_catalog = SummaryCatalog::from_plan(&request.precompute_plan).unwrap(); request .precompute_plan .bind_catalog(&request.summary_catalog) diff --git a/control_plane/src/opamp/mod.rs b/control_plane/src/opamp/mod.rs index 33759262a..dea69a85a 100644 --- a/control_plane/src/opamp/mod.rs +++ b/control_plane/src/opamp/mod.rs @@ -1004,7 +1004,7 @@ mod tests { generated_at_unix_ms: 1, activation_unix_ms: 1, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: crate::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "caps-1".into(), }, @@ -1033,7 +1033,7 @@ mod tests { transmission_rules: vec![crate::physical::compiler::TransmissionRule { materialization: asap_types::PolicyFingerprint(1).into(), producer_id: collector_id.into(), - schema_id: "asap-query-backend.v1:summary-state:v1:1".into(), + schema_id: "asap-query-backend.v2:summary-state:v1:1".into(), mode: crate::physical::compiler::TransmissionMode::Full, encoding: crate::physical::compiler::StateEncoding::SketchlibProtobufV1, emit_every_ms: 60_000, diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index b04e57d7f..251856e99 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -852,7 +852,7 @@ fn state_fingerprints( &physical_aggregation(query, state, query.query_id.clone(), target), &state.node, ) - .map(|config| config.policy_fingerprint()) + .map(|(config, _)| config.policy_fingerprint()) .map_err(|error| error.to_string()) }) .collect() @@ -1634,6 +1634,7 @@ impl DeploymentPlanCompiler { } let population_operators = super::maintained_population::operators(&request)?; let mut compiled_materializations = Vec::with_capacity(request.queries.len()); + let mut output_computations = BTreeMap::<_, OutputComputation>::new(); let mut collector_materializations = Vec::with_capacity(request.queries.len()); let mut plan_materializations = Vec::with_capacity(request.queries.len()); let mut shared_materializations = BTreeMap::new(); @@ -1843,7 +1844,7 @@ impl DeploymentPlanCompiler { .or_default() .push(kind.algorithm().clone()); } - let precompute_materialization = + let (precompute_materialization, _) = scoped_materialization(&aggregation, &selected.node)?; let materialization = precompute_materialization.policy_fingerprint(); let consumer_indices = consumers[&materialization] @@ -1911,7 +1912,7 @@ impl DeploymentPlanCompiler { // Preserve semantic window and evaluation cadence independently // from the selected storage representation. aggregation.window_secs = window_implementation.window_secs; - let mut runtime_materialization = + let (mut runtime_materialization, computation) = scoped_materialization(&aggregation, &selected.node)?; if canonical_nodes.contains(&(Rc::as_ptr(&selected.node) as usize)) { runtime_materialization.population_key_encoding = @@ -2059,7 +2060,9 @@ impl DeploymentPlanCompiler { reason, })?, ); + runtime_materialization.allocate_stored_output_id(&computation); let materialization = runtime_materialization.policy_fingerprint(); + output_computations.insert(materialization, computation.clone()); let consumer_query_ids = state_consumers .iter() .map(|query| query.query_id.clone()) @@ -2173,6 +2176,7 @@ impl DeploymentPlanCompiler { &mut node_bindings, &mut runtime_policies, &mut lifecycle_estimates, + &mut output_computations, ); } @@ -2293,11 +2297,15 @@ impl DeploymentPlanCompiler { })?; let (_, source_window, _) = selected_input_contract(node) .map_err(crate::query_plan::QueryPlanError::Invalid)?; - let materialization_family = materialization.accumulator_spec() - .map_err(|error| crate::query_plan::QueryPlanError::Invalid(error.to_string()))? - .family; + let materialization_family = &output_computations + .get(&fingerprint) + .ok_or_else(|| crate::query_plan::QueryPlanError::Invalid(format!( + "compiled binding {} has no state family", + fingerprint.0 + )))? + .0; let window_ms = materialization.window_size.saturating_mul(1_000); - if materialization_family != *node_family + if materialization_family != node_family || window_ms == 0 || source_window.map(|seconds| seconds.saturating_mul(1_000)).unwrap_or(query.query_lookback_ms) % window_ms != 0 @@ -2316,7 +2324,12 @@ impl DeploymentPlanCompiler { output_grouping: PhysicalGrouping::Reduce( materialization.grouping_labels.names(), ), - item_labels: materialization.aggregated_labels.labels.clone(), + item_labels: output_computations[&fingerprint].1["item_labels"] + .as_array() + .into_iter() + .flatten() + .filter_map(|label| label.as_str().map(str::to_owned)) + .collect(), window_ms: stored_interval_ms, pane_origin_ms: materialization.pane_origin_ms, }) @@ -2703,7 +2716,7 @@ impl DeploymentPlanCompiler { } } validate_retained_summary_footprint( - &materializations, + &with_families(materializations.clone(), &output_computations), request .data_workload .as_ref() @@ -2746,19 +2759,20 @@ impl DeploymentPlanCompiler { }) .collect::>()?; let mut precompute_plan = match environment.target { - PhysicalDeploymentTarget::DistributedCollectors => { - PrecomputePlan::build(envelope.clone(), materializations, &producer_ids).and_then( - |mut plan| { - plan.executable_dags = installed_dags; - plan.validate()?; - Ok(plan) - }, - ) - } + PhysicalDeploymentTarget::DistributedCollectors => PrecomputePlan::build( + envelope.clone(), + with_families(materializations, &output_computations), + &producer_ids, + ) + .and_then(|mut plan| { + plan.executable_dags = installed_dags; + plan.validate()?; + Ok(plan) + }), PhysicalDeploymentTarget::BackendLocalRemoteWrite => { PrecomputePlan::build_backend_local_with_dags( envelope.clone(), - materializations, + with_families(materializations, &output_computations), installed_dags, ) } @@ -2792,12 +2806,8 @@ impl DeploymentPlanCompiler { materializations: collector_materializations.clone(), }) .collect::>(); - let summary_catalog = super::summary_catalog::SummaryCatalog::from_materializations( - envelope.plan_id, - envelope.plan_version, - &precompute_plan.materializations, - ) - .map_err(|error| CompileError::Query { + let summary_catalog = super::summary_catalog::SummaryCatalog::from_plan(&precompute_plan) + .map_err(|error| CompileError::Query { query_id: "summary-catalog".into(), reason: error.to_string(), })?; @@ -3796,11 +3806,36 @@ pub(super) fn retained_state_count( /// Estimate the encoded bytes retained by one physical state. Sketch matrix /// cells use two words here, covering the counter plus observed serialization /// overhead. Heap and exact-state estimates include container slack. -fn retained_state_bytes(materialization: &asap_types::PrecomputeMaterialization) -> u128 { - estimated_state_bytes( - &materialization.aggregation_type, - &materialization.parameters, - ) +fn retained_state_bytes(family: &SummaryFamilyType) -> u128 { + let parameters = match family { + SummaryFamilyType::Sketch(kind, _) => sketch_params_json(kind.params()), + _ => json!({}), + }; + asap_types::aggregation_type_for_family(family).map_or(256, |aggregation_type| { + estimated_state_bytes( + &aggregation_type, + ¶meters + .as_object() + .into_iter() + .flatten() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), + ) + }) +} + +/// Pair each output with its state family for the installed plan. +fn with_families( + materializations: Vec, + computations: &BTreeMap, +) -> Vec<(asap_types::PrecomputeMaterialization, SummaryFamilyType)> { + materializations + .into_iter() + .map(|config| { + let family = computations[&config.policy_fingerprint()].0.clone(); + (config, family) + }) + .collect() } pub(crate) fn estimated_state_bytes( @@ -3849,23 +3884,25 @@ pub(crate) fn estimated_state_bytes( pub(crate) fn retained_partition_count( materialization: &asap_types::PrecomputeMaterialization, + family: &SummaryFamilyType, input_cardinality: Option, ) -> u128 { use asap_types::AggregationType as A; + let aggregation_type = asap_types::aggregation_type_for_family(family); // Reset-aware and min/max state remains source-series scoped even with an // empty output grouping. Grouped states have at most one partition per // input series. Other empty groupings are the Reduce([]) global singleton. if materialization.partitioning == Some(asap_types::sds::PopulationPartitioning::PerEntity) || matches!( - materialization.aggregation_type, - A::Increase | A::Rate | A::Min | A::Max + aggregation_type, + Some(A::Increase | A::Rate | A::Min | A::Max) ) || !materialization.grouping_labels.names().is_empty() || (materialization.derived_input.is_some() && matches!( - materialization.aggregation_type, - A::CountMinSketchWithHeap | A::CountSketchWithHeap | A::Sum + aggregation_type, + Some(A::CountMinSketchWithHeap | A::CountSketchWithHeap | A::Sum) )) { u128::from(input_cardinality.unwrap_or(1).max(1)) @@ -3875,19 +3912,23 @@ pub(crate) fn retained_partition_count( } fn validate_retained_summary_footprint( - materializations: &[asap_types::PrecomputeMaterialization], + outputs: &[(asap_types::PrecomputeMaterialization, SummaryFamilyType)], input_cardinality: Option, budget_bytes: u64, ) -> Result<(), CompileError> { - let estimated_bytes = materializations + let estimated_bytes = outputs .iter() - .fold(0u128, |total, materialization| { + .fold(0u128, |total, (materialization, family)| { total.saturating_add( - retained_state_bytes(materialization) + retained_state_bytes(family) .saturating_mul(u128::from( materialization.num_aggregates_to_retain.unwrap_or(1), )) - .saturating_mul(retained_partition_count(materialization, input_cardinality)), + .saturating_mul(retained_partition_count( + materialization, + family, + input_cardinality, + )), ) }); if estimated_bytes > u128::from(budget_bytes) { @@ -4156,71 +4197,7 @@ pub(crate) fn raw_materialization_input_contract( ) } -pub fn raw_time_series_input_contract( - expr: &QueryExpr, - exact: bool, -) -> Result<(String, Option, String), String> { - use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let (source, window_secs) = match expr { - QueryExpr::TimeRange { child, range } => { - if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { - return Err("warm producer requires a positive whole-second range".into()); - } - (child.as_ref(), Some(range.as_secs())) - } - QueryExpr::Scan { .. } if exact => { - return Err( - "instantaneous sample selection is not a temporal accumulator readout".into(), - ); - } - source => (source, None), - }; - match source { - QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } if !metric.is_empty() => { - let mut matchers = Vec::with_capacity(predicates.len()); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - return Err("materialization filter is not a label comparison".into()); - }; - let (QueryExpr::Column(column), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return Err("materialization filter must compare a label with a string".into()); - }; - let label = schema - .columns - .get(*column) - .map(|field| field.name.as_str()) - .ok_or_else(|| { - "materialization filter references an unknown column".to_string() - })?; - let operator = match op { - CompareOpKind::Eq => "=", - CompareOpKind::Ne => "!=", - CompareOpKind::Regex => "=~", - CompareOpKind::NotRegex => "!~", - _ => return Err("materialization filter uses a non-PromQL comparison".into()), - }; - let encoded = serde_json::to_string(value).map_err(|error| error.to_string())?; - matchers.push(format!("{label}{operator}{encoded}")); - } - matchers.sort(); - let spatial_filter = if matchers.is_empty() { - String::new() - } else { - format!("{{{}}}", matchers.join(",")) - }; - Ok((metric.clone(), window_secs, spatial_filter)) - } - _ => { - Err("source predicates or temporal modifiers require a Prometheus exact subtree".into()) - } - } -} +pub use asap_types::precompute_plan::raw_time_series_input_contract; /// Immutable inputs may combine explicit exact accumulator readouts using /// maintenance-time arithmetic. Population and window closure are checked by @@ -4438,11 +4415,11 @@ fn physical_aggregation( fn scoped_materialization( aggregation: &BackendAggregation, node: &SummaryNode, -) -> anyhow::Result { - let mut config = +) -> anyhow::Result<(asap_types::PrecomputeMaterialization, OutputComputation)> { + let (mut config, computation) = aggregation_config_for_materialization(aggregation, asap_types::QueryLanguage::PromQl)?; if !matches!(aggregation.aggregation_input, AggregationInput::Raw) { - return Ok(config); + return Ok((config, computation)); } let SummaryExpr::SummaryAgg { reduction, .. } = &node.expr else { anyhow::bail!("materialization lacks SummaryAgg partition contract"); @@ -4471,33 +4448,58 @@ fn scoped_materialization( asap_types::sds::PopulationPartitioning::Grouped } }); - Ok(config) + config.allocate_stored_output_id(&computation); + Ok((config, computation)) } -pub(crate) fn aggregation_config_for_materialization( +/// The Planner computation a stored output id is allocated from: the +/// output's state family and its canonical producer description (update, +/// input predicate and item projection). The compiler keeps it beside each +/// output so later layout decisions can reallocate the id; installed plans +/// carry neither, since the bound DAG node defines the computation. +pub(crate) type OutputComputation = (SummaryFamilyType, Value); + +pub(crate) fn output_computation( aggregation: &BackendAggregation, language: asap_types::QueryLanguage, -) -> anyhow::Result { - use anyhow::Context as _; - let mut json = crate::emit::backend_wire::build_backend_aggregation_json(aggregation); - // The selected physical duration is authoritative. The legacy edge emitter's - // 5..60 second clamp must not silently change a backend materialization. - json["windowSize"] = serde_json::json!(aggregation.window_secs); +) -> OutputComputation { + let json = crate::emit::backend_wire::build_backend_aggregation_json(aggregation); + let mut parameters = json["parameters"].clone(); if language == asap_types::QueryLanguage::ClickHouseSql { // Raw input does not imply PromQL's (start,end] convention. SQL bounds // are normalized and bound explicitly by the SQL compiler. - json["parameters"] + parameters .as_object_mut() .expect("emitter parameters are an object") .remove("promql_right_closed"); } - let yaml = serde_yaml::to_value(json).context("convert physical aggregation fields")?; - asap_types::PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - asap_types::QueryLanguage::PromQl, + // Deployment fields (metric, grouping, window) are hashed from the + // materialization itself, so layout changes can reallocate the id. + ( + aggregation.family.clone(), + serde_json::json!({ + "aggregation_type": json["aggregationType"], + "parameters": parameters, + "item_labels": json["labels"]["aggregated"], + "spatial_filter": asap_types::utils::normalize_spatial_filter(&aggregation.spatial_filter), + }), ) - .context("build materialization from physical aggregation") +} + +pub(crate) fn aggregation_config_for_materialization( + aggregation: &BackendAggregation, + language: asap_types::QueryLanguage, +) -> anyhow::Result<(asap_types::PrecomputeMaterialization, OutputComputation)> { + let computation = output_computation(aggregation, language); + let mut config = asap_types::PrecomputeMaterialization::new( + aggregation.metric_name.clone(), + asap_types::KeyByLabelNames::new(aggregation.grouping.clone()), + aggregation.window_secs, + aggregation.window_secs, + asap_types::WindowKind::Tumbling, + ); + config.allocate_stored_output_id(&computation); + Ok((config, computation)) } fn materialization_consumers<'a>( @@ -4532,7 +4534,7 @@ fn materialization_consumers<'a>( { continue; } - let config = scoped_materialization( + let (config, _) = scoped_materialization( &physical_aggregation(query, &state, query.query_id.clone(), target), &state.node, )?; @@ -5030,6 +5032,29 @@ impl QueryFrontend { } #[cfg(test)] pub(crate) mod tests { + /// The Planner update of the DAG node that produces `output`. + fn bound_summary_update( + plan: &PrecomputePlan, + output: asap_types::sds::StoredOutputId, + ) -> planner_types::post_asap::SummaryUpdate { + use asap_types::executable_plan::BackendNodeBinding; + plan.executable_dags + .values() + .find_map(|installed| { + let (node, _) = installed.binding.nodes.iter().find(|(_, binding)| { + matches!(binding, BackendNodeBinding::Materialization { stored_output } if *stored_output == output) + })?; + let dag = installed.document.decode().unwrap(); + match &dag.nodes.iter().find(|candidate| candidate.id == *node)?.payload { + planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { input, .. } => { + Some(input.clone()) + } + _ => None, + } + }) + .expect("output has a Planner SummaryAgg producer") + } + use super::*; /// Multiple current-value quantiles and TopK limits share one maintained population. @@ -5200,8 +5225,12 @@ pub(crate) mod tests { let plan = DeploymentPlanCompiler.compile_promql(input, env).unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - asap_types::AggregationType::Min + asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(plan.precompute_plan.materializations[0].stored_output_id) + .unwrap() + ), + Some(asap_types::AggregationType::Min) ); assert!(plan .query_plan @@ -5844,10 +5873,18 @@ pub(crate) mod tests { .compile_promql(request, environment(10000)) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1, "{query}"); - assert_eq!( - plan.precompute_plan.materializations[0].parameters["weight_mode"], mode, - "{query}" - ); + let weight = bound_summary_update( + &plan.precompute_plan, + plan.precompute_plan.materializations[0].stored_output_id, + ) + .weight; + let expected = match mode { + "count" => planner_types::post_asap::SummaryInputExpr::Constant(1.0), + _ => planner_types::post_asap::SummaryInputExpr::Column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + }; + assert_eq!(weight, expected, "{query}"); } } @@ -5935,38 +5972,71 @@ pub(crate) mod tests { environment(10_000), ) .unwrap(); - let mut materializations = plan.precompute_plan.materializations; - let cms = materializations + let mut outputs = plan + .precompute_plan + .materializations + .iter() + .map(|config| { + let family = plan + .precompute_plan + .state_family(config.stored_output_id) + .unwrap(); + (config.clone(), family.clone()) + }) + .collect::>(); + let heap_size = outputs + .iter() + .find_map(|(_, family)| match family { + SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::CmsWithHeap { heap_size, .. } => { + Some(*heap_size) + } + _ => None, + }, + _ => None, + }) + .unwrap(); + let heap = |width| { + SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::CmsWithHeap, + planner_types::post_asap::SketchParams::CmsWithHeap { + width, + depth: 7, + heap_size, + }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ) + }; + let cms = outputs .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap + .find(|(_, family)| { + asap_types::aggregation_type_for_family(family) + == Some(asap_types::AggregationType::CountMinSketchWithHeap) }) .unwrap(); - cms.parameters.remove("width"); - cms.parameters.remove("depth"); - cms.parameters.insert("w".into(), json!(524_288)); - cms.parameters.insert("d".into(), json!(7)); - cms.num_aggregates_to_retain = Some(80); + cms.1 = heap(524_288); + cms.0.num_aggregates_to_retain = Some(80); let error = validate_retained_summary_footprint( - &materializations, + &outputs, Some(5), DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, ) .unwrap_err(); assert!(error.to_string().contains("exceeds budget")); - let cms = materializations + let cms = outputs .iter_mut() - .find(|materialization| { - materialization.aggregation_type - == asap_types::AggregationType::CountMinSketchWithHeap + .find(|(_, family)| { + asap_types::aggregation_type_for_family(family) + == Some(asap_types::AggregationType::CountMinSketchWithHeap) }) .unwrap(); - cms.parameters.insert("w".into(), json!(4_096)); + cms.1 = heap(4_096); validate_retained_summary_footprint( - &materializations, + &outputs, Some(5), DEFAULT_RETAINED_SUMMARY_MEMORY_BUDGET_BYTES, ) @@ -6055,7 +6125,7 @@ pub(crate) mod tests { plan_version: 1, activation_unix_ms: now, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), } } @@ -7554,12 +7624,12 @@ pub(crate) mod tests { .compile_promql(request("q", "sum_over_time(m[1m])"), environment(10_000)) .unwrap(); let mut source = bundle.precompute_plan.materializations[0].clone(); - source.stored_output_id = Some(asap_types::sds::StoredOutputId(1)); + source.stored_output_id = asap_types::sds::StoredOutputId(1); source.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; let mut configs = vec![source.clone()]; for id in [3, 4] { let mut output = source.clone(); - output.stored_output_id = Some(asap_types::sds::StoredOutputId(id)); + output.stored_output_id = asap_types::sds::StoredOutputId(id); output.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([asap_types::sds::StoredOutputId(1)]), program_sha256: "0".repeat(64), @@ -8876,9 +8946,19 @@ pub(crate) mod tests { environment(10_000), ) .unwrap(); - let mut materializations = bundle.precompute_plan.materializations; - materializations[0].aggregation_type = asap_types::AggregationType::HLL; - materializations[0].parameters.clear(); + let hll = SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + planner_types::post_asap::SketchParams::Hll { precision: 14 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let materializations = bundle + .precompute_plan + .materializations + .into_iter() + .map(|config| (config, hll.clone())) + .collect::>(); let mut envelope_plan = PrecomputePlan::build( bundle.envelope.clone(), materializations.clone(), @@ -8909,7 +8989,15 @@ pub(crate) mod tests { .expect("compile"); let plan = PrecomputePlan::build_backend_local( bundle.envelope.clone(), - bundle.precompute_plan.materializations.clone(), + bundle + .precompute_plan + .materializations + .iter() + .map(|config| { + let family = bundle.precompute_plan.state_family(config.stored_output_id); + (config.clone(), family.unwrap().clone()) + }) + .collect(), ) .expect("backend-local contract"); @@ -9106,7 +9194,10 @@ pub(crate) mod tests { if expected_readout == crate::query_plan::ExactReadout::Rate { let materialization = plan.precompute_plan.materializations.first().unwrap(); assert_eq!( - materialization.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(materialization.stored_output_id) + .unwrap() + .clone(), SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Rate, planner_types::post_asap::ExactParams::Rate, diff --git a/control_plane/src/physical/erp.rs b/control_plane/src/physical/erp.rs index 72dfb3e2a..0a59d97dc 100644 --- a/control_plane/src/physical/erp.rs +++ b/control_plane/src/physical/erp.rs @@ -396,17 +396,17 @@ impl ErpPlanningInput { .summary_descriptors .get(&materialization.summary_descriptor_id) .ok_or("ERP summary descriptor is absent from the active catalog")?; - let actual_semantics = match &summary.operator { + let algorithm = match &summary.operator { asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::HLL, - .. - } => Some( + family: planner_types::post_asap::SummaryFamilyType::Sketch(kind, _), + } => Some(kind.algorithm()), + _ => None, + }; + let actual_semantics = match algorithm { + Some(planner_types::post_asap::SketchAlgorithm::Hll) => Some( asap_types::erp_observation::ErpObservationInputSemantics::ScalarSampleValue, ), - asap_types::sds::SummaryOperator::Configured { - aggregation_type: asap_types::AggregationType::UnivMon, - .. - } => Some( + Some(planner_types::post_asap::SketchAlgorithm::UnivMon) => Some( asap_types::erp_observation::ErpObservationInputSemantics::UnitSampleFrequency, ), _ => None, @@ -1468,14 +1468,24 @@ mod tests { // Catalog resolution is independent of Planner selection. Build an // HLL catalog fixture from a compiled source identity; production // selection cannot deploy HLL without a known confidence guarantee. - let mut materialization = plan.precompute_plan.materializations[0].clone(); - materialization.aggregation_type = asap_types::AggregationType::HLL; - materialization.parameters = - serde_json::from_value(serde_json::json!({"precision": 14})).unwrap(); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let materialization = &plan.precompute_plan.materializations[0]; + let hll = planner_types::post_asap::SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + SketchParams::Hll { precision: 14 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( plan.summary_catalog.plan_id, plan.summary_catalog.plan_version, - &[materialization], + vec![( + materialization, + &hll, + plan.precompute_plan + .population_filter(materialization) + .unwrap(), + )], ) .unwrap(); let (mut policy, mut observed) = online_population_fixture(); diff --git a/control_plane/src/physical/pane_reuse.rs b/control_plane/src/physical/pane_reuse.rs index e1e29a4dc..c7aad57c2 100644 --- a/control_plane/src/physical/pane_reuse.rs +++ b/control_plane/src/physical/pane_reuse.rs @@ -1,6 +1,7 @@ use super::compiler::{ derived_window_cost, gcd, retained_state_count, CollectorMaterialization, - MaterializationLifecycleEstimate, PhysicalCompilationRequest, RuntimeRulePolicy, + MaterializationLifecycleEstimate, OutputComputation, PhysicalCompilationRequest, + RuntimeRulePolicy, }; use asap_types::WindowMaterializationLayout; use planner_types::post_asap::{PostAsapNodeId, SummaryWindowFramework}; @@ -17,8 +18,10 @@ pub(super) fn share_additive_panes( bindings: &mut BTreeMap<(usize, PostAsapNodeId), asap_types::PolicyFingerprint>, policies: &mut BTreeMap, estimates: &mut BTreeMap, + computations: &mut BTreeMap, ) { - use asap_types::{AggregationType, WindowKind}; + use asap_types::WindowKind; + use planner_types::post_asap::{ExactKind, SummaryFamilyType}; let derived_sources = materializations .iter() .filter_map(|m| m.derived_input.as_ref()) @@ -34,7 +37,10 @@ pub(super) fn share_additive_panes( if !seen.insert(old) || m.derived_input.is_some() || derived_sources.contains(&old) - || !matches!(m.aggregation_type, AggregationType::Sum) + || !matches!( + computations.get(&old), + Some((SummaryFamilyType::ExactAggregate(ExactKind::Sum, _), _)) + ) { continue; } @@ -87,6 +93,7 @@ pub(super) fn share_additive_panes( canonical.window_type = WindowKind::Tumbling; canonical.window_layout = WindowMaterializationLayout::Pane { pane_secs: 1 }; canonical.pane_origin_ms = Some(0); + canonical.allocate_stored_output_id(&computations[&old]); let key = ( canonical.policy_fingerprint(), serde_json::to_string(&lifecycle).unwrap(), @@ -114,6 +121,8 @@ pub(super) fn share_additive_panes( pane_secs: group.pane_secs, }; canonical.pane_origin_ms = Some(group.origin); + let computation = &computations[&physical[index].0]; + physical[index].1.allocate_stored_output_id(computation); } } let mut target_counts = BTreeMap::new(); @@ -154,6 +163,8 @@ pub(super) fn share_additive_panes( combined.expected_updates = 0.0; combined.lifecycle_cost = group.cost; combined.window_realization_id = format!("shared-pane-{}", new.0); + let computation = computations[&physical[group.members[0]].0].clone(); + computations.insert(new, computation); for index in group.members { let old = physical[index].0; if let Some(estimate) = estimates.remove(&old) { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 8db586aa6..faf45a98a 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -918,26 +918,25 @@ mod catalog_binding_tests { use crate::physical::summary_catalog::SummaryCatalog; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - fn fixture() -> (QueryPlan, SummaryCatalog) { + /// A catalog over one `m` output grouped by `job`, with `kind` state. + fn output(kind: AggregationType) -> (PrecomputeMaterialization, SummaryCatalog) { + let family = kind.planner_exact_family().unwrap(); let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + "m", KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, ); config.pane_origin_ms = Some(0); - let catalog = SummaryCatalog::from_materializations(7, 2, &[config.clone()]).unwrap(); + config.allocate_stored_output_id(&family); + let catalog = + SummaryCatalog::from_outputs(7, 2, vec![(&config, &family, String::new())]).unwrap(); + (config, catalog) + } + + fn fixture() -> (QueryPlan, SummaryCatalog) { + let (config, catalog) = output(AggregationType::Sum); let entry = QueryPlanEntry { physical_dag: None, language: crate::query_plan::QueryLanguage::PromQl, @@ -1062,26 +1061,7 @@ mod catalog_binding_tests { .to_string() .contains("exact counter SDS")); - let mut counter = PrecomputeMaterialization::new( - AggregationType::Increase, - String::new(), - Default::default(), - KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 10, - 10, - WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, - ); - counter.pane_origin_ms = Some(0); - let counter_catalog = - SummaryCatalog::from_materializations(7, 2, &[counter.clone()]).unwrap(); + let (counter, counter_catalog) = output(AggregationType::Increase); let (mut counter_plan, _) = fixture(); binding(&mut counter_plan).materialization = counter.policy_fingerprint().into(); binding(&mut counter_plan).stored_output_reference = counter_catalog diff --git a/crates/asap_types/src/accumulator_spec.rs b/crates/asap_types/src/accumulator_spec.rs index bdf5949b4..20c9a0658 100644 --- a/crates/asap_types/src/accumulator_spec.rs +++ b/crates/asap_types/src/accumulator_spec.rs @@ -1,51 +1,6 @@ -//! Validate stored materialization descriptors against Planner summary families. -//! -//! This projection supports catalog identity and imported state metadata. It is -//! not an execution program. Raw and maintenance execution dispatch directly -//! on the selected post-ASAP DAG payload; the descriptor must agree with it. - -use serde_json::Value; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::key_by_label_names::KeyByLabelNames; -use crate::AggregationType; -use planner_types::post_asap::{ - ExactKind, ExactParams, GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, - SketchParams, SummaryFamilyType, -}; - -/// Data_plane's typed replacement for -/// `(aggregation_type, aggregation_sub_type, parameters)`: which -/// accumulator to run (`kind`), with what tuning (`params`), and -/// whether it's keyed by a group-by label set (`grouping`). -/// -/// Computed on demand from an [`PrecomputeMaterialization`] via -/// [`PrecomputeMaterialization::accumulator_spec`] — not stored on the config -/// itself, so there is exactly one source of truth for the fields that -/// feed [`crate::policy_fingerprint::PolicyFingerprint`]. -#[derive(Debug, Clone, PartialEq)] -pub struct AccumulatorSpec { - /// Planner-owned committed summary identity. For sketches this is a - /// validated `SketchKind` (category + algorithm + params), following the - /// ASAP-aware-mapping vocabulary. - pub family: SummaryFamilyType, - /// Physical keyed-state layout, independent of semantic family. - pub grouping: Option, -} - -/// Typed rule for turning an input sample into an accumulator update. -/// -/// This is execution semantics, separate from the summary family: the same -/// CMS-with-heap state can count events, sum sample values, or sum reset-aware -/// counter deltas. Stored descriptors encode the rule in -/// `parameters`; callers use [`PrecomputeMaterialization::sample_update_rule`] so the -/// runtime does not branch on ad-hoc strings. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum SampleUpdateRule { - Value { scale: f64 }, - Count, - CounterDelta { scale: f64 }, -} +//! Planner summary-update domains implemented by the raw accumulator kernels. +//! Stored outputs carry no update semantics; these predicates inspect the +//! bound Planner DAG node's `SummaryUpdate`. /// The implemented raw frequency domain counts occurrences of sample values. pub fn is_unit_sample_frequency(update: &planner_types::post_asap::SummaryUpdate) -> bool { @@ -74,747 +29,3 @@ pub fn is_scalar_sample_value(update: &planner_types::post_asap::SummaryUpdate) ) ) } - -impl PrecomputeMaterialization { - pub fn sample_update_rule(&self) -> SampleUpdateRule { - let scale = self - .parameters - .get("weight_scale") - .and_then(Value::as_f64) - .filter(|scale| scale.is_finite() && *scale > 0.0) - .unwrap_or(1.0); - match self - .parameters - .get("weight_mode") - .or_else(|| self.parameters.get("topk_weight")) - .and_then(Value::as_str) - .map(str::to_ascii_lowercase) - .as_deref() - { - Some("count" | "frequency" | "freq") => SampleUpdateRule::Count, - Some("counter_delta") => SampleUpdateRule::CounterDelta { scale }, - _ => SampleUpdateRule::Value { scale }, - } - } -} - -/// A storage descriptor cannot be resolved to a supported Planner family. -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum AccumulatorSpecError { - /// `aggregation_type` was `SingleSubpopulation` with an - /// `aggregation_sub_type` string not in the recognized alias list. - UnknownSingleSubpopulationSubType(String), - /// `aggregation_type` was `MultipleSubpopulation` with an - /// unrecognized `aggregation_sub_type`. - UnknownMultipleSubpopulationSubType(String), - /// `aggregation_type` itself has no accumulator-dispatch mapping. - /// Also returned for an invalid HLL precision. A resolved family identifies - /// stored state; it does not imply raw-value updater support. - UnmappedAggregationType(AggregationType), -} - -impl std::fmt::Display for AccumulatorSpecError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::UnknownSingleSubpopulationSubType(s) => { - write!(f, "Unknown SingleSubpopulation sub_type '{s}'") - } - Self::UnknownMultipleSubpopulationSubType(s) => { - write!(f, "Unknown MultipleSubpopulation sub_type '{s}'") - } - Self::UnmappedAggregationType(t) => write!(f, "Unknown aggregation_type '{t:?}'"), - } - } -} - -impl std::error::Error for AccumulatorSpecError {} - -impl PrecomputeMaterialization { - /// Resolve this config's `(aggregation_type, aggregation_sub_type, - /// parameters)` triple into a typed [`AccumulatorSpec`]. - /// - /// Unsupported descriptors return an error; this projection never chooses - /// a fallback family and cannot authorize DAG execution. - pub fn accumulator_spec(&self) -> Result { - use AggregationType::*; - - let sub_type = self.aggregation_sub_type.as_str(); - - let independent_sketch = |algorithm, params| { - SummaryFamilyType::Sketch( - SketchKind::new(algorithm, params), - GroupingStrategy::PerSubpopulationInstance, - ) - }; - let (family, keyed) = match self.aggregation_type { - Sum | Count | Increase | Rate | Min | Max => ( - self.aggregation_type.planner_exact_family().unwrap(), - !self.aggregated_labels.is_empty(), - ), - DatasketchesKLL => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - HydraKLL => { - let k = kll_k_param(self) as u32; - ( - SummaryFamilyType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), - GroupingStrategy::SharedMultiSubpopulation { - kind: HydraKind::HydraKll, - params: HydraParams::HydraKll { - k, - shared_buckets: k, - }, - }, - ), - true, - ) - } - CountMinSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountMinSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CmsWithHeap, - SketchParams::CmsWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - CountSketch => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::CountSketch, - SketchParams::CountSketch { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - CountSketchWithHeap => { - let (row_num, col_num) = cms_params(self); - let heap_size = heap_size_param(self); - ( - independent_sketch( - SketchAlgorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { - width: col_num as u32, - depth: row_num as u32, - heap_size: heap_size as u32, - }, - ), - true, - ) - } - DDSketch => ( - independent_sketch( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { - alpha: ddsketch_alpha_param(self), - }, - ), - false, - ), - UnivMon => { - let param = - |name: &str, default: u64, max: u64| -> Result { - let value = self - .parameters - .get(name) - .map_or(Some(default), |v| v.as_u64()) - .ok_or(AccumulatorSpecError::UnmappedAggregationType(UnivMon))?; - if value == 0 || value > max { - return Err(AccumulatorSpecError::UnmappedAggregationType(UnivMon)); - } - Ok(value as u32) - }; - ( - independent_sketch( - SketchAlgorithm::UnivMon, - SketchParams::UnivMon { - heap_size: param("heap_size", 32, u32::MAX as u64)?, - sketch_rows: param("sketch_rows", 5, 20)?, - sketch_cols: param("sketch_cols", 1024, u32::MAX as u64)?, - layers: param("layers", 4, 64)? as u8, - }, - ), - false, - ) - } - HLL => { - let precision = match self.parameters.get("precision") { - None => 14, - Some(value) => value - .as_u64() - .ok_or(AccumulatorSpecError::UnmappedAggregationType(HLL))?, - }; - if !(4..=18).contains(&precision) { - return Err(AccumulatorSpecError::UnmappedAggregationType(HLL)); - } - ( - independent_sketch( - SketchAlgorithm::Hll, - SketchParams::Hll { - precision: precision as u8, - }, - ), - false, - ) - } - SingleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - false, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - false, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - false, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - false, - ), - "DatasketchesKLL" | "datasketches_kll" | "KLL" | "kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - false, - ), - other => { - return Err(AccumulatorSpecError::UnknownSingleSubpopulationSubType( - other.to_string(), - )) - } - }, - MultipleSubpopulation => match sub_type { - "Sum" | "sum" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - true, - ), - "Min" | "min" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Min, ExactParams::Min), - true, - ), - "Max" | "max" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Max, ExactParams::Max), - true, - ), - "Increase" | "increase" => ( - SummaryFamilyType::ExactAggregate(ExactKind::Increase, ExactParams::Increase), - true, - ), - "CountMinSketch" | "count_min_sketch" | "CMS" | "cms" => { - let (row_num, col_num) = cms_params(self); - ( - independent_sketch( - SketchAlgorithm::Cms, - SketchParams::Cms { - width: col_num as u32, - depth: row_num as u32, - }, - ), - true, - ) - } - "HydraKLL" | "hydra_kll" => ( - independent_sketch( - SketchAlgorithm::Kll, - SketchParams::Kll { - k: kll_k_param(self) as u32, - }, - ), - true, - ), - other => { - return Err(AccumulatorSpecError::UnknownMultipleSubpopulationSubType( - other.to_string(), - )) - } - }, - }; - - let grouping = if keyed { - Some(self.grouping_labels.label_names()) - } else { - None - }; - - Ok(AccumulatorSpec { family, grouping }) - } -} - -// --------------------------------------------------------------------------- -// Raw-parameter extraction helpers. -// -// Relocated verbatim from `data_plane::precompute_engine::accumulator_factory` -// (same names, same behavior, same defaults) — this is now their one -// definition; `accumulator_factory.rs` re-exports them via `use` so its -// existing unit tests (`cms_params_reads_canonical_w_d_keys`, -// `test_kll_k_param_capital_k`, ...) keep passing unchanged. -// --------------------------------------------------------------------------- - -/// Extract the KLL `k` parameter. Capital `"K"` takes precedence over -/// lowercase `"k"` to match the convention used by the top-level -/// aggregation type arms. Defaults to 200. -pub fn kll_k_param(config: &PrecomputeMaterialization) -> u16 { - config - .parameters - .get("K") - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .and_then(|v| u16::try_from(v).ok()) - .unwrap_or(200) -} - -/// Extract `(row_num, col_num)` for CMS / HydraKLL configs. -/// -/// Reads canonical `d` (depth = rows) / `w` (width = cols) keys — -/// matches what the control plane's `sketch_params_to_json` emits and -/// what `sketch_config_to_params` uses for OTLP policy_fp content -/// matching. Defaults to `(4, 1000)`. -pub fn cms_params(config: &PrecomputeMaterialization) -> (usize, usize) { - let row_num = config - .parameters - .get("d") - .and_then(|v| v.as_u64()) - .unwrap_or(4) as usize; - let col_num = config - .parameters - .get("w") - .and_then(|v| v.as_u64()) - .unwrap_or(1000) as usize; - (row_num, col_num) -} - -/// Top-k heap size for the `*WithHeap` configs. Reads `heap_size` / `k` -/// from `parameters`; defaults to 20 (the heap holds the top-k -/// candidates — it must be >= the largest `k` a query asks for). -pub fn heap_size_param(config: &PrecomputeMaterialization) -> usize { - config - .parameters - .get("heap_size") - .or_else(|| config.parameters.get("k")) - .or_else(|| config.parameters.get("K")) - .and_then(|v| v.as_u64()) - .map(|v| v as usize) - .filter(|&v| v > 0) - .unwrap_or(20) -} - -/// Pull `relativeAccuracy` (or canonical aliases) out of a -/// streaming-config aggregation entry. Defaults to 0.01 (1% rel-err, -/// the same default the agent's `ddsketchprocessor` uses). -pub fn ddsketch_alpha_param(config: &PrecomputeMaterialization) -> f64 { - let parsed = param_f64(config, "relativeAccuracy") - .or_else(|| param_f64(config, "relative_accuracy")) - .or_else(|| param_f64(config, "alpha")) - .unwrap_or(0.01); - if parsed > 0.0 && parsed < 1.0 { - parsed - } else { - tracing::warn!( - "DDSketch relativeAccuracy {} out of (0,1); using default 0.01", - parsed - ); - 0.01 - } -} - -fn param_f64(config: &PrecomputeMaterialization, key: &str) -> Option { - config.parameters.get(key).and_then(Value::as_f64) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::key_by_label_names::KeyByLabelNames; - use std::collections::HashMap; - - fn assert_exact(spec: &AccumulatorSpec, expected: ExactKind) { - assert!(matches!( - &spec.family, - SummaryFamilyType::ExactAggregate(kind, _) if kind == &expected - )); - } - - fn assert_sketch( - spec: &AccumulatorSpec, - expected_algorithm: planner_types::post_asap::SketchAlgorithm, - expected_params: planner_types::post_asap::SketchParams, - ) { - match &spec.family { - SummaryFamilyType::Sketch(kind, _) => { - assert_eq!(kind.algorithm(), &expected_algorithm); - assert_eq!(kind.params(), &expected_params); - } - other => panic!("expected sketch family, got {other:?}"), - } - } - - #[allow(clippy::too_many_arguments)] - fn make_config( - agg_type: AggregationType, - sub_type: &str, - params: HashMap, - grouping_labels: Vec<&str>, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - sub_type.to_string(), - params, - KeyByLabelNames::new(grouping_labels.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) - } - - // ---- direct (non-wrapper) variants ------------------------------ - - #[test] - fn sum_is_unkeyed_sum() { - let cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - - #[test] - fn keyed_layout_preserves_sum_family() { - let mut cfg = make_config(AggregationType::Sum, "", HashMap::new(), vec!["zone"]); - cfg.aggregated_labels = KeyByLabelNames::new(vec!["host".into()]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert_eq!( - spec.grouping, - Some(KeyByLabelNames::new(vec!["zone".to_string()])) - ); - } - - #[test] - fn datasketches_kll_reads_k_param() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(128)); - let cfg = make_config(AggregationType::DatasketchesKLL, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 128 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hydra_kll_is_keyed_kll() { - let mut params = HashMap::new(); - params.insert("k".to_string(), serde_json::json!(64)); - let cfg = make_config(AggregationType::HydraKLL, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Kll, - planner_types::post_asap::SketchParams::Kll { k: 64 }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_maps_to_cms_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(7)); - params.insert("w".to_string(), serde_json::json!(2048)); - let cfg = make_config(AggregationType::CountMinSketch, "", params, vec!["host"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::Cms, - planner_types::post_asap::SketchParams::Cms { - width: 2048, - depth: 7, - }, - ); - assert!(spec.grouping.is_some()); - } - - #[test] - fn count_min_sketch_with_heap_maps_to_cms_with_heap_params() { - let mut params = HashMap::new(); - params.insert("d".to_string(), serde_json::json!(4)); - params.insert("w".to_string(), serde_json::json!(256)); - params.insert("heap_size".to_string(), serde_json::json!(8)); - let cfg = make_config( - AggregationType::CountMinSketchWithHeap, - "", - params, - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::CmsWithHeap, - planner_types::post_asap::SketchParams::CmsWithHeap { - width: 256, - depth: 4, - heap_size: 8, - }, - ); - } - - /// Documented existing quirk (see `accumulator_factory.rs`): bare - /// `CountSketch` gets its own `SketchAlgorithm` identity here, but - /// `accumulator_factory` routes it through the same - /// `CmsAccumulatorUpdater` as bare CMS — no dedicated heap-less - /// Count-Sketch accumulator exists. This test locks in the *identity* - /// resolution only; the shared-accumulator behavior is exercised in - /// `accumulator_factory.rs`'s own tests. - #[test] - fn count_sketch_gets_its_own_kind_identity() { - let cfg = make_config(AggregationType::CountSketch, "", HashMap::new(), vec!["h"]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::CountSketch - )); - } - - #[test] - fn ddsketch_reads_relative_accuracy_alpha() { - let mut params = HashMap::new(); - params.insert("relativeAccuracy".to_string(), serde_json::json!(0.02)); - let cfg = make_config(AggregationType::DDSketch, "", params, vec![]); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_sketch( - &spec, - planner_types::post_asap::SketchAlgorithm::DDSketch, - planner_types::post_asap::SketchParams::DDSketch { alpha: 0.02 }, - ); - assert!(spec.grouping.is_none()); - } - - #[test] - fn hll_has_catalog_identity_for_sketch_envelope_ingest() { - let cfg = make_config(AggregationType::HLL, "", HashMap::new(), vec![]); - let spec = cfg.accumulator_spec().unwrap(); - assert_sketch( - &spec, - SketchAlgorithm::Hll, - SketchParams::Hll { precision: 14 }, - ); - for precision in [ - serde_json::json!(3), - serde_json::json!(19), - serde_json::json!(4.5), - serde_json::json!("bad"), - ] { - let cfg = make_config( - AggregationType::HLL, - "", - HashMap::from([("precision".into(), precision)]), - vec![], - ); - assert!(cfg.accumulator_spec().is_err()); - } - } - - // ---- wrapper (Single/MultipleSubpopulation) variants ------------ - - #[test] - fn single_subpopulation_sum_alias() { - for alias in ["Sum", "sum"] { - let cfg = make_config( - AggregationType::SingleSubpopulation, - alias, - HashMap::new(), - vec![], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert_exact(&spec, ExactKind::Sum); - assert!(spec.grouping.is_none()); - } - } - - #[test] - fn multiple_subpopulation_cms_alias() { - for alias in ["CountMinSketch", "count_min_sketch", "CMS", "cms"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Cms - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn multiple_subpopulation_hydra_kll_alias() { - for alias in ["HydraKLL", "hydra_kll"] { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - alias, - HashMap::new(), - vec!["host"], - ); - let spec = cfg.accumulator_spec().expect("resolves"); - assert!(matches!( - &spec.family, - SummaryFamilyType::Sketch(kind, _) - if kind.algorithm() == &planner_types::post_asap::SketchAlgorithm::Kll - )); - assert!(spec.grouping.is_some()); - } - } - - #[test] - fn unknown_single_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::SingleSubpopulation, - "Bogus", - HashMap::new(), - vec![], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn unknown_multiple_subpopulation_sub_type_errors() { - let cfg = make_config( - AggregationType::MultipleSubpopulation, - "Bogus", - HashMap::new(), - vec!["host"], - ); - let err = cfg.accumulator_spec().expect_err("unknown sub_type"); - assert_eq!( - err, - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - ); - } - - #[test] - fn error_display_matches_pre_step5_warning_text() { - assert_eq!( - AccumulatorSpecError::UnknownSingleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown SingleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnknownMultipleSubpopulationSubType("Bogus".to_string()) - .to_string(), - "Unknown MultipleSubpopulation sub_type 'Bogus'" - ); - assert_eq!( - AccumulatorSpecError::UnmappedAggregationType(AggregationType::HLL).to_string(), - "Unknown aggregation_type 'HLL'" - ); - } - - #[test] - fn sample_update_rule_decodes_legacy_parameters_once() { - let mut parameters = HashMap::new(); - parameters.insert("weight_mode".into(), Value::String("counter_delta".into())); - parameters.insert("weight_scale".into(), Value::from(1_000_000.0)); - let counter = make_config( - AggregationType::CountMinSketchWithHeap, - "", - parameters, - vec![], - ); - assert_eq!( - counter.sample_update_rule(), - SampleUpdateRule::CounterDelta { scale: 1_000_000.0 } - ); - - let mut count_parameters = HashMap::new(); - count_parameters.insert("topk_weight".into(), Value::String("frequency".into())); - let count = make_config( - AggregationType::CountSketchWithHeap, - "", - count_parameters, - vec![], - ); - assert_eq!(count.sample_update_rule(), SampleUpdateRule::Count); - } - - // ---- PolicyFingerprint stability guard --------------------------- - - /// `accumulator_spec()` must be a pure, additional *read* of - /// `PrecomputeMaterialization` — it must not change what - /// `PolicyFingerprint::from_config` hashes. This locks in a fixed - /// fingerprint for a fixed config as a tripwire: if this test ever - /// needs its expected constant updated, `policy_fingerprint.rs` - /// changed in a way that breaks on-disk sid compatibility, which is - /// exactly what Step 5 was required not to do. - #[test] - fn accumulator_spec_does_not_perturb_policy_fingerprint() { - let cfg = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - let fp_before = cfg.policy_fp_u64(); - let _ = cfg.accumulator_spec(); // takes &self — must not mutate `cfg` - let fp_after = cfg.policy_fp_u64(); - assert_eq!( - fp_before, fp_after, - "calling accumulator_spec() must not change the fingerprint \ - PolicyFingerprint::from_config computes from this config" - ); - - // An independently-built config with identical content must - // still agree — proves accumulator_spec() reads, never writes, - // the fields PolicyFingerprint::from_config hashes. - let cfg2 = make_config(AggregationType::DDSketch, "", HashMap::new(), vec![]); - assert_eq!(fp_after, cfg2.policy_fp_u64()); - } -} diff --git a/crates/asap_types/src/aggregation_config.rs b/crates/asap_types/src/aggregation_config.rs index f256d74fd..a2ce6dd08 100644 --- a/crates/asap_types/src/aggregation_config.rs +++ b/crates/asap_types/src/aggregation_config.rs @@ -1,13 +1,8 @@ use serde::{Deserialize, Serialize}; -use serde_json::Value; -use serde_yaml; -use std::collections::HashMap; -use crate::enums::{QueryLanguage, WindowKind}; +use crate::enums::WindowKind; use crate::policy_fingerprint::PolicyFingerprint; -use crate::utils::normalize_spatial_filter; use crate::AggregationType; -use crate::KeyByLabelNames; /// Physical maintenance layout for one semantic windowed summary. /// @@ -86,21 +81,19 @@ impl WindowMaterializationLayout { } } -/// Physical materialization metadata with content-derived identity. -/// This descriptor cannot authorize execution: the enclosing PrecomputePlan -/// must bind it to a compatible Planner DAG producer. +/// Deployment metadata for one stored output: its identity, source binding, +/// window cadence and layout, retention and storage codec. What the output +/// computes (state family, update, input predicate and reduction) is defined +/// only by the Planner DAG node the enclosing PrecomputePlan binds to it. #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct PrecomputeMaterialization { - /// Explicit deployment output allocation; independent of semantic identity. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub stored_output_id: Option, + /// Deployment output allocation. Several outputs may share one semantic + /// definition; see [`Self::allocate_stored_output_id`]. + pub stored_output_id: crate::sds::StoredOutputId, /// Planner-selected dependency closure ending at the persisted output. #[serde(default, skip_serializing_if = "Option::is_none")] pub semantic_fragment: Option, - pub aggregation_type: AggregationType, - pub aggregation_sub_type: String, - pub parameters: HashMap, #[serde(serialize_with = "crate::grouping_projection::serialize_config_grouping")] pub grouping_labels: crate::GroupingProjection, #[serde( @@ -112,9 +105,6 @@ pub struct PrecomputeMaterialization { pub partitioning: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub derived_input: Option, - pub aggregated_labels: KeyByLabelNames, - pub rollup_labels: KeyByLabelNames, - pub original_yaml: String, pub window_size: u64, // Window size in seconds (e.g., 900s for 15m) pub slide_interval: u64, // Slide/hop interval in seconds (e.g., 30s) @@ -126,8 +116,6 @@ pub struct PrecomputeMaterialization { #[serde(default, skip_serializing_if = "Option::is_none")] pub pane_origin_ms: Option, - pub spatial_filter: String, - pub spatial_filter_normalized: String, pub metric: String, // PromQL mode: metric name; SQL mode: derived from table_name.value_column pub num_aggregates_to_retain: Option, @@ -145,13 +133,11 @@ pub struct PrecomputeMaterialization { /// /// **Why this is separate from [`Self::value_projection`]**: the /// projection is the materialization's *identity* — which column or - /// constant is summarised, and part of the policy fingerprint. This is how - /// the ingest path must *read* that column, which the identity does not - /// determine: an integer column needs an exactness guard on its way into - /// f64 summary state, and a nullable column needs SQL's "aggregates skip - /// NULL" rule reproduced at the reader rather than a decode failure on the - /// first NULL row. Two materializations over the same column are the same - /// policy either way, so this deliberately stays out of the fingerprint. + /// constant is summarised. This is how the ingest path must *read* that + /// column, which the identity does not determine: an integer column needs + /// an exactness guard on its way into f64 summary state, and a nullable + /// column needs SQL's "aggregates skip NULL" rule reproduced at the reader + /// rather than a decode failure on the first NULL row. /// /// `None` ⇒ PromQL-mode materializations and legacy SQL definitions, which /// keep the pre-typed behaviour (read the column as it comes). @@ -163,17 +149,11 @@ pub struct PrecomputeMaterialization { /// query. For single-population queries, key and value share the same /// fingerprint and type. For multi-population queries (e.g. Topk), they /// differ. -/// -/// **PR 5**: the `aggregation_id_for_*: u64` fields now carry the -/// `PolicyFingerprint::as_u64()` form of the matched config, NOT a -/// controller-allocated id. Callers that need typed identity can call -/// [`AggregationIdInfo::policy_fp_for_key`] / -/// [`AggregationIdInfo::policy_fp_for_value`]. #[derive(Debug, Clone)] pub struct AggregationIdInfo { - /// `PolicyFingerprint::as_u64()` of the key aggregation's config. + /// Stored output id of the key aggregation. pub key_policy_fingerprint: u64, - /// `PolicyFingerprint::as_u64()` of the value aggregation's config. + /// Stored output id of the value aggregation. pub value_policy_fingerprint: u64, pub aggregation_type_for_key: AggregationType, pub aggregation_type_for_value: AggregationType, @@ -182,6 +162,43 @@ pub struct AggregationIdInfo { impl AggregationIdInfo {} impl PrecomputeMaterialization { + /// An unallocated output over `metric`; the caller must allocate its id + /// with [`Self::allocate_stored_output_id`] once the deployment fields are final. + pub fn new( + metric: impl Into, + grouping_labels: impl Into, + window_size: u64, + slide_interval: u64, + window_type: WindowKind, + ) -> Self { + Self { + stored_output_id: crate::sds::StoredOutputId(0), + semantic_fragment: None, + grouping_labels: grouping_labels.into(), + population_key_encoding: Default::default(), + partitioning: None, + derived_input: None, + window_size, + slide_interval, + window_type, + window_layout: WindowMaterializationLayout::Pane { + pane_secs: if slide_interval == 0 { + window_size + } else { + slide_interval + }, + }, + pane_origin_ms: None, + metric: metric.into(), + num_aggregates_to_retain: None, + table_name: None, + value_projection: None, + table_timestamp_column: None, + table_population: None, + value_source_column: None, + } + } + pub fn effective_value_projection(&self) -> &crate::sds::ValueProjectionIdentity { self.value_projection .as_ref() @@ -214,13 +231,15 @@ impl PrecomputeMaterialization { } } - pub fn population_filter_canonical(&self) -> Result { + /// Validate the source binding. Returns the canonical typed table + /// population, which is part of the table source binding; a time-series + /// predicate belongs to the Planner DAG's scan. + pub fn table_population_canonical(&self) -> Result { if let Some(input) = &self.derived_input { input.validate()?; if self.table_name.is_some() || self.table_population.is_some() || self.table_timestamp_column.is_some() - || !self.spatial_filter.is_empty() { return Err("derived inputs cannot also declare a raw source/filter".into()); } @@ -238,78 +257,15 @@ impl PrecomputeMaterialization { } crate::table_population::validate_column_name(column)?; } - if self.table_name.is_some() && !self.spatial_filter.is_empty() { - return Err("table populations cannot use a PromQL label filter".into()); - } - if let Some(population) = &self.table_population { - if self.table_name.is_none() || !self.spatial_filter.is_empty() { - return Err( - "typed table population requires a table and no PromQL label filter".into(), - ); + match &self.table_population { + Some(population) => { + if self.table_name.is_none() { + return Err("typed table population requires a table".into()); + } + population.validate()?; + Ok(population.canonical()) } - population.validate()?; - Ok(population.canonical()) - } else { - Ok(normalize_spatial_filter(&self.spatial_filter)) - } - } - - #[allow(clippy::too_many_arguments)] - pub fn new( - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: HashMap, - grouping_labels: impl Into, - aggregated_labels: KeyByLabelNames, - rollup_labels: KeyByLabelNames, - original_yaml: String, - window_size: u64, - slide_interval: u64, - window_type: WindowKind, - spatial_filter: String, - metric: String, - num_aggregates_to_retain: Option, - // SQL-specific fields - table_name: Option, - value_column: Option, - ) -> Self { - // Generate normalized spatial filter (placeholder implementation) - let spatial_filter_normalized = normalize_spatial_filter(&spatial_filter); - - Self { - stored_output_id: None, - semantic_fragment: None, - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels: grouping_labels.into(), - population_key_encoding: Default::default(), - partitioning: None, - derived_input: None, - aggregated_labels, - rollup_labels, - original_yaml, - window_size, - slide_interval, - window_type, - window_layout: WindowMaterializationLayout::Pane { - pane_secs: if slide_interval == 0 { - window_size - } else { - slide_interval - }, - }, - pane_origin_ms: None, - spatial_filter, - spatial_filter_normalized, - metric, - num_aggregates_to_retain, - table_name, - value_projection: value_column - .map(|name| crate::sds::ValueProjectionIdentity::Column { name }), - table_population: None, - table_timestamp_column: None, - value_source_column: None, + None => Ok(String::new()), } } @@ -319,223 +275,127 @@ impl PrecomputeMaterialization { PolicyFingerprint::from_config(self) } - /// `PolicyFingerprint::as_u64()` — the u64-form handle used by the - /// policy-fingerprint-keyed call sites (e.g. `InstalledPrecomputePlan`'s - /// `HashMap` keys). **Always** equal to - /// `self.policy_fingerprint().as_u64()`. The value is content- - /// addressed identity, NOT a controller-allocated counter id. + /// The stored output id as a raw u64. pub fn policy_fp_u64(&self) -> u64 { self.policy_fingerprint().as_u64() } - pub fn from_yaml_data( - aggregation_data: &serde_yaml::Value, - num_aggregates_to_retain: Option, - query_language: QueryLanguage, - ) -> Result { - // `aggregationId` is silently dropped — identity is - // content-addressed via PolicyFingerprint (PR 5). Pre-PR-5 - // fixtures that still spell out the field parse cleanly. + /// Allocate the deterministic default id from the current deployment + /// fields and `computation`, the producer's computation identity taken + /// from the Planner DAG (state family, update, input predicate). Outputs + /// with equal deployment fields and computation share one id; any + /// difference, including pane layout or phase, yields a distinct output. + pub fn allocate_stored_output_id(&mut self, computation: &impl Serialize) { + let computation = crate::semantic_fragment::canonical_bytes(computation) + .expect("computation identity serializes"); + let id = self.deployment_digest(&computation); + // Zero is the unallocated sentinel. + self.stored_output_id = crate::sds::StoredOutputId(id.max(1)); + } - let labels = &aggregation_data["labels"]; - let grouping_labels: crate::GroupingProjection = - serde_yaml::from_value(labels["grouping"].clone())?; - let aggregated_labels = KeyByLabelNames::new( - labels["aggregated"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing aggregated labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), + fn deployment_digest(&self, computation: &[u8]) -> u64 { + use xxhash_rust::xxh64::xxh64; + let mut buf: Vec = Vec::with_capacity(512); + if !self.population_key_encoding.is_legacy() { + // UTF-8 raw metrics cannot alias this nonlegacy domain prefix. + buf.extend_from_slice(b"\xffpopulation-key-canonical-labels-v1\0"); + } + if self.derived_input.is_some() { + // Raw policies start with UTF-8 metric bytes; 0xff is impossible + // there, so a metric cannot impersonate this source domain. + buf.extend_from_slice(b"\xffderived-input-v1:"); + buf.extend_from_slice( + &serde_json::to_vec(&self.source_identity()).expect("typed source identity"), + ); + } else { + buf.extend_from_slice(self.metric.as_bytes()); + } + buf.push(0); + buf.extend_from_slice(computation); + buf.push(0); + if let Some(partitioning) = self.partitioning { + buf.extend_from_slice(format!("partition:{partitioning:?}\0").as_bytes()); + } + for label in &self.grouping_labels.names() { + buf.extend_from_slice(label.as_bytes()); + buf.push(b','); + } + buf.push(0); + if self.table_name.is_some() && !self.grouping_labels.is_empty() { + buf.extend_from_slice( + crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS.as_bytes(), + ); + buf.push(0); + } + if !self.grouping_labels.is_legacy_labels() { + buf.extend_from_slice(b"typed-grouping:"); + buf.extend_from_slice( + serde_json::to_string(&self.grouping_labels) + .expect("group projection serializes") + .as_bytes(), + ); + buf.push(0); + } + buf.extend_from_slice(&self.window_size.to_le_bytes()); + buf.push(0); + buf.extend_from_slice(&self.slide_interval.to_le_bytes()); + buf.push(0); + buf.extend_from_slice( + serde_json::to_string(&self.window_type) + .unwrap_or_default() + .as_bytes(), ); - let rollup_labels = KeyByLabelNames::new( - labels["rollup"] - .as_sequence() - .ok_or_else(|| anyhow::anyhow!("Missing rollup labels"))? - .iter() - .filter_map(|v| v.as_str()) - .map(|s| s.to_string()) - .collect(), + buf.push(0); + // A full overlapping window and a mergeable pane layout may share + // semantic descriptors but never share payload instances or lifecycle + // accounting. + buf.extend_from_slice( + serde_json::to_string(&self.window_layout) + .unwrap_or_default() + .as_bytes(), ); - - let aggregation_type: AggregationType = aggregation_data["aggregationType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationType"))? - .parse() - .map_err(|e: String| anyhow::anyhow!(e))?; - - let aggregation_sub_type = aggregation_data["aggregationSubType"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing aggregationSubType"))? - .to_string(); - - // Convert serde_yaml::Value to serde_json::Value for parameters - let parameters: HashMap = aggregation_data["parameters"] - .as_mapping() - .ok_or_else(|| anyhow::anyhow!("Missing parameters"))? - .iter() - .map(|(k, v)| { - let key = k.as_str().unwrap_or("").to_string(); - let value = serde_json::to_value(v).unwrap_or(Value::Null); - (key, value) - }) - .collect(); - - let window_size = aggregation_data["windowSize"] - .as_u64() - .ok_or_else(|| anyhow::anyhow!("Missing windowSize"))?; - - let window_type = aggregation_data - .get("windowType") - .and_then(|v| v.as_str()) - .unwrap_or("tumbling") - .parse::() - .unwrap_or_default(); - - let slide_interval = aggregation_data - .get("slideInterval") - .and_then(|v| v.as_u64()) - .unwrap_or(window_size); - - let pane_origin_ms = aggregation_data - .get("paneOriginMs") - .or_else(|| aggregation_data.get("pane_origin_ms")) - .and_then(|v| v.as_i64()); - - let spatial_filter = aggregation_data["spatialFilter"] - .as_str() - .unwrap_or("") - .to_string(); - - if [ - "valueColumn", - "value_column", - "valueProjection", - "value_projection", - ] - .iter() - .filter(|key| { - aggregation_data - .get(**key) - .is_some_and(|value| !value.is_null()) - }) - .count() - > 1 - { - return Err(anyhow::anyhow!( - "multiple value projection fields are not allowed" - )); + buf.push(0); + // Presence is explicit so an unknown phase cannot alias an + // epoch-aligned definition. + match self.pane_origin_ms { + Some(origin) => { + buf.push(1); + buf.extend_from_slice(&origin.to_le_bytes()); + } + None => buf.push(0), } - let typed_projection: Option = aggregation_data - .get("valueProjection") - .or_else(|| aggregation_data.get("value_projection")) - .filter(|value| !value.is_null()) - .map(|value| serde_json::to_value(value).and_then(serde_json::from_value)) - .transpose()?; - let (metric, table_name, value_column) = match query_language { - QueryLanguage::PromQl | QueryLanguage::MetricsQl => { - let metric = aggregation_data["metric"] - .as_str() - .ok_or_else(|| { - anyhow::anyhow!("Missing metric for time-series query language") - })? - .to_string(); - (metric, None, None) + buf.push(0); + if let Some(table) = &self.table_name { + buf.extend_from_slice(b"\0sql-source-v1\0"); + buf.extend_from_slice(table.as_bytes()); + buf.push(0); + if let Some(column) = self.effective_value_projection().column() { + buf.extend_from_slice(column.as_bytes()); } - QueryLanguage::ClickHouseSql => { - let table = aggregation_data["tableName"] - .as_str() - .ok_or_else(|| anyhow::anyhow!("Missing tableName for ClickHouse SQL"))? - .to_string(); - let column = aggregation_data["valueColumn"] - .as_str() - .or_else(|| { - typed_projection - .as_ref() - .and_then(|projection| projection.column()) - }) - .map(str::to_owned); - if column.is_none() && typed_projection.is_none() { - return Err(anyhow::anyhow!( - "Missing value projection for ClickHouse SQL" - )); - } - ( - format!("{table}.{}", column.as_deref().unwrap_or("constant")), - Some(table), - column, - ) + if matches!( + self.effective_value_projection(), + crate::sds::ValueProjectionIdentity::Constant { .. } + ) { + buf.extend_from_slice(b"\0constant-projection-v1\0"); + buf.extend_from_slice( + serde_json::to_string(self.effective_value_projection()) + .expect("finite validated projection serializes") + .as_bytes(), + ); } - }; - - let mut config = Self::new( - aggregation_type, - aggregation_sub_type, - parameters, - grouping_labels, - aggregated_labels, - rollup_labels, - String::new(), // original_yaml - empty as in Python - window_size, - slide_interval, - window_type, - spatial_filter, - metric, - num_aggregates_to_retain, - table_name, - value_column, - ); - if aggregation_data.get("windowLayout").is_some() - && aggregation_data.get("window_layout").is_some() - { - anyhow::bail!("multiple window layout fields are not allowed"); } - if let Some(layout) = aggregation_data - .get("windowLayout") - .or_else(|| aggregation_data.get("window_layout")) - { - config.window_layout = serde_yaml::from_value(layout.clone())?; - config - .window_layout - .validate(config.window_size, config.slide_interval) - .map_err(anyhow::Error::msg)?; + if let Some(population) = &self.table_population { + let canonical = population.canonical(); + if !canonical.is_empty() { + buf.push(0); + buf.extend_from_slice(canonical.as_bytes()); + } } - config.population_key_encoding = aggregation_data - .get("population_key_encoding") - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()? - .unwrap_or_default(); - config.derived_input = aggregation_data - .get("derived_input") - .filter(|v| !v.is_null()) - .map(|v| serde_yaml::from_value(v.clone())) - .transpose()?; - config.partitioning = aggregation_data - .get("partitioning") - .filter(|value| !value.is_null()) - .map(|value| serde_yaml::from_value(value.clone())) - .transpose()?; - if let Some(projection) = typed_projection { - config.value_projection = Some(projection); + if let Some(column) = &self.table_timestamp_column { + buf.extend_from_slice(b"\0timestamp-ms\0"); + buf.extend_from_slice(column.as_bytes()); } - config.pane_origin_ms = pane_origin_ms; - config.table_timestamp_column = aggregation_data - .get("tableTimestampColumn") - .or_else(|| aggregation_data.get("table_timestamp_column")) - .and_then(serde_yaml::Value::as_str) - .map(str::to_owned); - config.table_population = aggregation_data - .get("tablePopulation") - .or_else(|| aggregation_data.get("table_population")) - .filter(|value| !value.is_null()) - .cloned() - .map(serde_yaml::from_value) - .transpose()?; - config - .population_filter_canonical() - .map_err(anyhow::Error::msg)?; - Ok(config) + xxh64(&buf, 0) } } @@ -575,68 +435,22 @@ mod window_layout_tests { mod tests { use super::*; - fn sample_yaml(with_id: bool) -> serde_yaml::Value { - let id_line = if with_id { "aggregationId: 42\n" } else { "" }; - let yaml = format!( - "{id_line}aggregationType: DDSketch\naggregationSubType: ''\nmetric: http_latency_ms\nlabels:\n grouping: [zone]\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n", - id_line = id_line - ); - serde_yaml::from_str(&yaml).expect("yaml parses") - } - - /// PR 5: `aggregationId` in the YAML is silently dropped — identity - /// is derived from content. A fixture with the field parses to the - /// SAME config as a fixture without it. - #[test] - fn explicit_aggregation_id_in_yaml_is_ignored() { - let with = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(true), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - let without = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse ok"); - assert_eq!( - with.policy_fingerprint(), - without.policy_fingerprint(), - "explicit aggregationId in the YAML must not change the policy fingerprint", - ); - } - - /// Round-tripping the same content yields the same fingerprint. - #[test] - fn fingerprint_is_deterministic_per_content() { - let a = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse a"); - let b = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse b"); - assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); - assert_ne!( - a.policy_fingerprint().as_u64(), - 0, - "fingerprint is never the 0 sentinel for a real config", + fn config() -> PrecomputeMaterialization { + let mut config = PrecomputeMaterialization::new( + "http_latency_ms", + crate::KeyByLabelNames::new(vec!["zone".into()]), + 60, + 10, + WindowKind::Sliding, ); + config.allocate_stored_output_id(&"ddsketch"); + config } + // Window layout and pane origin survive the canonical JSON wire, and + // derived-style spellings are rejected. #[test] - fn explicit_window_layout_survives_custom_json_and_yaml_transport() { - use super::WindowMaterializationLayout; - let mut yaml = sample_yaml(false); - yaml["windowSize"] = serde_yaml::to_value(60).unwrap(); - yaml["slideInterval"] = serde_yaml::to_value(10).unwrap(); + fn window_layout_and_pane_origin_survive_canonical_json() { for layout in [ WindowMaterializationLayout::FullWindow, WindowMaterializationLayout::Pane { pane_secs: 5 }, @@ -645,110 +459,39 @@ mod tests { levels_secs: vec![10, 30], }, ] { - yaml["windowLayout"] = serde_yaml::to_value(&layout).unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl) - .unwrap(); - assert_eq!(config.window_layout, layout); + let mut config = config(); + config.window_layout = layout.clone(); + config.pane_origin_ms = Some(7_000); let mut wire = serde_json::to_value(&config).unwrap(); let decoded: PrecomputeMaterialization = serde_json::from_value(wire.clone()).unwrap(); assert_eq!(decoded.window_layout, layout); + assert_eq!(decoded.pane_origin_ms, Some(7_000)); assert_eq!(decoded.stored_window_ms(), config.stored_window_ms()); assert_eq!(decoded.policy_fingerprint(), config.policy_fingerprint()); wire["windowLayout"] = wire["window_layout"].clone(); assert!(serde_json::from_value::(wire).is_err()); } - yaml.as_mapping_mut() - .unwrap() - .remove(serde_yaml::Value::from("windowLayout")); - let legacy = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).unwrap(); - assert_eq!( - legacy.window_layout, - WindowMaterializationLayout::Pane { pane_secs: 10 } - ); - yaml["window_layout"] = serde_yaml::from_str("{kind: pane, pane_secs: 7}").unwrap(); - assert!( - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::PromQl).is_err() - ); } - #[test] - fn pane_origin_round_trips_and_changes_definition_identity() { - let mut epoch = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let unknown = epoch.policy_fingerprint(); - epoch.pane_origin_ms = Some(7_000); - let planned = epoch.policy_fingerprint(); - assert_ne!(unknown, planned); - - let wire = serde_json::to_value(&epoch).unwrap(); - assert_eq!(wire["pane_origin_ms"], serde_json::json!(7_000)); - let mut derived = serde_json::to_value(&epoch).unwrap(); - let origin = derived - .as_object_mut() - .unwrap() - .remove("pane_origin_ms") - .unwrap(); - derived - .as_object_mut() - .unwrap() - .insert("paneOriginMs".into(), origin); - assert!(serde_json::from_value::(derived).is_err()); - let decoded: PrecomputeMaterialization = - serde_json::from_value(serde_json::to_value(&epoch).unwrap()).unwrap(); - assert_eq!(decoded.pane_origin_ms, Some(7_000)); - } - - /// The `policy_fp_u64()` accessor is exactly the fingerprint u64. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - assert_eq!(cfg.policy_fp_u64(), cfg.policy_fingerprint().as_u64()); - } - - /// Installed materializations reject the removed externally assigned identity. + // The removed externally assigned identity is not accepted on the wire. #[test] fn canonical_wire_rejects_aggregation_id() { - let cfg = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .expect("parse"); - let mut json = serde_json::to_value(&cfg).unwrap(); - assert!( - json.get("aggregationId").is_none(), - "aggregationId must not appear in the canonical wire format" - ); + let mut json = serde_json::to_value(config()).unwrap(); + assert!(json.get("aggregationId").is_none()); json["aggregationId"] = serde_json::json!(42); assert!(serde_json::from_value::(json).is_err()); } + // Table value projections are typed, and a non-finite constant is invalid. #[test] fn typed_projection_roundtrips_and_untyped_column_is_rejected() { use crate::sds::ValueProjectionIdentity; use planner_types::pre_asap::ScalarValue; - let mut config = PrecomputeMaterialization::from_yaml_data( - &sample_yaml(false), - None, - QueryLanguage::PromQl, - ) - .unwrap(); + let mut config = config(); config.table_name = Some("telemetry".into()); config.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), }); - let column_identity = config.policy_fingerprint(); let mut legacy = serde_json::to_value(&config).unwrap(); legacy.as_object_mut().unwrap().remove("value_projection"); legacy["value_column"] = serde_json::json!("value"); @@ -759,34 +502,16 @@ mod tests { config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Int64(1), }); - let wire = serde_json::to_value(&config).unwrap(); - let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); - assert_eq!( - decoded.effective_value_projection(), - config.effective_value_projection() - ); - let mut yaml = sample_yaml(false); - yaml["tableName"] = serde_yaml::to_value("telemetry").unwrap(); - yaml["valueProjection"] = - serde_yaml::to_value(serde_json::to_value(&config.value_projection).unwrap()).unwrap(); - let decoded = - PrecomputeMaterialization::from_yaml_data(&yaml, None, QueryLanguage::ClickHouseSql) - .unwrap(); + let decoded: PrecomputeMaterialization = + serde_json::from_value(serde_json::to_value(&config).unwrap()).unwrap(); assert_eq!( decoded.effective_value_projection(), config.effective_value_projection() ); - yaml["valueColumn"] = serde_yaml::to_value("other_column").unwrap(); - assert!(PrecomputeMaterialization::from_yaml_data( - &yaml, - None, - QueryLanguage::ClickHouseSql - ) - .is_err()); - assert_ne!(config.policy_fingerprint(), column_identity); + assert!(config.table_population_canonical().is_ok()); config.value_projection = Some(ValueProjectionIdentity::Constant { value: ScalarValue::Float64(f64::NAN), }); - assert!(config.population_filter_canonical().is_err()); + assert!(config.table_population_canonical().is_err()); } } diff --git a/crates/asap_types/src/aggregation_type.rs b/crates/asap_types/src/aggregation_type.rs index ebcc60d21..71d70e615 100644 --- a/crates/asap_types/src/aggregation_type.rs +++ b/crates/asap_types/src/aggregation_type.rs @@ -1,2 +1,36 @@ //! Kernel identity of backend-stored summary state. pub use asap_summary_state::AggregationType; + +use planner_types::post_asap::{ExactKind, GroupingStrategy, SketchAlgorithm, SummaryFamilyType}; + +/// The backend kernel tag that stores state of the Planner `family`. `None` +/// for families the backend has no stored-state kernel for. +pub fn aggregation_type_for_family(family: &SummaryFamilyType) -> Option { + Some(match family { + SummaryFamilyType::ExactAggregate(kind, _) => match kind { + ExactKind::Sum => AggregationType::Sum, + ExactKind::Count => AggregationType::Count, + ExactKind::Min => AggregationType::Min, + ExactKind::Max => AggregationType::Max, + ExactKind::Increase => AggregationType::Increase, + ExactKind::Rate => AggregationType::Rate, + ExactKind::IRate => return None, + }, + SummaryFamilyType::Sketch(kind, grouping) => match (kind.algorithm(), grouping) { + (SketchAlgorithm::Kll, GroupingStrategy::SharedMultiSubpopulation { .. }) => { + AggregationType::HydraKLL + } + (_, GroupingStrategy::SharedMultiSubpopulation { .. }) => return None, + (SketchAlgorithm::Kll, _) => AggregationType::DatasketchesKLL, + (SketchAlgorithm::Cms, _) => AggregationType::CountMinSketch, + (SketchAlgorithm::CmsWithHeap, _) => AggregationType::CountMinSketchWithHeap, + (SketchAlgorithm::CountSketch, _) => AggregationType::CountSketch, + (SketchAlgorithm::CountSketchWithHeap, _) => AggregationType::CountSketchWithHeap, + (SketchAlgorithm::DDSketch, _) => AggregationType::DDSketch, + (SketchAlgorithm::Hll, _) => AggregationType::HLL, + (SketchAlgorithm::UnivMon, _) => AggregationType::UnivMon, + (SketchAlgorithm::Kmv | SketchAlgorithm::Theta, _) => return None, + }, + _ => return None, + }) +} diff --git a/crates/asap_types/src/derived_input.rs b/crates/asap_types/src/derived_input.rs index ffd1edea0..d7a79de6a 100644 --- a/crates/asap_types/src/derived_input.rs +++ b/crates/asap_types/src/derived_input.rs @@ -138,25 +138,40 @@ impl DerivedInputIdentity { mod tests { use super::*; use crate::summary_catalog::SummaryCatalog; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use crate::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; + + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + } + + fn allocated(mut config: PrecomputeMaterialization) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&"sum"); + config + } fn config() -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + allocated(PrecomputeMaterialization::new( + "m", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, + )) + } + + fn catalog( + plan_version: u64, + outputs: &[PrecomputeMaterialization], + ) -> Result { + let family = sum(); + SummaryCatalog::from_outputs( + 1, + plan_version, + outputs + .iter() + .map(|c| (c, &family, String::new())) + .collect(), ) } @@ -169,15 +184,18 @@ mod tests { inputs: BTreeSet::from([raw_id]), program_sha256: "a".repeat(64), }); + let derived = allocated(derived); assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); - let a = - SummaryCatalog::from_materializations(1, 1, &[raw.clone(), derived.clone()]).unwrap(); - let b = SummaryCatalog::from_materializations(2, 9, &[raw, derived.clone()]).unwrap(); + let a = catalog(1, &[raw.clone(), derived.clone()]).unwrap(); + let b = catalog(9, &[raw, derived.clone()]).unwrap(); assert_eq!(a.outputs, b.outputs); assert_eq!(a.data_descriptors, b.data_descriptors); let mut renamed = derived.clone(); renamed.metric = "output_alias".into(); - assert_eq!(renamed.policy_fingerprint(), derived.policy_fingerprint()); + assert_eq!( + allocated(renamed).policy_fingerprint(), + derived.policy_fingerprint() + ); let json = serde_json::to_value(&derived).unwrap(); let decoded: PrecomputeMaterialization = serde_json::from_value(json).unwrap(); assert_eq!(decoded.policy_fingerprint(), derived.policy_fingerprint()); @@ -190,13 +208,17 @@ mod tests { inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "d".repeat(64), }); + let derived = allocated(derived); let mut raw = derived.clone(); raw.metric = format!( "derived-input-v1:{}", serde_json::to_string(&derived.source_identity()).unwrap() ); raw.derived_input = None; - assert_ne!(raw.policy_fingerprint(), derived.policy_fingerprint()); + assert_ne!( + allocated(raw).policy_fingerprint(), + derived.policy_fingerprint() + ); } #[test] @@ -206,9 +228,11 @@ mod tests { inputs: BTreeSet::from([StoredOutputId::from(derived.policy_fingerprint())]), program_sha256: "b".repeat(64), }); - assert!(SummaryCatalog::from_materializations(1, 1, &[derived.clone()]).is_err()); + let derived = allocated(derived); + assert!(catalog(1, std::slice::from_ref(&derived)).is_err()); + let mut derived = derived; derived.table_name = Some("table".into()); - assert!(derived.population_filter_canonical().is_err()); + assert!(derived.table_population_canonical().is_err()); } fn program(source: u32, root: u32) -> OwnedPostAsapDag { use crate::executable_plan::{OwnedPostAsapEdge, OwnedPostAsapNode}; @@ -322,7 +346,7 @@ mod tests { 1, [( config.policy_fingerprint(), - SummaryDescriptor::from_config(&config).unwrap(), + SummaryDescriptor::from_family(&sum()).unwrap(), data, )] ) @@ -342,12 +366,16 @@ mod tests { generated_at_unix_ms: 0, activation_unix_ms: 0, expiry_unix_ms: None, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: crate::precompute_plan::BACKEND_COMPAT.into(), planner_revision: "test".into(), capability_snapshot_id: "test".into(), }; - let error = - PrecomputePlan::build(envelope, vec![config], &["producer".into()]).unwrap_err(); + let error = PrecomputePlan::build( + envelope, + vec![(allocated(config), sum())], + &["producer".into()], + ) + .unwrap_err(); assert!(error .to_string() .contains("installed aligned immutable maintenance DAG")); diff --git a/crates/asap_types/src/lib.rs b/crates/asap_types/src/lib.rs index 2275c0a4a..7c19837bd 100644 --- a/crates/asap_types/src/lib.rs +++ b/crates/asap_types/src/lib.rs @@ -12,10 +12,8 @@ pub mod monitor_spec; pub mod physical_plan_codec; pub mod plan_publication; pub mod policy_fingerprint; -pub mod policy_registry; pub mod producer_plan; pub mod query_requirements; -pub mod routing_index; pub mod sds; pub mod semantic_fragment; pub mod storage_backend; @@ -24,18 +22,15 @@ pub mod summary_semantics; pub mod table_population; pub mod utils; -pub use accumulator_spec::{AccumulatorSpec, AccumulatorSpecError, SampleUpdateRule}; pub use accuracy::{AccuracyKind, AccuracyProfile}; pub use aggregation_config::*; -pub use aggregation_type::AggregationType; +pub use aggregation_type::{aggregation_type_for_family, AggregationType}; pub use enums::*; pub use grouping_projection::{GroupingProjection, PopulationKeyEncoding}; pub use key_by_label_names::KeyByLabelNames; pub use monitor_spec::{MonitorFunctional, MonitorSpec}; pub use policy_fingerprint::PolicyFingerprint; -pub use policy_registry::PolicyRegistry; pub use query_requirements::*; -pub use routing_index::RoutingIndex; pub use storage_backend::*; pub mod precompute_plan; diff --git a/crates/asap_types/src/policy_fingerprint.rs b/crates/asap_types/src/policy_fingerprint.rs index 42fa2e475..69060cdcf 100644 --- a/crates/asap_types/src/policy_fingerprint.rs +++ b/crates/asap_types/src/policy_fingerprint.rs @@ -1,16 +1,11 @@ //! Legacy routing wrapper for a deployed stored output. //! -//! An explicit `PrecomputeMaterialization::stored_output_id` takes precedence. -//! Otherwise the compiler allocates a deterministic default from the existing -//! policy fields (including pane layout and cadence). This identifier is not -//! semantic identity: `SummaryDefinitionId` hashes the versioned semantic -//! definition, and several deployed outputs may share that definition. -//! Catalog installation checks that an output is never assigned conflicting -//! computation or format contracts. +//! The value is `PrecomputeMaterialization::stored_output_id`, allocated by the +//! compiler from the deployment fields and the Planner producer's computation +//! identity. It is not semantic identity: `SummaryDefinitionId` hashes the +//! versioned semantic definition, and several deployed outputs may share it. use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use xxhash_rust::xxh64::xxh64; use crate::aggregation_config::PrecomputeMaterialization; @@ -22,178 +17,17 @@ use crate::aggregation_config::PrecomputeMaterialization; pub struct PolicyFingerprint(pub u64); impl PolicyFingerprint { - /// Unset sentinel returned by `Default`. Sinks cannot resolve an unset policy; - /// construct valid fingerprints with [`Self::from_config`]. + /// Unset sentinel returned by `Default`. Sinks cannot resolve an unset policy. pub const UNSET: PolicyFingerprint = PolicyFingerprint(0); /// True when this fingerprint is the [`Self::UNSET`] sentinel. pub fn is_unset(self) -> bool { self.0 == 0 } -} -impl PolicyFingerprint { - /// Compute the fingerprint of an [`PrecomputeMaterialization`]. - /// - /// Hash inputs are concatenated with `\0` byte separators and - /// canonicalized so that map/iteration order can't affect the - /// outcome. Parameter values are rendered via `serde_json::to_string` - /// for nested-shape determinism (matches the existing - /// `parameters_canonical` form used in `AggKind::ExactAgg`). + /// The stored output id of `cfg`. pub fn from_config(cfg: &PrecomputeMaterialization) -> Self { - if let Some(output) = cfg.stored_output_id { - return output.fingerprint(); - } - let mut buf: Vec = Vec::with_capacity(512); - - if !cfg.population_key_encoding.is_legacy() { - // UTF-8 raw metrics cannot alias this nonlegacy domain prefix. - buf.extend_from_slice(b"\xffpopulation-key-canonical-labels-v1\0"); - } - // 1. metric name - if cfg.derived_input.is_some() { - // Raw policies start with UTF-8 metric bytes; 0xff is impossible - // there, so a metric cannot impersonate this source domain. - buf.extend_from_slice(b"\xffderived-input-v1:"); - buf.extend_from_slice( - &serde_json::to_vec(&cfg.source_identity()).expect("typed source identity"), - ); - } else { - buf.extend_from_slice(cfg.metric.as_bytes()); - } - buf.push(0); - - // 2. aggregation_type (Serialize impl is the stable form) - buf.extend_from_slice( - serde_json::to_string(&cfg.aggregation_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 3. aggregation_sub_type - buf.extend_from_slice(cfg.aggregation_sub_type.as_bytes()); - buf.push(0); - - // 4. parameters — canonicalized (sorted keys, JSON-rendered values) - let sorted: BTreeMap<&String, &serde_json::Value> = cfg.parameters.iter().collect(); - for (k, v) in sorted { - buf.extend_from_slice(k.as_bytes()); - buf.push(b'='); - buf.extend_from_slice(serde_json::to_string(v).unwrap_or_default().as_bytes()); - buf.push(b';'); - } - buf.push(0); - - if let Some(partitioning) = cfg.partitioning { - buf.extend_from_slice(format!("partition:{partitioning:?}\0").as_bytes()); - } - - // 5. grouping_labels (already sorted at construction per - // KeyByLabelNames invariant; encode as `,`-joined list) - for l in &cfg.grouping_labels.names() { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - if cfg.table_name.is_some() && !cfg.grouping_labels.is_empty() { - buf.extend_from_slice( - crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS.as_bytes(), - ); - buf.push(0); - } - if !cfg.grouping_labels.is_legacy_labels() { - buf.extend_from_slice(b"typed-grouping:"); - buf.extend_from_slice( - serde_json::to_string(&cfg.grouping_labels) - .expect("group projection serializes") - .as_bytes(), - ); - buf.push(0); - } - - // 6. aggregated_labels - for l in &cfg.aggregated_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 7. rollup_labels - for l in &cfg.rollup_labels.labels { - buf.extend_from_slice(l.as_bytes()); - buf.push(b','); - } - buf.push(0); - - // 8. window_size + slide_interval + window_type (cadence) - buf.extend_from_slice(&cfg.window_size.to_le_bytes()); - buf.push(0); - buf.extend_from_slice(&cfg.slide_interval.to_le_bytes()); - buf.push(0); - buf.extend_from_slice( - serde_json::to_string(&cfg.window_type) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // Physical representation is part of state identity. A full overlapping - // window and a mergeable pane layout may share semantic descriptors but - // never share payload instances or lifecycle accounting. - buf.extend_from_slice( - serde_json::to_string(&cfg.window_layout) - .unwrap_or_default() - .as_bytes(), - ); - buf.push(0); - - // 9. pane origin. Presence is explicit so a legacy definition with - // unknown phase cannot alias an epoch-aligned definition. - match cfg.pane_origin_ms { - Some(origin) => { - buf.push(1); - buf.extend_from_slice(&origin.to_le_bytes()); - } - None => buf.push(0), - } - buf.push(0); - - // 10. spatial_filter_normalized — canonicalized predicate - buf.extend_from_slice(cfg.spatial_filter_normalized.as_bytes()); - if let Some(table) = &cfg.table_name { - buf.extend_from_slice(b"\0sql-source-v1\0"); - buf.extend_from_slice(table.as_bytes()); - buf.push(0); - if let Some(column) = cfg.effective_value_projection().column() { - buf.extend_from_slice(column.as_bytes()); - } - if matches!( - cfg.effective_value_projection(), - crate::sds::ValueProjectionIdentity::Constant { .. } - ) { - buf.extend_from_slice(b"\0constant-projection-v1\0"); - buf.extend_from_slice( - serde_json::to_string(cfg.effective_value_projection()) - .expect("finite validated projection serializes") - .as_bytes(), - ); - } - } - if let Some(population) = &cfg.table_population { - let canonical = population.canonical(); - if !canonical.is_empty() { - buf.push(0); - buf.extend_from_slice(canonical.as_bytes()); - } - } - if let Some(column) = &cfg.table_timestamp_column { - buf.extend_from_slice(b"\0timestamp-ms\0"); - buf.extend_from_slice(column.as_bytes()); - } - - Self(xxh64(&buf, 0)) + cfg.stored_output_id.fingerprint() } /// The raw u64. Use sparingly — prefer comparing `PolicyFingerprint` @@ -215,55 +49,40 @@ impl std::fmt::Display for PolicyFingerprint { mod tests { use super::*; use crate::enums::WindowKind; - use crate::AggregationType; use crate::KeyByLabelNames; - use std::collections::HashMap; - fn cfg( - metric: &str, - agg_type: AggregationType, - params: HashMap, - group_by: Vec<&str>, - window_size: u64, - spatial_filter: &str, - ) -> PrecomputeMaterialization { + fn cfg(metric: &str, group_by: Vec<&str>, window_size: u64) -> PrecomputeMaterialization { PrecomputeMaterialization::new( - agg_type, - String::new(), - params, + metric, KeyByLabelNames::new(group_by.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), window_size, window_size, WindowKind::Tumbling, - spatial_filter.to_string(), - metric.to_string(), - None, - None, - None, ) } + fn allocated( + mut config: PrecomputeMaterialization, + computation: &str, + ) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&computation); + config + } + + // The canonical wire keeps the legacy encoding implicit, and the + // population key encoding separates allocated output identities. #[test] fn population_encoding_preserves_legacy_wire_and_separates_identity() { use crate::grouping_projection::PopulationKeyEncoding; - let legacy = cfg( - "m", - AggregationType::Sum, - HashMap::new(), - vec!["host"], - 60, - "", - ); + let legacy = allocated(cfg("m", vec!["host"], 60), "sum"); let wire = serde_json::to_value(&legacy).unwrap(); assert!(wire.get("population_key_encoding").is_none()); let decoded: PrecomputeMaterialization = serde_json::from_value(wire).unwrap(); assert!(decoded.population_key_encoding.is_legacy()); assert_eq!(legacy.policy_fingerprint(), decoded.policy_fingerprint()); - let mut canonical = legacy.clone(); + let mut canonical = cfg("m", vec!["host"], 60); canonical.population_key_encoding = PopulationKeyEncoding::CanonicalLabelsV1; + let canonical = allocated(canonical, "sum"); assert_ne!(legacy.policy_fingerprint(), canonical.policy_fingerprint()); let wire = serde_json::to_value(&canonical).unwrap(); assert_eq!(wire["population_key_encoding"], "canonical_labels_v1"); @@ -271,234 +90,52 @@ mod tests { assert_eq!(decoded.policy_fingerprint(), canonical.policy_fingerprint()); } + // Equal deployment fields and computation allocate the same output. #[test] - fn same_config_yields_same_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn physical_window_layout_is_part_of_state_identity() { - let panes = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let mut full = panes.clone(); - full.window_layout = crate::WindowMaterializationLayout::FullWindow; - assert_ne!( - PolicyFingerprint::from_config(&panes), - PolicyFingerprint::from_config(&full) - ); - } - - #[test] - fn different_metric_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "cpu_pct", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_window_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 300, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_spatial_filter_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200""#, - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - #[test] - fn different_group_by_yields_different_fingerprint() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone"], - 60, - "", - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec!["zone", "service"], - 60, - "", - ); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - } - - /// Pre-PR-5 the `aggregation_id` field on `PrecomputeMaterialization` was - /// excluded from the fingerprint hash. PR 5 deletes the field - /// entirely — identity *is* the fingerprint — so this is now - /// vacuously true. Kept as a doc-comment anchor; no runtime test - /// is needed. - #[test] - fn policy_fp_u64_accessor_equals_fingerprint_u64() { - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - assert_eq!( - a.policy_fp_u64(), - PolicyFingerprint::from_config(&a).as_u64(), - ); + fn same_deployment_and_computation_yield_same_output() { + let a = allocated(cfg("http_lat", vec!["zone"], 60), "sum"); + let b = allocated(cfg("http_lat", vec!["zone"], 60), "sum"); + assert_eq!(a.policy_fingerprint(), b.policy_fingerprint()); + assert!(!a.policy_fingerprint().is_unset()); } + // Any deployment or computation difference allocates a distinct output. #[test] - fn num_aggregates_to_retain_does_not_affect_fingerprint() { - let mut a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - "", - ); - let mut b = a.clone(); - a.num_aggregates_to_retain = Some(100); - b.num_aggregates_to_retain = Some(500); - assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b), - "retention is a separate concern from policy identity" - ); + fn deployment_and_computation_differences_yield_distinct_outputs() { + let base = allocated(cfg("m", vec!["zone"], 60), "sum"); + let mut panes = cfg("m", vec!["zone"], 60); + panes.window_layout = crate::WindowMaterializationLayout::FullWindow; + let mut phased = cfg("m", vec!["zone"], 60); + phased.pane_origin_ms = Some(7_000); + for other in [ + allocated(cfg("other", vec!["zone"], 60), "sum"), + allocated(cfg("m", vec!["zone"], 30), "sum"), + allocated(cfg("m", vec!["host"], 60), "sum"), + allocated(cfg("m", vec!["zone"], 60), "count"), + allocated(panes, "sum"), + allocated(phased, "sum"), + ] { + assert_ne!(base.policy_fingerprint(), other.policy_fingerprint()); + } } + // Retention is not part of an output's identity. #[test] - fn parameter_map_order_does_not_affect_fingerprint() { - // HashMap iteration order is non-deterministic; the fingerprint - // must be order-independent. - let mut p1 = HashMap::new(); - p1.insert("a".into(), serde_json::json!(1)); - p1.insert("b".into(), serde_json::json!(2)); - let mut p2 = HashMap::new(); - p2.insert("b".into(), serde_json::json!(2)); - p2.insert("a".into(), serde_json::json!(1)); - let a = cfg("http_lat", AggregationType::Sum, p1, vec![], 60, ""); - let b = cfg("http_lat", AggregationType::Sum, p2, vec![], 60, ""); + fn retention_does_not_affect_output_identity() { + let a = allocated(cfg("m", vec![], 60), "sum"); + let mut b = cfg("m", vec![], 60); + b.num_aggregates_to_retain = Some(100); assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) + a.policy_fingerprint(), + allocated(b, "sum").policy_fingerprint() ); } #[test] - fn spatial_filter_canonicalization_drives_fingerprint() { - // Two filters that differ only in matcher ordering produce the - // SAME normalized form, hence the SAME fingerprint. The - // canonicalization step in `PrecomputeMaterialization::new` (via - // `normalize_spatial_filter`) sorts matchers by key. - let a = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"status="200",zone="us""#, - ); - let b = cfg( - "http_lat", - AggregationType::Sum, - HashMap::new(), - vec![], - 60, - r#"zone="us",status="200""#, - ); + fn display_format_is_hex_with_prefix() { assert_eq!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) + format!("{}", PolicyFingerprint(0xdead_beef)), + "policy_fp:00000000deadbeef" ); } - - #[test] - fn display_format_is_hex_with_prefix() { - let fp = PolicyFingerprint(0xdeadbeef); - assert_eq!(format!("{}", fp), "policy_fp:00000000deadbeef"); - } } diff --git a/crates/asap_types/src/policy_registry.rs b/crates/asap_types/src/policy_registry.rs deleted file mode 100644 index 90138b139..000000000 --- a/crates/asap_types/src/policy_registry.rs +++ /dev/null @@ -1,169 +0,0 @@ -//! Content-addressed policy registry. -//! -//! Derived view over a collection of `PrecomputeMaterialization`s that maps -//! [`PolicyFingerprint`] → [`PrecomputeMaterialization`]. This is the -//! merged-sid-identity-chain replacement for the controller-allocated -//! `aggregation_id`-keyed `HashMap` that `data_plane`'s `InstalledPrecomputePlan` -//! carries (see `data_plane::storage_engines::types::installed_precompute_plan`'s -//! module doc for why that type lives there, not here). -//! -//! ## Dual-keyed transition -//! -//! PR 3 (where this lives): the registry exists alongside the -//! `aggregation_id`-keyed map. Callers can opt into either lookup. -//! Construction is `O(N)` over the source configs; computation is -//! pure (no I/O, no mutation of the source). -//! -//! Subsequent PRs migrate one set of callers at a time off -//! `aggregation_id` → `PolicyFingerprint`, until the final PR can -//! delete the legacy index. -//! -//! ## Identity invariants -//! -//! Two `PrecomputeMaterialization`s that produce the same `PolicyFingerprint` -//! ARE the same policy. The registry treats this as a *deduplication* -//! invariant — if two distinct entries in the source `materializations_by_output` -//! map produce the same fingerprint, the later one wins (last-write -//! semantics). In practice the source should never contain duplicates; -//! if it does, that's a control-plane bug worth surfacing in telemetry -//! (see [`PolicyRegistry::from_configs_with_collisions`]). - -use std::collections::HashMap; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; - -/// Content-addressed lookup table for active aggregation policies. -#[derive(Debug, Clone, Default)] -pub struct PolicyRegistry { - policies: HashMap, -} - -impl PolicyRegistry { - /// Construct from a list of configs. Duplicates (same fingerprint) - /// collapse to the last entry; use - /// [`Self::from_configs_with_collisions`] when you want to detect - /// them. - pub fn from_configs(configs: I) -> Self - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - for cfg in configs { - policies.insert(PolicyFingerprint::from_config(&cfg), cfg); - } - Self { policies } - } - - #[cfg(test)] - /// Construct + report the count of duplicate fingerprints (entries - /// where the source contained two configs producing the same - /// fingerprint and the later one displaced the earlier). Zero in - /// the well-formed case; non-zero is a control-plane bug worth - /// surfacing. - pub fn from_configs_with_collisions(configs: I) -> (Self, usize) - where - I: IntoIterator, - { - let mut policies = HashMap::new(); - let mut collisions = 0usize; - for cfg in configs { - let fp = PolicyFingerprint::from_config(&cfg); - if policies.insert(fp, cfg).is_some() { - collisions += 1; - } - } - (Self { policies }, collisions) - } - - /// Look up the config for a fingerprint. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.policies.get(&fp) - } - - /// Iterate fingerprint → config pairs. - pub fn iter(&self) -> impl Iterator { - self.policies.iter() - } - - /// Live policy count. - pub fn len(&self) -> usize { - self.policies.len() - } - - pub fn is_empty(&self) -> bool { - self.policies.is_empty() - } - - /// All fingerprints currently registered. Useful for diffing two - /// registries during a hot-reload swap. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.policies.keys().copied() - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(_id: u64, metric: &str) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is derived from - // content. Kept as a parameter so existing call sites in the - // tests below don't churn. - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn from_configs_round_trips_lookup() { - let a = cfg(1, "http_lat"); - let b = cfg(2, "cpu_pct"); - let fp_a = PolicyFingerprint::from_config(&a); - let fp_b = PolicyFingerprint::from_config(&b); - let reg = PolicyRegistry::from_configs(vec![a.clone(), b.clone()]); - assert_eq!(reg.len(), 2); - assert_eq!(reg.get(fp_a).unwrap().metric, "http_lat"); - assert_eq!(reg.get(fp_b).unwrap().metric, "cpu_pct"); - } - - #[test] - fn identical_configs_collapse_to_one_entry() { - // PR 5: identity IS the content. Two configs that are - // byte-for-byte identical on the policy-relevant fields - // collapse to ONE entry — there's no way to distinguish them - // anymore. - let a = cfg(1, "http_lat"); - let b = cfg(99, "http_lat"); - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 1); - assert_eq!(collisions, 1); - } - - #[test] - fn distinct_policies_keep_distinct_entries() { - let a = cfg(1, "http_lat"); - let b = cfg(1, "cpu_pct"); // different metric → distinct policies - let (reg, collisions) = PolicyRegistry::from_configs_with_collisions(vec![a, b]); - assert_eq!(reg.len(), 2); - assert_eq!(collisions, 0); - } -} diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index 3da190514..cefe2cb5c 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -3,7 +3,7 @@ mod catalog; -use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; +use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; use planner_types::pre_asap::Source; use serde::{Deserialize, Serialize}; use std::collections::{BTreeMap, BTreeSet, HashMap}; @@ -87,7 +87,9 @@ pub fn validated_source_window_cohort<'a>( Ok(sources.to_vec()) } -pub const BACKEND_COMPAT: &str = "asap-query-backend.v1"; +/// Installed-plan schema. v2 materializations carry only deployment fields; +/// v1 plans duplicated computation semantics and are rejected on decode. +pub const BACKEND_COMPAT: &str = "asap-query-backend.v2"; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct PlanEnvelope { @@ -105,7 +107,7 @@ pub struct PlanEnvelope { /// edges define execution; materializations attach storage/window placement. /// Raw source-to-SummaryAgg paths lower to streaming kernels. Derived paths /// execute through the maintenance DAG scheduler at stored-state frontiers. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize)] pub struct PrecomputePlan { #[serde(default, skip_serializing_if = "Option::is_none")] pub summary_catalog: Option, @@ -119,6 +121,42 @@ pub struct PrecomputePlan { pub executable_dags: BTreeMap, } +/// Serde mirror of [`PrecomputePlan`]; the compiler checks it stays in sync. +#[derive(Deserialize)] +#[serde(remote = "PrecomputePlan")] +struct PrecomputePlanDef { + #[serde(default)] + summary_catalog: Option, + envelope: PlanEnvelope, + ingest: IngestContract, + schemas: Vec, + producers: Vec, + materializations: Vec, + #[serde(default)] + executable_dags: BTreeMap, +} + +impl<'de> Deserialize<'de> for PrecomputePlan { + /// Check the schema before the body so an older plan fails with its + /// version rather than with its first unrecognized field. + fn deserialize>(deserializer: D) -> Result { + use serde::de::Error; + let value = serde_json::Value::deserialize(deserializer)?; + let compat = value + .get("envelope") + .and_then(|envelope| envelope.get("backend_compat")) + .and_then(serde_json::Value::as_str); + if compat != Some(BACKEND_COMPAT) { + return Err(D::Error::custom(format!( + "unsupported installed precompute plan schema {}: this backend requires \ + {BACKEND_COMPAT}; recompile the plan", + compat.unwrap_or("") + ))); + } + PrecomputePlanDef::deserialize(value).map_err(D::Error::custom) + } +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum IngestProtocol { @@ -218,7 +256,9 @@ impl TryFrom<&SummaryFamilyType> for StateFamilyContract { } } -/// Decoder/schema contract for one content-addressed materialization. +/// Decoder/schema contract for one stored output. `family` is the stored +/// state's type, which readers need without executing the producer; when a +/// Planner DAG node produces the output, validation requires its family. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct StateSchemaContract { @@ -226,7 +266,7 @@ pub struct StateSchemaContract { pub schema_id: String, pub schema_version: u32, pub materialization: crate::sds::StoredOutputId, - pub family: StateFamilyContract, + pub family: SummaryFamilyType, pub source: Source, pub value_projection: crate::sds::ValueProjectionIdentity, pub group_by: crate::GroupingProjection, @@ -288,36 +328,29 @@ pub enum PrecomputePlanError { } impl PrecomputePlan { + /// Each output is paired with its stored state family. pub fn build( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, producer_ids: &[String], ) -> Result { - Self::build_complete( - envelope, - materializations, - producer_ids, - false, - BTreeMap::new(), - ) + Self::build_complete(envelope, outputs, producer_ids, false, BTreeMap::new()) } fn build_complete( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, producer_ids: &[String], backend_local: bool, executable_dags: BTreeMap, ) -> Result { - let schemas = materializations + let schemas = outputs .iter() - .map(|materialization| { + .map(|(materialization, family)| { let fingerprint = materialization.policy_fingerprint(); - let accumulator = materialization - .accumulator_spec() - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; - let family = StateFamilyContract::try_from(&accumulator.family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(fingerprint.0))?; + if StateFamilyContract::try_from(family).is_err() { + return Err(PrecomputePlanError::UnsupportedFamily(fingerprint.0)); + } let source = materialization.table_name.as_ref().map_or_else( || Source::TimeSeries { metric: materialization.metric.clone(), @@ -334,7 +367,7 @@ impl PrecomputePlan { schema_id: state_schema_id(fingerprint), schema_version: 1, materialization: fingerprint.into(), - family, + family: family.clone(), source, value_projection, group_by: materialization.grouping_labels.clone(), @@ -350,10 +383,11 @@ impl PrecomputePlan { }, pane_origin_ms: materialization.pane_origin_ms, }, - encodings: state_encodings(&accumulator.family), + encodings: state_encodings(family), }) }) .collect::, PrecomputePlanError>>()?; + let materializations: Vec<_> = outputs.into_iter().map(|(config, _)| config).collect(); let producers = producer_ids .iter() .flat_map(|producer_id| { @@ -407,19 +441,97 @@ impl PrecomputePlan { /// are precomputed inside ASAPQuery rather than by ASAPCollector. pub fn build_backend_local( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, ) -> Result { - Self::build_backend_local_with_dags(envelope, materializations, BTreeMap::new()) + Self::build_backend_local_with_dags(envelope, outputs, BTreeMap::new()) } /// Construct the complete backend-local contract before validation. Derived /// definitions are never validated without their actual executable bindings. pub fn build_backend_local_with_dags( envelope: PlanEnvelope, - materializations: Vec, + outputs: Vec<(crate::PrecomputeMaterialization, SummaryFamilyType)>, executable_dags: BTreeMap, ) -> Result { - Self::build_complete(envelope, materializations, &[], true, executable_dags) + Self::build_complete(envelope, outputs, &[], true, executable_dags) + } + + /// The stored state family of `output`, from its schema contract. + pub fn state_family(&self, output: crate::sds::StoredOutputId) -> Option<&SummaryFamilyType> { + self.schemas + .iter() + .find(|schema| schema.materialization == output) + .map(|schema| &schema.family) + } + + /// The Planner SummaryAgg node that produces `output`, with the source + /// expression feeding it when that input is a raw source. `None` when no + /// installed DAG produces the output with a SummaryAgg. + #[allow(clippy::type_complexity)] + pub fn summary_producer( + &self, + output: crate::sds::StoredOutputId, + ) -> Result< + Option<( + planner_types::post_asap::PostAsapDagNode, + Option, + )>, + String, + > { + use planner_types::post_asap::PostAsapOperatorPayload; + for installed in self.executable_dags.values() { + let Some(node) = installed.binding.nodes.iter().find_map(|(node, binding)| { + matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } if *stored_output == output).then_some(*node) + }) else { + continue; + }; + let dag = installed.document.decode()?; + let producer = dag + .nodes + .iter() + .find(|candidate| candidate.id == node) + .ok_or("bound materialization node is absent from its DAG")?; + if !matches!(producer.payload, PostAsapOperatorPayload::SummaryAgg { .. }) { + continue; + } + let inputs: Vec<_> = dag.edges.iter().filter(|e| e.consumer == node).collect(); + let source = match inputs.as_slice() { + [edge] => dag + .nodes + .iter() + .find(|candidate| candidate.id == edge.producer) + .and_then(|source| match &source.payload { + PostAsapOperatorPayload::Fallback { expression } => { + Some(expression.clone()) + } + _ => None, + }), + _ => None, + }; + return Ok(Some((producer.clone(), source))); + } + Ok(None) + } + + /// Canonical population predicate of `config`'s input: the typed table + /// population, or the label filter of its Planner DAG time-series scan. + pub fn population_filter( + &self, + config: &crate::PrecomputeMaterialization, + ) -> Result { + let table = config.table_population_canonical()?; + if config.table_name.is_some() || config.derived_input.is_some() { + return Ok(table); + } + let Some((_, Some(expression))) = self.summary_producer(config.stored_output_id)? else { + return Ok(String::new()); + }; + let family = self.state_family(config.stored_output_id); + let (_, _, filter) = raw_time_series_input_contract( + &expression, + matches!(family, Some(SummaryFamilyType::ExactAggregate(..))), + )?; + Ok(crate::utils::normalize_spatial_filter(&filter)) } pub fn runtime_materializations( @@ -630,11 +742,16 @@ impl PrecomputePlan { .map_err(|e| PrecomputePlanError::CatalogContract(e.to_string())) .map(|contract| !asap_physical_operators::physical_planner::precompute::is_population_schema(&contract.schema))? .then_some(native); - if sources.iter().any(|source| { - !matches!(source.aggregation_type, crate::AggregationType::Sum) - && !(native.is_some() - && matches!(source.aggregation_type, crate::AggregationType::Rate)) - }) { + let exact = |config: &crate::PrecomputeMaterialization, kinds: &[ExactKind]| { + matches!(self.state_family(config.stored_output_id), + Some(SummaryFamilyType::ExactAggregate(kind, _)) if kinds.contains(kind)) + }; + let source_kinds: &[ExactKind] = if native.is_some() { + &[ExactKind::Sum, ExactKind::Rate] + } else { + &[ExactKind::Sum] + }; + if sources.iter().any(|source| !exact(source, source_kinds)) { return Err(invalid()); } if native.is_none() { @@ -643,12 +760,11 @@ impl PrecomputePlan { } validate_maintenance_reduction(config, target_node) .map_err(PrecomputePlanError::CatalogContract)?; - } else if !matches!( - config.aggregation_type, - crate::AggregationType::CountMinSketchWithHeap - | crate::AggregationType::CountSketchWithHeap - | crate::AggregationType::Sum - ) { + } else if !exact(config, &[ExactKind::Sum]) + && !matches!(self.state_family(config.stored_output_id), + Some(SummaryFamilyType::Sketch(kind, _)) if matches!(kind.algorithm(), + SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + { return Err(invalid()); } let inputs: Vec<_> = dag @@ -773,32 +889,38 @@ impl PrecomputePlan { "DAG materialization has no runtime configuration".into(), )); }; - if self.ingest.protocol == IngestProtocol::PrometheusRemoteWriteV1 - && matches!( - config.aggregation_type, - crate::AggregationType::HLL | crate::AggregationType::UnivMon - ) + if let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { + family, + input, + .. + } = &node.payload { - if let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { - input, - .. - } = &node.payload - { - let supported = match config.aggregation_type { - crate::AggregationType::HLL => { - crate::accumulator_spec::is_scalar_sample_value(input) - || crate::accumulator_spec::is_unit_sample_frequency(input) - } - crate::AggregationType::UnivMon => { - crate::accumulator_spec::is_unit_sample_frequency(input) - } - _ => unreachable!(), - }; - if !supported { - return Err(PrecomputePlanError::CatalogContract( - "raw materialization input does not match its accumulator update semantics".into(), - )); + if self.state_family(config.stored_output_id) != Some(family) { + return Err(PrecomputePlanError::CatalogContract( + "stored output schema family differs from its Planner producer".into(), + )); + } + let algorithm = match family { + SummaryFamilyType::Sketch(kind, _) => Some(kind.algorithm()), + _ => None, + }; + let supported = match algorithm { + _ if self.ingest.protocol != IngestProtocol::PrometheusRemoteWriteV1 => { + true } + Some(SketchAlgorithm::Hll) => { + crate::accumulator_spec::is_scalar_sample_value(input) + || crate::accumulator_spec::is_unit_sample_frequency(input) + } + Some(SketchAlgorithm::UnivMon) => { + crate::accumulator_spec::is_unit_sample_frequency(input) + } + _ => true, + }; + if !supported { + return Err(PrecomputePlanError::CatalogContract( + "raw materialization input does not match its accumulator update semantics".into(), + )); } } if let Some(partitioning) = config.partitioning { @@ -827,6 +949,11 @@ impl PrecomputePlan { } let mut materializations = BTreeSet::new(); for materialization in &self.materializations { + if materialization.stored_output_id.as_u64() == 0 { + return Err(PrecomputePlanError::CatalogContract( + "materialization has no allocated stored output id".into(), + )); + } if materialization.table_name.is_none() && !materialization.grouping_labels.is_legacy_labels() { @@ -899,12 +1026,13 @@ impl PrecomputePlan { candidate.policy_fingerprint() == schema.materialization.fingerprint() }) .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let accumulator = materialization.accumulator_spec().map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; - let family = StateFamilyContract::try_from(&accumulator.family).map_err(|_| { - PrecomputePlanError::UnsupportedFamily(schema.materialization.as_u64()) - })?; + if StateFamilyContract::try_from(&schema.family).is_err() + || crate::aggregation_type_for_family(&schema.family).is_none() + { + return Err(PrecomputePlanError::UnsupportedFamily( + schema.materialization.as_u64(), + )); + } let source = materialization.table_name.as_ref().map_or_else( || Source::TimeSeries { metric: materialization.metric.clone(), @@ -915,7 +1043,6 @@ impl PrecomputePlan { ); let value_projection = materialization.effective_value_projection().clone(); if schema.schema_id != state_schema_id(schema.materialization.fingerprint()) - || schema.family != family || schema.source != source || schema.value_projection != value_projection || schema.group_by != materialization.grouping_labels @@ -930,7 +1057,7 @@ impl PrecomputePlan { crate::WindowKind::Session => None, } || schema.window.pane_origin_ms != materialization.pane_origin_ms - || !state_encodings_match(&accumulator.family, &schema.encodings) + || !state_encodings_match(&schema.family, &schema.encodings) { return Err(PrecomputePlanError::InvalidSchema { schema_id: schema.schema_id.clone(), @@ -974,6 +1101,74 @@ impl PrecomputePlan { } } +/// Decompose a Planner raw time-series input expression into its metric, +/// optional whole-second range and PromQL label filter. +pub fn raw_time_series_input_contract( + expr: &planner_types::pre_asap::QueryExpr, + exact: bool, +) -> Result<(String, Option, String), String> { + use planner_types::pre_asap::{CompareOpKind, QueryExpr, ScalarValue}; + let (source, window_secs) = match expr { + QueryExpr::TimeRange { child, range } => { + if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { + return Err("warm producer requires a positive whole-second range".into()); + } + (child.as_ref(), Some(range.as_secs())) + } + QueryExpr::Scan { .. } if exact => { + return Err( + "instantaneous sample selection is not a temporal accumulator readout".into(), + ); + } + source => (source, None), + }; + match source { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + predicates, + schema, + } if !metric.is_empty() => { + let mut matchers = Vec::with_capacity(predicates.len()); + for predicate in predicates { + let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { + return Err("materialization filter is not a label comparison".into()); + }; + let (QueryExpr::Column(column), QueryExpr::Literal(ScalarValue::Utf8(value))) = + (left.as_ref(), right.as_ref()) + else { + return Err("materialization filter must compare a label with a string".into()); + }; + let label = schema + .columns + .get(*column) + .map(|field| field.name.as_str()) + .ok_or_else(|| { + "materialization filter references an unknown column".to_string() + })?; + let operator = match op { + CompareOpKind::Eq => "=", + CompareOpKind::Ne => "!=", + CompareOpKind::Regex => "=~", + CompareOpKind::NotRegex => "!~", + _ => return Err("materialization filter uses a non-PromQL comparison".into()), + }; + let encoded = serde_json::to_string(value).map_err(|error| error.to_string())?; + matchers.push(format!("{label}{operator}{encoded}")); + } + matchers.sort(); + let spatial_filter = if matchers.is_empty() { + String::new() + } else { + format!("{{{}}}", matchers.join(",")) + }; + Ok((metric.clone(), window_secs, spatial_filter)) + } + _ => { + Err("source predicates or temporal modifiers require a Prometheus exact subtree".into()) + } + } +} + pub(crate) fn state_schema_id(fingerprint: crate::PolicyFingerprint) -> String { format!("{}:summary-state:v1:{}", BACKEND_COMPAT, fingerprint.0) } @@ -1046,16 +1241,34 @@ pub(crate) fn state_encodings(family: &SummaryFamilyType) -> Vec mod source_window_cohort_tests { use super::*; fn full_window() -> crate::PrecomputeMaterialization { - serde_json::from_value(serde_json::json!({ - "aggregation_type":"Sum", "aggregation_sub_type":"", "parameters":{}, - "grouping_labels":{"labels":[]}, "aggregated_labels":{"labels":[]}, - "rollup_labels":{"labels":[]}, "original_yaml":"", - "window_size":60, "slide_interval":10, "window_type":"sliding", - "window_layout":{"kind":"full_window"}, "pane_origin_ms":0, - "spatial_filter":"", "spatial_filter_normalized":"", "metric":"m", - "num_aggregates_to_retain":null, "table_name":null, "value_projection":null - })) - .unwrap() + let mut config: crate::PrecomputeMaterialization = + serde_json::from_value(serde_json::json!({ + "stored_output_id":0, "grouping_labels":{"labels":[]}, + "window_size":60, "slide_interval":10, "window_type":"sliding", + "window_layout":{"kind":"full_window"}, "pane_origin_ms":0, "metric":"m", + "num_aggregates_to_retain":null, "table_name":null, "value_projection":null + })) + .unwrap(); + config.allocate_stored_output_id(&"sum"); + config + } + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate( + ExactKind::Sum, + planner_types::post_asap::ExactParams::Sum, + ) + } + fn envelope(plan_version: u64) -> PlanEnvelope { + PlanEnvelope { + plan_id: 1, + plan_version, + generated_at_unix_ms: 0, + activation_unix_ms: 0, + expiry_unix_ms: None, + backend_compat: BACKEND_COMPAT.into(), + planner_revision: "test".into(), + capability_snapshot_id: "test".into(), + } } // New native-format support must not invalidate persisted older codec subsets. #[test] @@ -1089,20 +1302,11 @@ mod source_window_cohort_tests { #[test] fn producer_roster_roundtrip_and_watermark_scope() { - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 2, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; let mut config = full_window(); config.slide_interval = config.window_size; config.window_type = crate::WindowKind::Tumbling; - let mut plan = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap(); + let mut plan = + PrecomputePlan::build_backend_local(envelope(2), vec![(config, sum())]).unwrap(); let generation = crate::sds::CatalogGeneration { schema_version: 1, plan_id: 1, @@ -1222,17 +1426,8 @@ mod source_window_cohort_tests { config.slide_interval = config.window_size; config.population_key_encoding = crate::grouping_projection::PopulationKeyEncoding::CanonicalLabelsV1; - let envelope = PlanEnvelope { - plan_id: 1, - plan_version: 1, - generated_at_unix_ms: 0, - activation_unix_ms: 0, - expiry_unix_ms: None, - backend_compat: "test".into(), - planner_revision: "test".into(), - capability_snapshot_id: "test".into(), - }; - let error = PrecomputePlan::build_backend_local(envelope, vec![config]).unwrap_err(); + let error = + PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap_err(); assert!( error.to_string().contains("population key encoding"), "{error}" @@ -1252,6 +1447,7 @@ mod source_window_cohort_tests { assert!(std::ptr::eq(result[0], &source)); let mut other = source.clone(); other.metric = "other".into(); + other.allocate_stored_output_id(&"sum"); assert_eq!( validated_source_window_cohort(&target, &[&source, &other]) .unwrap() @@ -1269,8 +1465,65 @@ mod source_window_cohort_tests { crate::WindowMaterializationLayout::Pane { pane_secs: 10 } } } + changed.allocate_stored_output_id(&"sum"); assert_ne!(source.policy_fingerprint(), changed.policy_fingerprint()); assert!(validated_source_window_cohort(&target, &[&changed]).is_err()); } } + + // A plan compiled for the v1 schema, which duplicated computation fields + // on materializations, is rejected by version before its body is read. + #[test] + fn rejects_plans_from_the_computation_carrying_schema_by_version() { + let mut config = full_window(); + config.slide_interval = config.window_size; + config.window_type = crate::WindowKind::Tumbling; + let plan = PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap(); + let mut wire = serde_json::to_value(&plan).unwrap(); + assert!(serde_json::from_value::(wire.clone()).is_ok()); + wire["envelope"]["backend_compat"] = "asap-query-backend.v1".into(); + wire["materializations"][0]["aggregation_type"] = "Sum".into(); + let error = serde_json::from_value::(wire).unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported installed precompute plan schema asap-query-backend.v1"), + "{error}" + ); + } + + // Materializations name no computation: the removed v1 fields are unknown. + #[test] + fn materializations_reject_removed_computation_fields() { + let wire = serde_json::to_value(full_window()).unwrap(); + for field in [ + "aggregation_type", + "aggregation_sub_type", + "parameters", + "spatial_filter", + "spatial_filter_normalized", + "aggregated_labels", + "rollup_labels", + "original_yaml", + ] { + let mut legacy = wire.clone(); + legacy[field] = serde_json::json!(""); + assert!( + serde_json::from_value::(legacy).is_err(), + "{field}" + ); + } + } + + // An output without an allocated id cannot be installed. + #[test] + fn rejects_unallocated_stored_output_ids() { + let mut config = full_window(); + config.slide_interval = config.window_size; + config.window_type = crate::WindowKind::Tumbling; + config.stored_output_id = crate::sds::StoredOutputId(0); + let error = + PrecomputePlan::build_backend_local(envelope(1), vec![(config, sum())]).unwrap_err(); + assert!(error.to_string().contains("stored output id"), "{error}"); + } } diff --git a/crates/asap_types/src/precompute_plan/catalog.rs b/crates/asap_types/src/precompute_plan/catalog.rs index 80bfbcaa3..eaf49df13 100644 --- a/crates/asap_types/src/precompute_plan/catalog.rs +++ b/crates/asap_types/src/precompute_plan/catalog.rs @@ -44,12 +44,7 @@ impl PrecomputePlan { catalog: &SummaryCatalog, ) -> Result<(), PrecomputePlanError> { catalog.validate().map_err(|e| invalid(e.to_string()))?; - let expected = SummaryCatalog::from_materializations( - catalog.plan_id, - catalog.plan_version, - &self.materializations, - ) - .map_err(|e| invalid(e.to_string()))?; + let expected = SummaryCatalog::from_plan(self).map_err(|e| invalid(e.to_string()))?; if catalog.outputs != expected.outputs || catalog.definitions != expected.definitions { return Err(invalid( "stored output semantics differ from installed writer computation", @@ -118,8 +113,11 @@ impl PrecomputePlan { for config in &self.materializations { let id = StoredOutputId::from(config.policy_fingerprint()); let binding = &catalog.outputs[&id]; + let family = self + .state_family(id) + .ok_or(PrecomputePlanError::SchemaSetMismatch)?; let expected = - SummaryDescriptor::from_config(config).map_err(|e| invalid(e.to_string()))?; + SummaryDescriptor::from_family(family).map_err(|e| invalid(e.to_string()))?; if binding.summary_descriptor_id != expected.id { return Err(invalid( "summary operator/update contract differs from catalog", @@ -149,27 +147,16 @@ impl PrecomputePlan { || data.source != expected_source || &data.value_projection != expected_projection || data.population_filter_canonical - != config.population_filter_canonical().map_err(invalid)? + != self.population_filter(config).map_err(invalid)? || data.group_by_keys != config.grouping_labels { return Err(invalid("source/population/grouping differs from catalog")); } - if config.spatial_filter_normalized - != crate::utils::normalize_spatial_filter(&config.spatial_filter) - { - return Err(invalid("normalized population predicate drift")); - } let schema = self .schemas .iter() .find(|s| s.materialization == id) .ok_or(PrecomputePlanError::SchemaSetMismatch)?; - let family = config - .accumulator_spec() - .map_err(|e| invalid(e.to_string()))? - .family; - let expected_family = StateFamilyContract::try_from(&family) - .map_err(|_| PrecomputePlanError::UnsupportedFamily(id.as_u64()))?; let source = if let Some(table) = &config.table_name { Source::Table { table_ref: table.clone(), @@ -202,7 +189,6 @@ impl PrecomputePlan { .output_reference(id) .map_err(|e| invalid(e.to_string()))? || schema.schema_id != state_schema_id(id.fingerprint()) - || schema.family != expected_family || schema.source != source || &schema.value_projection != projection || schema.group_by != config.grouping_labels @@ -215,7 +201,7 @@ impl PrecomputePlan { schema_id: schema.schema_id.clone(), }); } - if !state_encodings_match(&family, &schema.encodings) { + if !state_encodings_match(family, &schema.encodings) { return Err(invalid("encoding does not match state family")); } if config.num_aggregates_to_retain == Some(0) { diff --git a/crates/asap_types/src/producer_plan.rs b/crates/asap_types/src/producer_plan.rs index 846c4df83..dd5a7cfd9 100644 --- a/crates/asap_types/src/producer_plan.rs +++ b/crates/asap_types/src/producer_plan.rs @@ -386,7 +386,9 @@ impl TransmissionPlan { if !schema.encodings.contains(&rule.encoding) { return Err(TransmissionPlanError::InvalidRule(rule.producer_id.clone())); } - validate_runtime_rule_policy(rule, &schema.family)?; + let family = StateFamilyContract::try_from(&schema.family) + .map_err(|()| TransmissionPlanError::InvalidRule(rule.producer_id.clone()))?; + validate_runtime_rule_policy(rule, &family)?; } Ok(()) } diff --git a/crates/asap_types/src/routing_index.rs b/crates/asap_types/src/routing_index.rs deleted file mode 100644 index ec20b7ef2..000000000 --- a/crates/asap_types/src/routing_index.rs +++ /dev/null @@ -1,270 +0,0 @@ -//! `RoutingIndex` — a metric-bucketed structural index over a -//! [`PolicyRegistry`]. It is sourced from the content-addressed view over a -//! `InstalledPrecomputePlan`'s `PrecomputeMaterialization`s, so it represents planned policy -//! rather than a reconstruction from ingest side effects. -//! -//! **Tier 1** (exact `PolicyFingerprint` → config) is [`PolicyRegistry::get`] -//! itself — already O(1), nothing to add here. -//! -//! **Tier 2** (structural match: "every policy registered for this metric") -//! is what this type adds. Before this existed, -//! `control_plane::asap_tier_analysis::find_matching_policies` scanned -//! *every* policy in the registry for every candidate, checking each one's -//! metric name first — i.e. it paid for every OTHER metric's policies on -//! every lookup. `RoutingIndex` buckets by metric once, at construction -//! time, so a lookup only ever touches the policies that could possibly -//! match. -//! -//! ## Lifecycle -//! -//! Built fresh from a `PolicyRegistry` snapshot — construction is `O(N)` -//! over the registry, same order as `PolicyRegistry::from_configs` itself. -//! Callers that build one per query (mirroring today's -//! `streaming_snap.policy_registry()` call) still get the Tier-2 win for -//! any query with more than one candidate sharing the same snapshot -//! (composed PromQL shapes routinely do). Building it once per -//! `InstalledPrecomputePlan` hot-reload swap instead of once per query — the same -//! "cheap, but call at swap time not per query, if it shows up in -//! profiles" note `InstalledPrecomputePlan::policy_registry`'s own doc comment -//! already flags — is a further, larger change (it means threading a -//! cached derived value through `InstalledPrecomputePlanHandle`'s swap path) -//! and is not done by this type on its own. - -use std::collections::{BTreeSet, HashMap}; - -use crate::aggregation_config::PrecomputeMaterialization; -use crate::policy_fingerprint::PolicyFingerprint; -use crate::policy_registry::PolicyRegistry; - -/// See module docs. -#[derive(Debug, Clone, Default)] -pub struct RoutingIndex { - registry: PolicyRegistry, - by_metric: HashMap>, -} - -impl RoutingIndex { - /// Build from a `PolicyRegistry` snapshot. Takes ownership rather than - /// borrowing — callers that still need their own `PolicyRegistry` - /// handle after this should `.clone()` it first (cheap-ish, but real; - /// most callers don't need the raw registry once they have the index, - /// since [`Self::get`] delegates straight through). - pub fn build(registry: PolicyRegistry) -> Self { - let mut by_metric: HashMap> = HashMap::new(); - for (fp, cfg) in registry.iter() { - by_metric.entry(cfg.metric.clone()).or_default().push(*fp); - } - Self { - registry, - by_metric, - } - } - - /// Tier 1 — exact fingerprint lookup. Delegates to the underlying - /// registry; see [`PolicyRegistry::get`]. - pub fn get(&self, fp: PolicyFingerprint) -> Option<&PrecomputeMaterialization> { - self.registry.get(fp) - } - - /// Tier 2 — every policy fingerprint registered for `metric`, in - /// registration order. Empty slice (not an `Option`/error) when - /// nothing is registered for this metric — callers already treat "no - /// candidates" as a normal, expected outcome (capability miss → - /// archive fallback), not a failure to report. - pub fn candidates_for_metric(&self, metric: &str) -> &[PolicyFingerprint] { - self.by_metric.get(metric).map(Vec::as_slice).unwrap_or(&[]) - } - - /// Live policy count (same as the underlying registry's). - pub fn len(&self) -> usize { - self.registry.len() - } - - pub fn is_empty(&self) -> bool { - self.registry.is_empty() - } - - /// All fingerprints currently registered. Delegates to the underlying - /// registry — see [`PolicyRegistry::fingerprints`]. - pub fn fingerprints(&self) -> impl Iterator + '_ { - self.registry.fingerprints() - } - - /// Resolve an ingest-side sketch shape to exactly one configured policy. - /// Ambiguous or absent matches fail closed. - pub fn find_policy_by_content( - &self, - metric: &str, - group_by_keys: &BTreeSet, - agg_type: crate::AggregationType, - expected_params: &HashMap, - ) -> Option { - let mut hit = None; - for fp in self.candidates_for_metric(metric) { - let cfg = self.get(*fp)?; - let policy_keys: BTreeSet<_> = cfg.grouping_labels.iter().cloned().collect(); - if cfg.aggregation_type != agg_type - || &policy_keys != group_by_keys - || !cfg.spatial_filter_normalized.is_empty() - || !expected_params.iter().all(|(key, value)| { - cfg.parameters.get(key).or_else(|| match key.as_str() { - // The typed physical-plan compiler names this field - // after SummaryParams, while legacy collector YAML - // uses the equivalent runtime-facing name. - "relative_accuracy" => cfg.parameters.get("alpha"), - "alpha" => cfg.parameters.get("relative_accuracy"), - _ => None, - }) == Some(value) - }) - { - continue; - } - if hit.is_some() { - return None; - } - hit = Some(*fp); - } - hit - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::enums::WindowKind; - use crate::AggregationType; - use crate::KeyByLabelNames; - use std::collections::HashMap as StdHashMap; - - fn cfg(metric: &str) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) - } - - #[test] - fn buckets_by_metric() { - let a1 = cfg("http_lat"); - let a2 = cfg("cpu_pct"); - let fp_a1 = PolicyFingerprint::from_config(&a1); - let fp_a2 = PolicyFingerprint::from_config(&a2); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a1, a2])); - - assert_eq!(idx.candidates_for_metric("http_lat"), &[fp_a1]); - assert_eq!(idx.candidates_for_metric("cpu_pct"), &[fp_a2]); - } - - #[test] - fn multiple_policies_for_the_same_metric_all_bucket_together() { - // Same metric, distinct group-by shapes -> distinct fingerprints, - // same bucket. - let a = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - StdHashMap::new(), - KeyByLabelNames::new(vec!["zone".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "http_lat".to_string(), - None, - None, - None, - ); - let b = cfg("http_lat"); - assert_ne!( - PolicyFingerprint::from_config(&a), - PolicyFingerprint::from_config(&b) - ); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a, b])); - assert_eq!(idx.candidates_for_metric("http_lat").len(), 2); - } - - #[test] - fn unknown_metric_returns_empty_slice_not_missing() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("http_lat")])); - assert!(idx.candidates_for_metric("no_such_metric").is_empty()); - } - - #[test] - fn get_delegates_to_underlying_registry() { - let a = cfg("http_lat"); - let fp = PolicyFingerprint::from_config(&a); - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![a])); - assert_eq!( - idx.get(fp).map(|c| c.metric.clone()), - Some("http_lat".to_string()) - ); - assert!(idx - .get(PolicyFingerprint::from_config(&cfg("nope"))) - .is_none()); - } - - #[test] - fn len_and_is_empty_match_registry() { - let idx = RoutingIndex::build(PolicyRegistry::from_configs( - Vec::::new(), - )); - assert!(idx.is_empty()); - assert_eq!(idx.len(), 0); - - let idx = RoutingIndex::build(PolicyRegistry::from_configs(vec![cfg("m")])); - assert!(!idx.is_empty()); - assert_eq!(idx.len(), 1); - } - - #[test] - fn ddsketch_alpha_and_relative_accuracy_are_wire_compatible() { - let mut parameters = StdHashMap::new(); - parameters.insert("alpha".to_string(), serde_json::json!(0.01)); - let config = PrecomputeMaterialization::new( - AggregationType::DDSketch, - String::new(), - parameters, - KeyByLabelNames::new(vec!["service".to_string()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 5, - 5, - WindowKind::Tumbling, - String::new(), - "latency".to_string(), - None, - None, - None, - ); - let fingerprint = PolicyFingerprint::from_config(&config); - let index = RoutingIndex::build(PolicyRegistry::from_configs(vec![config])); - let expected = - StdHashMap::from([("relative_accuracy".to_string(), serde_json::json!(0.01))]); - - assert_eq!( - index.find_policy_by_content( - "latency", - &BTreeSet::from(["service".to_string()]), - AggregationType::DDSketch, - &expected, - ), - Some(fingerprint) - ); - } -} diff --git a/crates/asap_types/src/sds.rs b/crates/asap_types/src/sds.rs index a6870ea20..1619c6ba0 100644 --- a/crates/asap_types/src/sds.rs +++ b/crates/asap_types/src/sds.rs @@ -3,7 +3,7 @@ //! belong to the stored summary identified by this metadata. pub const TIMESTAMPED_OBSERVATION_SEMANTICS: &str = "asap.timestamped-observations.v2"; -use crate::{AggregationType, PrecomputeMaterialization}; +use crate::AggregationType; use planner_types::post_asap::{SketchAlgorithm, SketchParams, SummaryFamilyType}; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; @@ -492,15 +492,11 @@ pub enum SummaryOperator { agg_type: AggregationType, parameters_canonical: String, }, - /// Complete planner materialization configuration, including heap/Hydra - /// dimensions and readout/update subtype. Never equal to a legacy projection. + /// The Planner-selected state family of a stored output, including + /// heap/Hydra dimensions. Never equal to a legacy projection. Grouping and + /// pane layout live in the data descriptor and summary definition. Configured { - /// Planner-selected semantic family; grouping and pane layout live in - /// the data descriptor and summary definition, respectively. family: planner_types::post_asap::SummaryFamilyType, - aggregation_type: AggregationType, - aggregation_sub_type: String, - parameters: BTreeMap, }, } @@ -553,32 +549,19 @@ impl FidelityGuarantee { /// Model IDs name parameterized error families/scopes, not certified numeric /// epsilon/confidence values. Heap membership and Hydra cross-cell readouts /// need separate models; point-frequency/per-cell rank does not attest them. - pub fn from_config(config: &PrecomputeMaterialization) -> Self { - if config.aggregation_type == AggregationType::HLL { - let precision = config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(Value::as_u64) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(14); - return Self::HllCardinalityError { - precision, - model: "asap.hll.relative-cardinality.v1".into(), - }; - } - match config.accumulator_spec().map(|s| s.family) { - Ok(SummaryFamilyType::ExactAggregate( + pub fn from_family(family: &SummaryFamilyType) -> Self { + match family { + SummaryFamilyType::ExactAggregate( planner_types::post_asap::ExactKind::Increase | planner_types::post_asap::ExactKind::Rate | planner_types::post_asap::ExactKind::IRate, _, - )) => Self::ExactCounter { + ) => Self::ExactCounter { model: "prometheus.extrapolated-rate.v1".into(), full_pane_coverage_required: true, }, - Ok(SummaryFamilyType::ExactAggregate(..)) => Self::Exact, - Ok(SummaryFamilyType::Sketch(kind, _)) => match kind.params() { + SummaryFamilyType::ExactAggregate(..) => Self::Exact, + SummaryFamilyType::Sketch(kind, grouping) => match kind.params() { SketchParams::UnivMon { heap_size, sketch_rows, @@ -592,7 +575,10 @@ impl FidelityGuarantee { }, SketchParams::Kll { k } => Self::KllRankError { k: *k, - model: if config.aggregation_type == AggregationType::HydraKLL { + model: if matches!( + grouping, + planner_types::post_asap::GroupingStrategy::SharedMultiSubpopulation { .. } + ) { "asap.hydra-kll.per-cell-rank.v1" } else { "asap.kll.normalized-rank.v1" @@ -713,41 +699,6 @@ impl SummaryDescriptor { return Err(SdsError("state schema version must be positive".into())); } fidelity.validate()?; - if let SummaryOperator::Configured { - family, - aggregation_type, - .. - } = &operator - { - if let Some(expected) = aggregation_type.planner_exact_family() { - if family != &expected { - return Err(SdsError( - "configured storage type disagrees with Planner family".into(), - )); - } - } else { - use AggregationType as A; - let expected = match aggregation_type { - A::DatasketchesKLL | A::HydraKLL => Some(SketchAlgorithm::Kll), - A::CountMinSketch => Some(SketchAlgorithm::Cms), - A::CountMinSketchWithHeap => Some(SketchAlgorithm::CmsWithHeap), - A::CountSketch => Some(SketchAlgorithm::CountSketch), - A::CountSketchWithHeap => Some(SketchAlgorithm::CountSketchWithHeap), - A::DDSketch => Some(SketchAlgorithm::DDSketch), - A::HLL => Some(SketchAlgorithm::Hll), - A::UnivMon => Some(SketchAlgorithm::UnivMon), - _ => None, - }; - if let Some(expected) = expected { - if !matches!(family, planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) if kind.algorithm() == &expected) - { - return Err(SdsError( - "configured sketch storage disagrees with Planner family".into(), - )); - } - } - } - } if !fidelity.is_compatible_with(&operator) { return Err(SdsError( "summary operator and fidelity guarantee are incompatible".into(), @@ -777,12 +728,9 @@ impl SummaryDescriptor { Ok(()) } - /// Preserve every configured state/update parameter. Omitted defaults remain - /// distinct from explicit defaults (conservative identity, never false sharing). - /// Legacy AggKind projections intentionally have different operator variants: - /// they cannot attest heap, Hydra, or aggregation-subtype semantics they lost. - pub fn from_config(config: &PrecomputeMaterialization) -> Result { - let fidelity = FidelityGuarantee::from_config(config); + /// The descriptor of a stored output whose state has the Planner `family`. + pub fn from_family(family: &SummaryFamilyType) -> Result { + let fidelity = FidelityGuarantee::from_family(family); let state_schema_version = if matches!(fidelity, FidelityGuarantee::ExactCounter { .. }) { 2 } else { @@ -790,17 +738,7 @@ impl SummaryDescriptor { }; Self::new( SummaryOperator::Configured { - family: config - .accumulator_spec() - .map_err(|error| SdsError(error.to_string()))? - .family, - aggregation_type: config.aggregation_type, - aggregation_sub_type: config.aggregation_sub_type.clone(), - parameters: config - .parameters - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect(), + family: family.clone(), }, fidelity, state_schema_version, @@ -844,11 +782,7 @@ impl FidelityGuarantee { match operator { SummaryOperator::LegacyPartial { .. } => matches!(self, Unknown { .. }), SummaryOperator::ExactAgg { agg_type, .. } => configured(*agg_type), - SummaryOperator::Configured { - aggregation_type, - parameters, - .. - } => configured(*aggregation_type) && self.matches_parameters(parameters), + SummaryOperator::Configured { family } => self == &Self::from_family(family), SummaryOperator::Sketch { algorithm, parameters, @@ -1653,9 +1587,6 @@ mod tests { assert!(SummaryDescriptor::new( SummaryOperator::Configured { family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), }, FidelityGuarantee::KllRankError { k: 200, @@ -1678,70 +1609,67 @@ mod tests { .is_err()); } + // A configured descriptor's fidelity must be the one its family declares. #[test] - fn configured_descriptor_rejects_family_storage_disagreement() { + fn configured_descriptor_rejects_fidelity_family_disagreement() { assert!(SummaryDescriptor::new( SummaryOperator::Configured { family: AggregationType::Rate.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Increase, - aggregation_sub_type: String::new(), - parameters: BTreeMap::new(), - }, - FidelityGuarantee::ExactCounter { - model: "prometheus.extrapolated-rate.v1".into(), - full_pane_coverage_required: true, }, + FidelityGuarantee::Exact, 2, ) .is_err()); } + // Configured identity preserves every heap/Hydra/state parameter of the + // family and names no population. #[test] - fn configured_identity_preserves_heap_hydra_and_subtype_and_excludes_population() { - let yaml:serde_yaml::Value=serde_yaml::from_str("aggregationType: DDSketch\naggregationSubType: ''\nmetric: m\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters:\n relative_accuracy: 0.01\nwindowSize: 30\nwindowType: tumbling\nspatialFilter: ''\n").unwrap(); - let mut config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); - assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, - FidelityGuarantee::DdSketchRelativeError { .. } - )); - for (kind, key) in [ - (AggregationType::CountMinSketchWithHeap, "heap_size"), - (AggregationType::HydraKLL, "row"), - (AggregationType::HydraKLL, "col"), - (AggregationType::HydraKLL, "k"), - ] { - config.aggregation_type = kind; - config.parameters.insert(key.into(), json!(10)); - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.parameters.insert(key.into(), json!(11)); - let after = SummaryDescriptor::from_config(&config).unwrap(); - assert_ne!(before.id, after.id, "{key}"); - config.metric = "other".into(); - config.spatial_filter_normalized = "job=a".into(); - assert_eq!( - after.id, - SummaryDescriptor::from_config(&config).unwrap().id - ); - } - let before = SummaryDescriptor::from_config(&config).unwrap(); - config.aggregation_sub_type = "max".into(); + fn configured_identity_preserves_state_parameters() { + use planner_types::post_asap::{GroupingStrategy, HydraKind, HydraParams, SketchKind}; + let heap = |heap_size| { + SummaryFamilyType::Sketch( + SketchKind::new( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width: 64, + depth: 4, + heap_size, + }, + ), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let hydra = |k, shared_buckets| { + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraKll, + params: HydraParams::HydraKll { k, shared_buckets }, + }, + ) + }; + let id = |family: &SummaryFamilyType| SummaryDescriptor::from_family(family).unwrap().id; + assert_ne!(id(&heap(10)), id(&heap(11))); + assert_ne!(id(&hydra(10, 10)), id(&hydra(10, 11))); assert_ne!( - before.id, - SummaryDescriptor::from_config(&config).unwrap().id + id(&hydra(10, 10)), + id(&SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 10 }), + GroupingStrategy::PerSubpopulationInstance, + )) ); + assert!(matches!( + SummaryDescriptor::from_family(&hydra(10, 10)).unwrap().fidelity, + FidelityGuarantee::KllRankError { ref model, .. } if model == "asap.hydra-kll.per-cell-rank.v1" + )); } + // Increase state declares the Prometheus counter fidelity model. #[test] - fn increase_config_declares_prometheus_counter_fidelity() { - let yaml: serde_yaml::Value = serde_yaml::from_str( - "aggregationType: Increase\naggregationSubType: ''\nmetric: requests_total\nlabels:\n grouping: []\n rollup: []\n aggregated: []\nparameters: {}\nwindowSize: 60\nwindowType: tumbling\nspatialFilter: ''\n", - ) - .unwrap(); - let config = - PrecomputeMaterialization::from_yaml_data(&yaml, None, crate::QueryLanguage::PromQl) - .unwrap(); + fn increase_declares_prometheus_counter_fidelity() { assert!(matches!( - SummaryDescriptor::from_config(&config).unwrap().fidelity, + SummaryDescriptor::from_family(&AggregationType::Increase.planner_exact_family().unwrap()) + .unwrap() + .fidelity, FidelityGuarantee::ExactCounter { ref model, full_pane_coverage_required: true @@ -1750,25 +1678,27 @@ mod tests { } #[test] fn canonical_nested_parameters_and_model_versions_are_identity() { - let a = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"z":1,"a":2}))]), + let a = SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([ + ("k".into(), json!(200)), + ("nested".into(), json!({"z":1,"a":2})), + ]), }; - let b = SummaryOperator::Configured { - family: AggregationType::Sum.planner_exact_family().unwrap(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: BTreeMap::from([("nested".into(), json!({"a":2,"z":1}))]), + let b = SummaryOperator::Sketch { + algorithm: SketchAlgorithm::Kll, + parameters: BTreeMap::from([ + ("nested".into(), json!({"a":2,"z":1})), + ("k".into(), json!(200)), + ]), + }; + let fidelity = FidelityGuarantee::KllRankError { + k: 200, + model: "rank.v1".into(), }; assert_eq!( - SummaryDescriptor::new(a.clone(), FidelityGuarantee::Exact, 1) - .unwrap() - .id, - SummaryDescriptor::new(b, FidelityGuarantee::Exact, 1) - .unwrap() - .id + SummaryDescriptor::new(a, fidelity.clone(), 1).unwrap().id, + SummaryDescriptor::new(b, fidelity, 1).unwrap().id ); let kll = SummaryOperator::Sketch { algorithm: SketchAlgorithm::Kll, diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index ec924e704..614f09750 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -12,7 +12,7 @@ use crate::sds::{ use crate::PolicyFingerprint; use serde::{Deserialize, Serialize}; -pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 6; +pub const SUMMARY_CATALOG_SCHEMA_VERSION: u32 = 7; /// Canonical definition binds operator and population descriptors. Writer /// layout and concrete state belong to installed plans and runtime instances. @@ -98,24 +98,51 @@ impl SummaryCatalog { }) } - pub fn from_materializations( + /// The catalog of every output of `plan`: its schema's state family and + /// its Planner DAG input predicate. + pub fn from_plan( + plan: &crate::precompute_plan::PrecomputePlan, + ) -> Result { + let outputs = plan + .materializations + .iter() + .map(|config| { + let family = plan.state_family(config.stored_output_id).ok_or( + SummaryCatalogError::MissingDescriptor(config.stored_output_id.as_u64()), + )?; + let filter = plan + .population_filter(config) + .map_err(SummaryCatalogError::Descriptor)?; + Ok((config, family, filter)) + }) + .collect::, SummaryCatalogError>>()?; + Self::from_outputs(plan.envelope.plan_id, plan.envelope.plan_version, outputs) + } + + /// Each output is paired with its state family and canonical input predicate. + pub fn from_outputs( plan_id: u64, plan_version: u64, - materializations: &[crate::PrecomputeMaterialization], + outputs: Vec<( + &crate::PrecomputeMaterialization, + &planner_types::post_asap::SummaryFamilyType, + String, + )>, ) -> Result { - let entries = materializations + let entries = outputs .iter() - .map(|config| { - let summary = SummaryDescriptor::from_config(config) + .map(|(config, family, filter)| { + let summary = SummaryDescriptor::from_family(family) .map_err(|error| SummaryCatalogError::Descriptor(error.to_string()))?; let source = config.source_identity(); let value_projection = config.effective_value_projection().clone(); + config + .table_population_canonical() + .map_err(SummaryCatalogError::Descriptor)?; let data = DataDescriptor::new_typed( source, value_projection, - config - .population_filter_canonical() - .map_err(SummaryCatalogError::Descriptor)?, + filter.clone(), config.grouping_labels.names(), if config.table_name.is_some() && !config.grouping_labels.is_empty() { crate::grouping_projection::TABLE_GROUP_OBSERVATION_SEMANTICS @@ -132,7 +159,7 @@ impl SummaryCatalog { .collect::, SummaryCatalogError>>()?; let mut catalog = Self::build(plan_id, plan_version, entries)?; let mut semantic_bindings = BTreeMap::new(); - for config in materializations { + for (config, _, _) in &outputs { let output = catalog .outputs .get_mut(&StoredOutputId::from(config.policy_fingerprint())) @@ -343,25 +370,51 @@ impl SummaryCatalog { mod tests { use super::*; use crate::sds::ValueProjectionIdentity; - use crate::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use crate::{KeyByLabelNames, PrecomputeMaterialization, WindowKind}; + use planner_types::post_asap::{ExactKind, ExactParams, SummaryFamilyType}; - fn config(metric: &str, filter: &str, window: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + fn sum() -> SummaryFamilyType { + SummaryFamilyType::ExactAggregate(ExactKind::Sum, ExactParams::Sum) + } + + fn allocated(mut config: PrecomputeMaterialization, filter: &str) -> PrecomputeMaterialization { + config.allocate_stored_output_id(&("sum", filter)); + config + } + + /// An allocated output with the given DAG input filter. + fn config(metric: &str, filter: &str, window: u64) -> (PrecomputeMaterialization, String) { + let config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::new(vec!["job".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), window, window, WindowKind::Tumbling, - filter.into(), - metric.into(), - None, - None, - None, + ); + (allocated(config, filter), filter.into()) + } + + fn catalog( + plan_id: u64, + plan_version: u64, + outputs: &[(PrecomputeMaterialization, String)], + ) -> Result { + with_family(plan_id, plan_version, outputs, &sum()) + } + + fn with_family( + plan_id: u64, + plan_version: u64, + outputs: &[(PrecomputeMaterialization, String)], + family: &SummaryFamilyType, + ) -> Result { + SummaryCatalog::from_outputs( + plan_id, + plan_version, + outputs + .iter() + .map(|(config, filter)| (config, family, filter.clone())) + .collect(), ) } @@ -370,7 +423,7 @@ mod tests { fn table_populations_have_distinct_materialization_and_data_identities() { use crate::table_population::{TableColumnPredicate, TablePopulation}; use planner_types::pre_asap::{CompareOpKind, ScalarValue}; - let mut requests = config("raw_samples.value", "", 60); + let (mut requests, _) = config("raw_samples.value", "", 60); requests.table_name = Some("raw_samples".into()); requests.value_projection = Some(ValueProjectionIdentity::Column { name: "value".into(), @@ -382,15 +435,17 @@ mod tests { value: ScalarValue::Utf8("requests".into()), }], }); + let requests = allocated(requests, ""); let mut errors = requests.clone(); errors.table_population.as_mut().unwrap().predicates[0].value = ScalarValue::Utf8("errors".into()); + let errors = allocated(errors, ""); assert_ne!(requests.policy_fingerprint(), errors.policy_fingerprint()); let mut other_table = requests.clone(); other_table.table_name = Some("other_samples".into()); assert_ne!( requests.policy_fingerprint(), - other_table.policy_fingerprint() + allocated(other_table, "").policy_fingerprint() ); let mut other_value = requests.clone(); other_value.value_projection = Some(ValueProjectionIdentity::Column { @@ -398,23 +453,27 @@ mod tests { }); assert_ne!( requests.policy_fingerprint(), - other_value.policy_fingerprint() + allocated(other_value, "").policy_fingerprint() ); let mut other_time = requests.clone(); other_time.table_timestamp_column = Some("event_time_ms".into()); + let other_time = allocated(other_time, ""); assert_ne!( requests.policy_fingerprint(), other_time.policy_fingerprint() ); - let mut invalid_labels = requests.clone(); - invalid_labels.table_population = None; - invalid_labels.spatial_filter = "job=\"requests\"".into(); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_labels]).is_err()); let mut invalid_time = requests.clone(); invalid_time.table_timestamp_column = Some("time; DROP TABLE samples".into()); - assert!(SummaryCatalog::from_materializations(1, 1, &[invalid_time]).is_err()); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[requests, errors, other_time]).unwrap(); + assert!(catalog(1, 1, &[(allocated(invalid_time, ""), String::new())]).is_err()); + let catalog = catalog( + 1, + 1, + &[requests, errors, other_time].map(|config| { + let filter = config.table_population_canonical().unwrap(); + (config, filter) + }), + ) + .unwrap(); assert_eq!(catalog.data_descriptors.len(), 3); assert_eq!(catalog.outputs.len(), 3); } @@ -423,10 +482,10 @@ mod tests { fn snapshot_reference_is_deterministic_and_content_sensitive() { let a = config("requests", "", 60); let b = config("errors", "", 60); - let left = SummaryCatalog::from_materializations(1, 2, &[a.clone(), b.clone()]).unwrap(); - let reordered = SummaryCatalog::from_materializations(1, 2, &[b, a.clone()]).unwrap(); + let left = catalog(1, 2, &[a.clone(), b.clone()]).unwrap(); + let reordered = catalog(1, 2, &[b, a.clone()]).unwrap(); assert_eq!(left.reference().unwrap(), reordered.reference().unwrap()); - let changed = SummaryCatalog::from_materializations(1, 2, &[a]).unwrap(); + let changed = catalog(1, 2, &[a]).unwrap(); assert_ne!(left.reference().unwrap(), changed.reference().unwrap()); left.reference() .unwrap() @@ -444,8 +503,7 @@ mod tests { fn shares_descriptors_across_windows_and_deduplicates_materializations() { let one = config("requests", "", 60); let two = config("requests", "", 120); - let catalog = - SummaryCatalog::from_materializations(7, 2, &[one.clone(), two, one]).unwrap(); + let catalog = catalog(7, 2, &[one.clone(), two, one]).unwrap(); assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); assert_eq!(catalog.outputs.len(), 2); @@ -455,7 +513,7 @@ mod tests { // Stored pane duration identifies a deployment output, not the summary meaning. #[test] fn semantic_definition_is_shared_across_deployed_pane_outputs() { - let catalog = SummaryCatalog::from_materializations( + let catalog = catalog( 1, 1, &[config("requests", "", 60), config("requests", "", 120)], @@ -469,10 +527,10 @@ mod tests { #[test] fn hot_and_rebuild_have_one_definition_and_two_bound_outputs() { let mut hot = config("latency", "", 60); - hot.stored_output_id = Some(StoredOutputId(41)); + hot.0.stored_output_id = StoredOutputId(41); let mut rebuild = hot.clone(); - rebuild.stored_output_id = Some(StoredOutputId(42)); - let catalog = SummaryCatalog::from_materializations(7, 42, &[hot, rebuild]).unwrap(); + rebuild.0.stored_output_id = StoredOutputId(42); + let catalog = catalog(7, 42, &[hot, rebuild]).unwrap(); assert_eq!(catalog.definitions.len(), 1); let hot = catalog.output_reference(StoredOutputId(41)).unwrap(); let rebuild = catalog.output_reference(StoredOutputId(42)).unwrap(); @@ -499,7 +557,7 @@ mod tests { // Source/population changes never alias, while the operator can be reused. #[test] fn separates_population_and_operator_identity() { - let catalog = SummaryCatalog::from_materializations( + let catalog = catalog( 1, 1, &[ @@ -516,15 +574,23 @@ mod tests { // Operator changes share population metadata without sharing state identity. #[test] fn changing_operator_parameters_creates_a_new_summary_descriptor() { - let mut a = config("requests", "", 60); - a.aggregation_type = AggregationType::DatasketchesKLL; - a.parameters.insert("K".into(), serde_json::json!(100)); - let mut b = a.clone(); - b.parameters.insert("K".into(), serde_json::json!(200)); - let catalog = SummaryCatalog::from_materializations(1, 1, &[a, b]).unwrap(); - assert_eq!(catalog.summary_descriptors.len(), 2); + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, + }; + let kll = |k| { + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let a = with_family(1, 1, &[config("requests", "", 60)], &kll(100)).unwrap(); + let b = with_family(1, 1, &[config("requests", "", 60)], &kll(200)).unwrap(); + assert_ne!(a.summary_descriptors, b.summary_descriptors); + assert_eq!(a.data_descriptors, b.data_descriptors); + let catalog = a; + assert_eq!(catalog.summary_descriptors.len(), 1); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.outputs.len(), 2); + assert_eq!(catalog.outputs.len(), 1); } // Construction order cannot affect the published snapshot bytes. @@ -532,8 +598,8 @@ mod tests { fn snapshot_is_deterministic_and_round_trips() { let a = config("requests", "", 60); let b = config("errors", "", 60); - let forward = SummaryCatalog::from_materializations(7, 2, &[a.clone(), b.clone()]).unwrap(); - let backward = SummaryCatalog::from_materializations(7, 2, &[b, a]).unwrap(); + let forward = catalog(7, 2, &[a.clone(), b.clone()]).unwrap(); + let backward = catalog(7, 2, &[b, a]).unwrap(); let bytes = serde_json::to_vec(&forward).unwrap(); assert_eq!(bytes, serde_json::to_vec(&backward).unwrap()); let decoded: SummaryCatalog = serde_json::from_slice(&bytes).unwrap(); @@ -543,10 +609,11 @@ mod tests { #[test] fn catalog_definitions_do_not_store_writer_layout() { - let mut materialization = config("requests", "", 60); + let (mut materialization, _) = config("requests", "", 60); materialization.pane_origin_ms = Some(7_000); + let materialization = allocated(materialization, ""); let id = StoredOutputId::from(materialization.policy_fingerprint()); - let catalog = SummaryCatalog::from_materializations(7, 2, &[materialization]).unwrap(); + let catalog = catalog(7, 2, &[(materialization, String::new())]).unwrap(); assert!(catalog.outputs.contains_key(&id)); assert!( !serde_json::to_value(&catalog).unwrap()["outputs"][id.as_u64().to_string()] @@ -565,8 +632,7 @@ mod tests { fn conflicting_materialization_is_rejected() { let first = config("requests", "", 60); let second = config("errors", "", 60); - let catalog = - SummaryCatalog::from_materializations(1, 1, &[first.clone(), second]).unwrap(); + let catalog = catalog(1, 1, &[first.clone(), second]).unwrap(); let summary = catalog.summary_descriptors.values().next().unwrap().clone(); let data = catalog .data_descriptors @@ -577,8 +643,12 @@ mod tests { 1, 1, [ - (first.policy_fingerprint(), summary.clone(), data[0].clone()), - (first.policy_fingerprint(), summary, data[1].clone()), + ( + first.0.policy_fingerprint(), + summary.clone(), + data[0].clone(), + ), + (first.0.policy_fingerprint(), summary, data[1].clone()), ], ) .unwrap_err(); @@ -591,8 +661,7 @@ mod tests { // Imported catalogs must resolve every foreign key and content identity. #[test] fn rejects_dangling_refs_tampered_keys_and_unknown_schema() { - let catalog = - SummaryCatalog::from_materializations(1, 1, &[config("requests", "", 60)]).unwrap(); + let catalog = catalog(1, 1, &[config("requests", "", 60)]).unwrap(); let mut broken = catalog.clone(); broken.data_descriptors.clear(); assert!(matches!( @@ -631,7 +700,7 @@ mod tests { // Native exact-only plans have a valid empty catalog, not dummy state. #[test] fn empty_catalog_is_valid() { - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); + let catalog = catalog(1, 1, &[]).unwrap(); assert!(catalog.outputs.is_empty()); catalog.validate().unwrap(); } diff --git a/crates/asap_types/src/summary_semantics.rs b/crates/asap_types/src/summary_semantics.rs index 757a7e15b..b8ef77da5 100644 --- a/crates/asap_types/src/summary_semantics.rs +++ b/crates/asap_types/src/summary_semantics.rs @@ -24,8 +24,6 @@ pub enum SummarySemantics { Configured { computation: Box, value_source_column: Option, - aggregated_labels: crate::KeyByLabelNames, - rollup_labels: crate::KeyByLabelNames, }, } @@ -57,8 +55,6 @@ impl SummaryDefinition { semantic_format_version: 1, semantics: SummarySemantics::Configured { value_source_column: None, - aggregated_labels: crate::KeyByLabelNames::empty(), - rollup_labels: crate::KeyByLabelNames::empty(), computation: Box::new(SummaryComputation { operator: summary.operator.clone(), source: data.source.clone(), @@ -80,14 +76,10 @@ impl SummaryDefinition { }; } else if let SummarySemantics::Configured { value_source_column, - aggregated_labels, - rollup_labels, .. } = &mut self.semantics { *value_source_column = config.value_source_column.clone(); - *aggregated_labels = config.aggregated_labels.clone(); - *rollup_labels = config.rollup_labels.clone(); } self } diff --git a/data_plane/benches/sketch_db.rs b/data_plane/benches/sketch_db.rs index 48f02c7ed..0d58010f6 100644 --- a/data_plane/benches/sketch_db.rs +++ b/data_plane/benches/sketch_db.rs @@ -401,30 +401,20 @@ fn matching_streaming_config( use asap_types::enums::WindowKind; use asap_types::AggregationType as AT; use asap_types::KeyByLabelNames; - use std::collections::HashMap; - let cfg = PrecomputeMaterialization::new( - AT::Sum, - String::new(), - HashMap::new(), + let family = AT::Sum.planner_exact_family().unwrap(); + let mut cfg = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, ); + cfg.allocate_stored_output_id(&family); let runtime = data_plane::storage_engines::types::InstalledPrecomputePlan::default(); - let plan = control_plane::physical::compiler::PrecomputePlan::build( + let plan = control_plane::physical::compiler::PrecomputePlan::build_backend_local( runtime.plan().envelope.clone(), - vec![cfg], - &[], + vec![(cfg, family)], ) .unwrap(); data_plane::storage_engines::types::InstalledPrecomputePlan::from_precompute_plan(plan).unwrap() diff --git a/data_plane/examples/audit_clickhouse_fallback.rs b/data_plane/examples/audit_clickhouse_fallback.rs index 02e2f085e..742d7c490 100644 --- a/data_plane/examples/audit_clickhouse_fallback.rs +++ b/data_plane/examples/audit_clickhouse_fallback.rs @@ -69,7 +69,7 @@ async fn main() { capability_snapshot_id: "sql27-main-eval".into(), }; let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(27, 1, &[]).unwrap(); + asap_types::summary_catalog::SummaryCatalog::from_outputs(27, 1, Vec::new()).unwrap(); let reference = catalog.reference().unwrap(); let mut precompute_plan = PrecomputePlan::build_backend_local(envelope.clone(), vec![]).unwrap(); diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 5a3ef0eb4..5691b0ddc 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -684,10 +684,7 @@ async fn route_otlp_to_precompute( let ts_ms = (point.timestamp_nanos / 1_000_000) as i64; let mut matched = false; for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { + if config.metric != point.name { continue; } let group_key = IngestState::extract_group_key_for(&series_key, config); @@ -761,10 +758,7 @@ async fn route_otlp_to_precompute( let sketch_type = identify_sketch_type(&point.payload); let mut matched = false; for config in agg_configs.values() { - if config.metric != point.name - && config.spatial_filter_normalized != point.name - && config.spatial_filter != point.name - { + if config.metric != point.name { continue; } let group_key = IngestState::extract_group_key_for(&series_key, config); @@ -1399,18 +1393,15 @@ async fn route_modified_otlp_sketches_to_precompute( // into the matched policy's parameters — recorded on the sid // below so the query engine can answer per-item estimate(key) // (the CMS/CountSketch FrequencyEstimate gate consults it). - let item_label_for_sid: Option = { - snap.get_aggregation_config(policy_fp.into()) - .or_else(|| { - snap.materializations() - .values() - .find(|c| c.metric == canonical_name) - }) - .and_then(|c| c.parameters.get("item_label")) - .and_then(|v| v.as_str()) - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - }; + let item_label_for_sid: Option = snap + .get_aggregation_config(policy_fp.into()) + .or_else(|| { + snap.materializations() + .values() + .find(|c| c.metric == canonical_name) + }) + .and_then(|c| snap.item_label(c.stored_output_id)) + .map(str::to_owned); ingest_state.summary_store.register(SummarySeriesMetadata { storage_handle: sid, metric_name: canonical_name.clone(), @@ -1763,11 +1754,7 @@ async fn route_modified_otlp_sketches_to_precompute( &asap_types::aggregation_config::PrecomputeMaterialization, > = agg_configs .values() - .filter(|config| { - config.metric == canonical_name - || config.spatial_filter_normalized == canonical_name - || config.spatial_filter == canonical_name - }) + .filter(|config| config.metric == canonical_name) .collect(); let matched_any = !matching_configs.is_empty(); @@ -1887,87 +1874,6 @@ async fn route_modified_otlp_sketches_to_precompute( }) } -/// Map `SketchAlgorithm` to the corresponding wire-format -/// `AggregationType`. Inverse direction is in -/// `sketch_algorithm_for` above. Used by -/// [`derive_sketch_policy_fp`] to find the policy whose -/// `PrecomputeMaterialization.aggregation_type` matches a freshly-ingested -/// sketch. -/// -/// `Any` is a control-plane analysis-time wildcard — it doesn't -/// appear on the ingest path. Returns `None` so the policy lookup -/// fails the (rare) defensive path explicitly. -fn aggregation_type_for_sketch_algorithm( - handle: crate::storage_engines::sketch_db::index::SketchAlgorithm, -) -> Option { - use crate::storage_engines::sketch_db::index::SketchAlgorithm; - use asap_types::AggregationType; - match handle { - SketchAlgorithm::DDSketch => Some(AggregationType::DDSketch), - SketchAlgorithm::Kll => Some(AggregationType::DatasketchesKLL), - SketchAlgorithm::Hll => Some(AggregationType::HLL), - SketchAlgorithm::UnivMon => Some(AggregationType::UnivMon), - SketchAlgorithm::CountSketch => Some(AggregationType::CountSketch), - SketchAlgorithm::CountSketchWithHeap => Some(AggregationType::CountSketchWithHeap), - SketchAlgorithm::Cms => Some(AggregationType::CountMinSketch), - SketchAlgorithm::CmsWithHeap => Some(AggregationType::CountMinSketchWithHeap), - SketchAlgorithm::Kmv | SketchAlgorithm::Theta => None, - } -} - -/// Render a `SketchConfig` into the param map the streaming-config -/// stores. The control plane authors these as -/// `parameters: {: }` JSON; the data plane has the -/// parameters typed in `SketchConfig`. This function converts. -/// -/// Keys MUST match what the control plane emits (see -/// `crates/asap_types/src/aggregation_config.rs::from_yaml_data` for -/// the canonical names). Drift here surfaces as policy lookups that -/// silently miss. -fn sketch_config_to_params( - cfg: &crate::storage_engines::sketch_db::data::SketchConfig, -) -> std::collections::HashMap { - use crate::storage_engines::sketch_db::data::SketchConfig; - let mut params = std::collections::HashMap::new(); - match cfg { - SketchConfig::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - } => { - params.insert("heap_size".into(), serde_json::json!(heap_size)); - params.insert("sketch_rows".into(), serde_json::json!(sketch_rows)); - params.insert("sketch_cols".into(), serde_json::json!(sketch_cols)); - params.insert("layers".into(), serde_json::json!(layers)); - } - SketchConfig::DDSketch { relative_accuracy } => { - params.insert( - "relative_accuracy".to_string(), - serde_json::json!(*relative_accuracy), - ); - } - SketchConfig::Kll { k } => { - params.insert("k".to_string(), serde_json::json!(*k)); - } - SketchConfig::Hll { precision } => { - params.insert("precision".to_string(), serde_json::json!(*precision)); - } - SketchConfig::CountSketch { rows, cols } | SketchConfig::CountMin { rows, cols } => { - // Canonical key mapping (matches the controller's - // `sketch_params_to_json` in - // `control_plane::emit::stage_config`): `w` is the - // matrix width (=cols), `d` is the depth (=rows). The - // controller writes `{w, d}` into the streaming-config - // `parameters`, so the policy_fp content match has to - // probe the same keys. - params.insert("w".to_string(), serde_json::json!(*cols)); - params.insert("d".to_string(), serde_json::json!(*rows)); - } - } - params -} - // Snapshot and delta bases are scoped to the producer, never only its semantics. fn bound_sketch_series_key( name: &str, @@ -1997,21 +1903,34 @@ fn derive_sketch_policy_fp( group_by_keys: &std::collections::BTreeSet, bound_output: Option, ) -> asap_types::PolicyFingerprint { - let Some(agg_type) = aggregation_type_for_sketch_algorithm(kind) else { - return asap_types::PolicyFingerprint::UNSET; - }; - let params = sketch_config_to_params(cfg); + use crate::storage_engines::sketch_db::index::AggKind; + // An unfiltered output of this exact sketch state and grouping; ambiguous + // or absent matches fail closed. + let expected = AggKind::Sketch { + algorithm: kind, + config: cfg.clone(), + spatial_filter_canonical: String::new(), + } + .canonical_string(); let snap = ingest_state.config_snapshot(); - let registry = snap.policy_registry(); - let registry = if let Some(output) = bound_output { - asap_types::PolicyRegistry::from_configs(registry.get(output.fingerprint()).cloned()) - } else { - registry - }; - let index = asap_types::RoutingIndex::build(registry); - index - .find_policy_by_content(metric, group_by_keys, agg_type, ¶ms) - .unwrap_or(asap_types::PolicyFingerprint::UNSET) + let mut hit = None; + for config in snap.materializations().values() { + let keys: std::collections::BTreeSet<_> = config.grouping_labels.iter().cloned().collect(); + if bound_output.is_some_and(|output| output != config.stored_output_id) + || config.metric != metric + || &keys != group_by_keys + || snap + .agg_kind(config.stored_output_id) + .is_none_or(|installed| installed.canonical_string() != expected) + { + continue; + } + if hit.is_some() { + return asap_types::PolicyFingerprint::UNSET; + } + hit = Some(config.policy_fingerprint()); + } + hit.unwrap_or(asap_types::PolicyFingerprint::UNSET) } /// Phase 5 helper — map a `ModifiedOtlpSketchDp` to the matching @@ -3325,70 +3244,6 @@ mod policy_fp_lookup_tests { use crate::storage_engines::sketch_db::data::SketchConfig; use crate::storage_engines::sketch_db::index::SketchAlgorithm; use asap_types::AggregationType; - - #[test] - fn handle_to_agg_type_round_trips_canonical_kinds() { - // Locks in the data-plane → control-plane name mapping. - // Drift surfaces as policy lookups that silently miss because - // the handle resolves to an `AggregationType` no policy uses. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::DDSketch), - Some(AggregationType::DDSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kll), - Some(AggregationType::DatasketchesKLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Hll), - Some(AggregationType::HLL) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Cms), - Some(AggregationType::CountMinSketch) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CmsWithHeap), - Some(AggregationType::CountMinSketchWithHeap) - ); - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::CountSketch), - Some(AggregationType::CountSketch) - ); - // `Any` is a control-plane wildcard, not a real DP shape. - assert_eq!( - aggregation_type_for_sketch_algorithm(SketchAlgorithm::Kmv), - None - ); - } - - #[test] - fn sketch_config_to_params_uses_canonical_keys() { - // The param-name vocabulary must match what the control plane - // writes in streaming-config YAML (see - // `asap_types::aggregation_config::PrecomputeMaterialization::from_yaml_data`). - // Drift surfaces as `find_policy_by_content` missing matches. - let dd = sketch_config_to_params(&SketchConfig::DDSketch { - relative_accuracy: 0.01, - }); - assert_eq!(dd.get("relative_accuracy"), Some(&serde_json::json!(0.01))); - - let kll = sketch_config_to_params(&SketchConfig::Kll { k: 200 }); - assert_eq!(kll.get("k"), Some(&serde_json::json!(200))); - - let hll = sketch_config_to_params(&SketchConfig::Hll { precision: 14 }); - assert_eq!(hll.get("precision"), Some(&serde_json::json!(14))); - - let cs = sketch_config_to_params(&SketchConfig::CountSketch { rows: 4, cols: 256 }); - // Canonical keys: w (=cols, width) and d (=rows, depth) — - // matches `control_plane::emit::stage_config::sketch_params_to_json`. - assert_eq!(cs.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cs.get("d"), Some(&serde_json::json!(4))); - - let cm = sketch_config_to_params(&SketchConfig::CountMin { rows: 4, cols: 256 }); - assert_eq!(cm.get("w"), Some(&serde_json::json!(256))); - assert_eq!(cm.get("d"), Some(&serde_json::json!(4))); - } } #[cfg(test)] @@ -4481,23 +4336,19 @@ mod sid_bucketing_tests { } fn sum_agg_config(metric: &str, grouping: &[&str]) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::SingleSubpopulation, - "Sum".to_string(), - HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 10, 10, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.allocate_stored_output_id(&sum_family()); + config + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } /// Build a Gauge request with one DataPoint per (zone, value) entry. @@ -4573,9 +4424,7 @@ mod sid_bucketing_tests { let metric = "cpu_seconds"; let cfg = sum_agg_config(metric, &["zone"]); let policy_fp = asap_types::PolicyFingerprint(cfg.policy_fp_u64()); - let mut configs = HashMap::new(); - configs.insert(cfg.policy_fp_u64(), cfg.clone()); - let streaming = InstalledPrecomputePlan::from_raw_ids(configs); + let streaming = InstalledPrecomputePlan::new([(cfg.clone(), sum_family())]); let hot_reload = InstalledPrecomputePlanHandle::new(streaming); let resolver = Arc::new(SeriesIdResolver::new()); @@ -4659,8 +4508,10 @@ mod sid_bucketing_tests { // regardless of group_key shape — the test pin is on sid // assignment, not on group_key content), then verify the // sid matches the resolver mint for THAT zone. - let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(&cfg); + let agg_kind_canonical = crate::storage_engines::sketch_db::data::materialization_kind( + &cfg, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), ""), + ); for (sid, _, _, samples) in &groups { let mut vals: Vec = samples.iter().map(|(_, _, v)| *v).collect(); vals.sort_by(|a, b| a.partial_cmp(b).unwrap()); @@ -4693,7 +4544,7 @@ mod sid_bucketing_tests { ("asap.frame.plan_version".into(), "3".into()), ( "asap.frame.backend_compat".into(), - "asap-query-backend.v1".into(), + "asap-query-backend.v2".into(), ), ("asap.frame.materialization".into(), "99".into()), ( diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index ff117d7b0..14fe449b4 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -452,11 +452,7 @@ impl PrometheusRemoteWriteReceiver { .zip(group_key.values().labels) .collect(); } - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = snapshot.right_closed_panes((*policy_fp).into()); let affected = crate::precompute_engine::maintenance_runtime::affected_materializations( &physical_plan.precompute_plan, (*policy_fp).into(), @@ -726,7 +722,7 @@ fn route_messages( .values() .filter(|config| config.derived_input.is_none()) .filter_map(|config| { - compile_spatial_filter(&config.spatial_filter_normalized) + compile_spatial_filter(snapshot.population_filter(config.stored_output_id)) .ok() .map(|filter| (config, filter)) }) @@ -754,11 +750,14 @@ fn route_messages( == Some(asap_types::sds::PopulationPartitioning::PerEntity) || (config.partitioning.is_none() && matches!( - config.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::Rate - | asap_types::AggregationType::Min - | asap_types::AggregationType::Max + snapshot.state_family(config.stored_output_id), + Some(planner_types::post_asap::SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate + | planner_types::post_asap::ExactKind::Min + | planner_types::post_asap::ExactKind::Max, + _ + )) )); let grouping_pairs: Vec<(&str, &str)> = if series_scoped { Vec::new() @@ -986,14 +985,25 @@ mod tests { planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - let configs = streaming - .materializations_by_output - .values() - .cloned() - .collect::>(); let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 3, &configs) - .unwrap(), + asap_types::summary_catalog::SummaryCatalog::from_outputs( + 7, + 3, + streaming + .materializations_by_output + .values() + .map(|config| { + ( + config, + streaming.state_family(config.stored_output_id).unwrap(), + streaming + .population_filter(config.stored_output_id) + .to_owned(), + ) + }) + .collect(), + ) + .unwrap(), ); let reference = catalog.reference().unwrap(); let generation = asap_types::sds::CatalogGeneration { @@ -1066,37 +1076,19 @@ mod tests { fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let aggregation = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(vec!["job".into()]).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 60, - slide_interval: 60, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: "requests_total".into(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let policy_fp = aggregation.policy_fp_u64(); + let mut aggregation = PrecomputeMaterialization::new( + "requests_total", + KeyByLabelNames::new(vec!["job".into()]), + 60, + 60, + WindowKind::Tumbling, + ); + aggregation.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }; let streaming = - InstalledPrecomputePlan::from_raw_ids(HashMap::from([(policy_fp, aggregation)])); + InstalledPrecomputePlan::new([crate::tests::test_utilities::outputs::allocated( + aggregation, + AggregationType::Sum.planner_exact_family().unwrap(), + )]); let (sender, receiver) = mpsc::channel(8); let ingest = Arc::new(IngestState { router: SeriesRouter::new(vec![sender]), @@ -1134,10 +1126,13 @@ mod tests { let mut config = snapshot.precompute_plan.materializations[0].clone(); config.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; config.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let hot = physical_config(InstalledPrecomputePlan::from_raw_ids(HashMap::from([( - config.policy_fp_u64(), - config.clone(), - )]))); + let (config, family) = crate::tests::test_utilities::outputs::allocated( + config, + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap(), + ); + let hot = physical_config(InstalledPrecomputePlan::new([(config.clone(), family)])); let physical = hot.active_physical_plan_snapshot().unwrap(); // A direct routing fixture; public installation remains intentionally gated. let (sender, _worker) = mpsc::channel(8); @@ -1195,67 +1190,42 @@ mod tests { use asap_types::enums::WindowKind; use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization}; - let config = |aggregation_type, grouping: Vec, aggregated: Vec| { - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + let config = |aggregation_type: AggregationType, grouping: Vec, partitioning| { + let family = crate::tests::test_utilities::outputs::family( aggregation_type, - aggregation_sub_type: String::new(), - parameters: match aggregation_type { - AggregationType::CountMinSketchWithHeap => HashMap::from([ - ("w".into(), serde_json::json!(128)), - ("d".into(), serde_json::json!(5)), - ("heap_size".into(), serde_json::json!(2)), - ]), - AggregationType::DatasketchesKLL => { - HashMap::from([("k".into(), serde_json::json!(200))]) - } - _ => HashMap::new(), - }, - grouping_labels: KeyByLabelNames::new(grouping).into(), - aggregated_labels: KeyByLabelNames::new(aggregated), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 60, - slide_interval: 60, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }, - pane_origin_ms: Some(0), - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: "cpu_seconds_total".into(), - num_aggregates_to_retain: Some(80), - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + &serde_json::json!({"w": 128, "d": 5, "heap_size": 2, "k": 200}), + ); + let mut config = PrecomputeMaterialization::new( + "cpu_seconds_total", + KeyByLabelNames::new(grouping), + 60, + 60, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 60 }; + config.pane_origin_ms = Some(0); + config.num_aggregates_to_retain = Some(80); + config.partitioning = partitioning; + crate::tests::test_utilities::outputs::allocated(config, family) }; - let cms = config( - AggregationType::CountMinSketchWithHeap, + let cms = config(AggregationType::CountMinSketchWithHeap, vec![], None); + let counter = config(AggregationType::Increase, vec!["job".into()], None); + let kll = config( + AggregationType::DatasketchesKLL, + vec![], + Some(asap_types::sds::PopulationPartitioning::PerEntity), + ); + let kll_fp = kll.0.policy_fingerprint(); + let pooled_kll = config( + AggregationType::DatasketchesKLL, vec![], - vec!["job".into()], + Some(asap_types::sds::PopulationPartitioning::Grouped), ); - let counter = config(AggregationType::Increase, vec!["job".into()], vec![]); - let mut kll = config(AggregationType::DatasketchesKLL, vec![], vec![]); - kll.partitioning = Some(asap_types::sds::PopulationPartitioning::PerEntity); - let kll_fp = kll.policy_fingerprint(); - let mut pooled_kll = kll.clone(); - pooled_kll.partitioning = Some(asap_types::sds::PopulationPartitioning::Grouped); - let pooled_kll_fp = pooled_kll.policy_fingerprint(); + let pooled_kll_fp = pooled_kll.0.policy_fingerprint(); assert_ne!(kll_fp, pooled_kll_fp); - let cms_fp = cms.policy_fingerprint(); - let counter_fp = counter.policy_fingerprint(); - let streaming = InstalledPrecomputePlan::from_raw_ids(HashMap::from([ - (cms_fp.0, cms), - (counter_fp.0, counter), - (kll_fp.0, kll), - (pooled_kll_fp.0, pooled_kll), - ])); + let cms_fp = cms.0.policy_fingerprint(); + let counter_fp = counter.0.policy_fingerprint(); + let streaming = InstalledPrecomputePlan::new([cms, counter, kll, pooled_kll]); let hot_reload = physical_config(streaming); let physical_plan = hot_reload.active_physical_plan_snapshot().unwrap(); let (sender, _worker) = mpsc::channel(8); @@ -1824,7 +1794,9 @@ impl PrometheusRemoteWriteReceiver { return Err(crate::precompute_engine::revisions::AdmissionRejected("Remote Write request exceeds the correction horizon or contains future input").into()); } for config in plan.precompute_plan.materializations.iter().filter(|c| c.derived_input.is_none()) { - let filter = compile_spatial_filter(&config.spatial_filter_normalized)?; + let filter = compile_spatial_filter( + plan.installed_precompute_plan.population_filter(config.stored_output_id), + )?; let program = plan.installed_precompute_plan.raw_programs.get(&config.policy_fp_u64()).ok_or("revision raw program missing")?; let mut matching = samples.iter().filter_map(|sample| { let labels = sample.labels.iter().map(|(k,v)|(k.clone(),v.clone())).collect(); @@ -1921,7 +1893,7 @@ impl PrometheusRemoteWriteReceiver { if let Some(previous) = committed.get(&output.0) { let config = plan.precompute_plan.materializations.iter().find(|c| c.policy_fp_u64() == output.0).ok_or("committed raw output missing")?; let mut groups: BTreeMap, BTreeMap<(u64,u64),Arc>> = BTreeMap::new(); - for record in previous { groups.entry(record.group.clone()).or_default().insert((record.start_ms,record.end_ms),crate::precompute_engine::revisions::decode_state(record,config)?); } + for record in previous { groups.entry(record.group.clone()).or_default().insert((record.start_ms,record.end_ms),crate::precompute_engine::revisions::decode_state(record, plan.precompute_plan.state_family(config.stored_output_id).ok_or("committed raw output has no state schema")?)?); } for (group,windows) in groups { frozen.push(crate::storage_engines::sketch_db::index::FrozenExactWindows { stored_output_reference: plan.installed_precompute_plan.stored_output_reference(output).ok_or("committed raw binding missing")?, storage_handle: output.0, definition: output, generation:Arc::new(generation.clone()), group, windows, singleton_population_complete:true, diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 610f57a5b..19556793b 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -2531,7 +2531,7 @@ mod tests { capability_snapshot_id: "test".into(), }; let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(7, 1, &[]).unwrap(), + asap_types::summary_catalog::SummaryCatalog::from_outputs(7, 1, Vec::new()).unwrap(), ); let generation = catalog.reference().unwrap(); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -3267,46 +3267,30 @@ mod tests { // retirement) can find them. The matching sid in the catalog // is registered via the canonical ingest path so its content // hash matches what `create_checked` would compute. - let mut agg_map = HashMap::new(); + let mut agg_map = Vec::new(); let mut marker_to_fp = std::collections::HashMap::new(); for marker in active_agg_ids { let metric = format!("metric_{marker}"); - let cfg = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::empty().into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.clone(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; + let mut cfg = PrecomputeMaterialization::new( + metric.clone(), + KeyByLabelNames::empty(), + 1, + 1, + WindowKind::Tumbling, + ); + cfg.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + let cfg = crate::tests::test_utilities::outputs::allocated( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + ); // PR 5: streaming-config is keyed on the policy // fingerprint. Build a marker→fingerprint map so the test // POSTs the right id on the wire. - let fp = cfg.policy_fp_u64(); + let fp = cfg.0.policy_fp_u64(); marker_to_fp.insert(*marker, fp); - agg_map.insert(fp, cfg); + agg_map.push(cfg); } - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::from_raw_ids(agg_map)); + let installed_precompute_plan = Arc::new(InstalledPrecomputePlan::new(agg_map)); let hot_reload = InstalledPrecomputePlanHandle::from_arc(installed_precompute_plan.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); let summary_store = Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); @@ -3687,10 +3671,8 @@ mod tests { }; // Pin `storage_backend` on the streaming config so the http // dispatcher reads it back through the hot-reload handle. - let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( - Default::default(), - metric_storage_backend, - ); + let streaming_cfg = + InstalledPrecomputePlan::with_storage_backend(Vec::new(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); @@ -5017,10 +4999,8 @@ mod tests { handle_http_requests: true, adapter_config, }; - let streaming_cfg = InstalledPrecomputePlan::with_storage_backend( - Default::default(), - metric_storage_backend, - ); + let streaming_cfg = + InstalledPrecomputePlan::with_storage_backend(Vec::new(), metric_storage_backend); let streaming_arc = Arc::new(streaming_cfg); let hot_reload = InstalledPrecomputePlanHandle::from_arc(streaming_arc.clone()); let query_engine = Arc::new(ASAPQueryEngine::new(15000)); @@ -6557,6 +6537,25 @@ mod catalog_install_tests { ) } + // A physical plan compiled before materializations became deployment-only + // is refused at decode by its schema version, not by a field-level error. + #[test] + fn install_request_from_the_computation_carrying_schema_is_rejected_by_version() { + let mut encoded = serde_json::to_value(request()).unwrap(); + assert!(serde_json::from_value::(encoded.clone()).is_ok()); + encoded["precompute_plan"]["envelope"]["backend_compat"] = + serde_json::json!("asap-query-backend.v1"); + encoded["precompute_plan"]["materializations"][0]["aggregation_type"] = + serde_json::json!("DDSketch"); + let error = serde_json::from_value::(encoded) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsupported installed precompute plan schema asap-query-backend.v1"), + "{error}" + ); + } + #[test] fn complete_dag_cannot_be_installed_as_maintenance() { let mut request = request(); diff --git a/data_plane/src/precompute_engine/erp_observer.rs b/data_plane/src/precompute_engine/erp_observer.rs index e97045fe5..c1384acb6 100644 --- a/data_plane/src/precompute_engine/erp_observer.rs +++ b/data_plane/src/precompute_engine/erp_observer.rs @@ -56,18 +56,22 @@ impl RuntimeErpObserver { &self, generation: &CatalogGeneration, coordinates: SummaryInstanceCoordinates, - config: &asap_types::PrecomputeMaterialization, + program: &crate::precompute_engine::raw_dag::RawDagProgram, timestamp_ms: i64, value: f64, ) { - use asap_types::{AggregationType, SampleUpdateRule}; - let (semantics, sketch, implementation) = match config.aggregation_type { - AggregationType::HLL => ( + use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; + let algorithm = match &program.family { + SummaryFamilyType::Sketch(kind, _) => kind.algorithm(), + _ => return, + }; + let (semantics, sketch, implementation) = match algorithm { + SketchAlgorithm::Hll => ( ErpObservationInputSemantics::ScalarSampleValue, "hll", "asap-sketchlib-hll-regular-v1", ), - AggregationType::UnivMon => ( + SketchAlgorithm::UnivMon => ( ErpObservationInputSemantics::UnitSampleFrequency, "univmon", "asap-sketchlib-univmon-standard-v1", @@ -83,12 +87,10 @@ impl RuntimeErpObserver { if state.invalid.is_some() { return; } - if !value.is_finite() - || !matches!( - config.sample_update_rule(), - SampleUpdateRule::Value { scale: 1.0 } - ) - { + // Both implemented raw update domains observe the raw sample value. + let untransformed = asap_types::accumulator_spec::is_scalar_sample_value(&program.input) + || asap_types::accumulator_spec::is_unit_sample_frequency(&program.input); + if !value.is_finite() || !untransformed { state.invalid = Some("unsupported non-finite or transformed summary input".into()); for population in state.populations.values_mut() { population.observer = BoundedFrequencyObserver::new(1, 64).unwrap(); @@ -264,23 +266,20 @@ impl RuntimeErpObserver { #[cfg(test)] mod tests { use super::*; - fn fixture() -> (CatalogGeneration, asap_types::PrecomputeMaterialization) { - let config = asap_types::PrecomputeMaterialization::new( - asap_types::AggregationType::HLL, - String::new(), - Default::default(), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), - 60, - 60, - asap_types::enums::WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, + fn fixture() -> ( + CatalogGeneration, + crate::precompute_engine::raw_dag::RawDagProgram, + ) { + use planner_types::post_asap::{ + GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, SummaryFamilyType, + SummaryUpdate, + }; + let config = crate::precompute_engine::raw_dag::RawDagProgram::fixture( + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Hll, SketchParams::Hll { precision: 14 }), + GroupingStrategy::PerSubpopulationInstance, + ), + SummaryUpdate::column(planner_types::pre_asap::ColumnRef::SampleValue), ); ( CatalogGeneration { diff --git a/data_plane/src/precompute_engine/frame_lineage.rs b/data_plane/src/precompute_engine/frame_lineage.rs index da79aad47..ba0ba21db 100644 --- a/data_plane/src/precompute_engine/frame_lineage.rs +++ b/data_plane/src/precompute_engine/frame_lineage.rs @@ -198,7 +198,7 @@ mod tests { identity_version: 1, plan_id: 7, plan_version: 3, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), materialization: asap_types::PolicyFingerprint(41).into(), series_identity: "service=checkout,zone=a".into(), schema_id: "schema-41".into(), diff --git a/data_plane/src/precompute_engine/ingest_handler.rs b/data_plane/src/precompute_engine/ingest_handler.rs index a60e9fbd2..feba9a10e 100644 --- a/data_plane/src/precompute_engine/ingest_handler.rs +++ b/data_plane/src/precompute_engine/ingest_handler.rs @@ -301,27 +301,26 @@ mod tests { use std::sync::Arc; use tokio::sync::mpsc; - fn make_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { - // `_agg_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Kept as a parameter to - // avoid churning the call sites below. - PrecomputeMaterialization::new( - AggregationType::CountMinSketch, - String::new(), - std::collections::HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + fn make_config( + agg_id: u64, + metric: &str, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.stored_output_id = asap_types::sds::StoredOutputId(agg_id); + let family = crate::tests::test_utilities::outputs::family( + AggregationType::CountMinSketch, + &serde_json::json!({}), + ); + (config, family) } /// Set up an `IngestState` with one Active agg for `metric` and a @@ -334,9 +333,7 @@ mod tests { let (tx, mut rx) = mpsc::channel(1024); let router = SeriesRouter::new(vec![tx]); - let mut map = std::collections::HashMap::new(); - map.insert(agg_id, make_config(agg_id, metric)); - let streaming = InstalledPrecomputePlan::from_raw_ids(map); + let streaming = InstalledPrecomputePlan::new([make_config(agg_id, metric)]); let hot_reload = crate::storage_engines::types::InstalledPrecomputePlanHandle::new(streaming.clone()); diff --git a/data_plane/src/precompute_engine/maintenance_runtime.rs b/data_plane/src/precompute_engine/maintenance_runtime.rs index fe42f2d2f..6eb59d33f 100644 --- a/data_plane/src/precompute_engine/maintenance_runtime.rs +++ b/data_plane/src/precompute_engine/maintenance_runtime.rs @@ -501,10 +501,18 @@ pub(crate) fn execute_completed_maintenance_cohort( target.fingerprint(), ); output.catalog_generation = Some(Arc::clone(generation)); + let planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { family, .. } = + &target_node.payload + else { + return Err("maintenance target node is not a Planner SummaryAgg".into()); + }; + // Derived outputs read stored states, so their input has no predicate. + let kind = crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""); store .publish_frozen_maintenance_output( target_sid, target_config, + &kind, &output, state.as_ref(), &cohort, @@ -784,7 +792,12 @@ fn execute_finite_complete_populations( batch.schema(), ) { output.population_labels = Some(Population::new()); - store.publish_native_summary_output(resolver, config, &output, batch, max_bytes)?; + let family = plan + .state_family(config.stored_output_id) + .ok_or("physical output has no state schema")?; + store.publish_native_summary_output( + resolver, config, family, &output, batch, max_bytes, + )?; continue; } let derived = config @@ -821,9 +834,13 @@ fn execute_finite_complete_populations( continue; } output.population_labels = Some(group); + let family = plan + .state_family(config.stored_output_id) + .ok_or("complete maintenance target has no state schema")?; store.publish_complete_raw_maintenance_output( target_sid, config, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""), &output, state.as_ref(), &cohort, @@ -1720,6 +1737,45 @@ mod tests { } } + /// Raw sources of these fixtures store exact sums. + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap() + } + + /// The maintained output of these fixtures stores a KLL. + fn kll_family() -> planner_types::post_asap::SummaryFamilyType { + crate::tests::test_utilities::outputs::family( + asap_types::AggregationType::DatasketchesKLL, + &serde_json::json!({"k": 200}), + ) + } + + fn sum_kind() -> crate::storage_engines::sketch_db::index::AggKind { + crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), "") + } + + /// The catalog of raw sum sources followed by the maintained KLL output. + fn fixture_catalog( + plan_id: u64, + plan_version: u64, + configs: &[asap_types::PrecomputeMaterialization], + ) -> asap_types::summary_catalog::SummaryCatalog { + let (sum, kll) = (sum_family(), kll_family()); + let (sources, target) = configs.split_at(configs.len() - 1); + asap_types::summary_catalog::SummaryCatalog::from_outputs( + plan_id, + plan_version, + sources + .iter() + .map(|config| (config, &sum, String::new())) + .chain(target.iter().map(|config| (config, &kll, String::new()))) + .collect(), + ) + .unwrap() + } + fn sum(value: f64) -> SummaryState { let mut accumulator = crate::tests::accumulator_fixture::sum_state(0.0); accumulator.update(None, value, 0); @@ -1754,17 +1810,14 @@ mod tests { source_config.window_type = asap_types::WindowKind::Tumbling; source_config.window_layout = asap_types::aggregation_config::WindowMaterializationLayout::Pane { pane_secs: 1 }; + source_config.allocate_stored_output_id(&sum_family()); let source_definition = source_config.policy_fingerprint().into(); let mut target_config = source_config.clone(); target_config.window_layout = asap_types::aggregation_config::WindowMaterializationLayout::FullWindow; - target_config.aggregation_type = asap_types::AggregationType::DatasketchesKLL; - target_config.aggregation_sub_type = "quantile".into(); - target_config - .parameters - .insert("k".into(), serde_json::json!(200)); + target_config.allocate_stored_output_id(&kll_family()); let target = target_config.policy_fingerprint().into(); - let target_family = target_config.accumulator_spec().unwrap().family; + let target_family = kll_family(); let configs = [source_config, target_config]; let binding = BackendExecutableBinding { nodes: BTreeMap::from([ @@ -1815,7 +1868,7 @@ mod tests { read.output_state = planner_types::post_asap::ExecutionDataState::INGESTION_ROWS; aggregate.output_schema.fields = vec![SummaryField { name: "state".into(), - dtype: configs[1].accumulator_spec().unwrap().family, + dtype: kll_family(), nullable: false, }]; let mut query = node(4); @@ -1868,6 +1921,7 @@ mod tests { ) .unwrap(), ); + durable_configs[1].allocate_stored_output_id(&kll_family()); document.schema_version = asap_types::executable_plan::PRECOMPUTE_DAG_SCHEMA_VERSION; let mut durable_binding = scheduled_binding.clone(); durable_binding.nodes.insert( @@ -1887,14 +1941,7 @@ mod tests { document, binding: durable_binding, }; - let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 1, - &durable_configs, - ) - .unwrap(), - ); + let catalog = Arc::new(fixture_catalog(1, 1, &durable_configs)); let directory = tempfile::tempdir().unwrap(); let persistence_config = || { let mut config = SketchStorePersistenceConfig::with_memory_limit( @@ -1941,6 +1988,7 @@ mod tests { writer.ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &output, sum(value).as_ref(), ) @@ -1977,6 +2025,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &rejected_output, sum(99.0).as_ref() ) @@ -2066,6 +2115,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &correction, sum(100.0).as_ref() ) @@ -2108,6 +2158,7 @@ mod tests { .ingest_precompute_with_series_id( 600, &durable_configs[0], + &sum_kind(), &correction, sum(100.0).as_ref() ) @@ -2130,14 +2181,7 @@ mod tests { }) .sum::(); assert_eq!(target_entries, 1); - let next_catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 2, - &durable_configs, - ) - .unwrap(), - ); + let next_catalog = Arc::new(fixture_catalog(1, 2, &durable_configs)); restored.install_summary_catalog(next_catalog).unwrap(); let next_generation = restored.active_catalog_generation().unwrap(); assert!(restored @@ -2154,6 +2198,7 @@ mod tests { restored.ingest_precompute_with_series_id( 602, &durable_configs[0], + &sum_kind(), &new_population, sum(20.0).as_ref() ), @@ -2188,8 +2233,10 @@ mod tests { first.population_key_encoding = asap_types::PopulationKeyEncoding::CanonicalLabelsV1; first.partitioning = Some(asap_types::sds::PopulationPartitioning::PerEntity); } + first.allocate_stored_output_id(&sum_family()); let mut second = first.clone(); second.metric = "second_maintenance_source".into(); + second.allocate_stored_output_id(&sum_family()); let first_id = first.policy_fingerprint().into(); let second_id = second.policy_fingerprint().into(); let mut dag = template.clone(); @@ -2277,10 +2324,7 @@ mod tests { binding, }; let configs = [first, second, target]; - let catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(2, 1, &configs) - .unwrap(), - ); + let catalog = Arc::new(fixture_catalog(2, 1, &configs)); let directory = tempfile::tempdir().unwrap(); let persistence_config = || { let mut config = @@ -2316,12 +2360,22 @@ mod tests { // is tested with the real compiler/process integration separately. config.population_key_encoding = asap_types::PopulationKeyEncoding::LegacyDelimited; - config + (config, sum_family()) }) .collect(), ) .unwrap(); plan.materializations = configs.to_vec(); + // The unvalidated fixture target still names its stored state schema. + plan.schemas + .push(asap_types::precompute_plan::StateSchemaContract { + materialization: configs[2].stored_output_id, + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + configs[2].stored_output_id, + ), + family: kll_family(), + ..plan.schemas[0].clone() + }); plan.summary_catalog = Some(generation.as_ref().clone()); plan.executable_dags = BTreeMap::from([("cohort-fixture".into(), installed.clone())]); let resolver_path = directory.path().join("resolver.wal"); @@ -2380,6 +2434,7 @@ mod tests { writer.ingest_precompute_with_series_id( sid, config, + &sum_kind(), &output, sum(value + population_index as f64 * 10.0).as_ref(), ) @@ -2569,10 +2624,7 @@ mod tests { ) .unwrap()); assert_eq!(persistence.manifest.live_parts().len(), parts); - let next_catalog = Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations(2, 2, &configs) - .unwrap(), - ); + let next_catalog = Arc::new(fixture_catalog(2, 2, &configs)); restored.install_summary_catalog(next_catalog).unwrap(); assert_ne!(generation, restored.active_catalog_generation().unwrap()); // Stale scheduling fails at the captured catalog boundary, before any @@ -2790,7 +2842,7 @@ mod tests { summary_catalog: Some(Arc::new(bundle.summary_catalog)), precompute_plan: bundle.precompute_plan, transmission_plan: bundle.transmission_plan, - installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Default::default())), + installed_precompute_plan: Arc::new(InstalledPrecomputePlan::new(Vec::new())), query_plan: Arc::new(bundle.query_plan), storage_routing: Arc::new(Default::default()), }; @@ -3018,7 +3070,11 @@ pub(crate) fn execute_revision_outputs( for (group, state) in super::native_precompute::population_states(&batch, window.1)? { - let payload = encode_state(state, config.accumulator_spec()?.family)?; + let family = plan + .precompute_plan + .state_family(config.stored_output_id) + .ok_or("physical output has no state schema")?; + let payload = encode_state(state, family.clone())?; if payload.len() > limit { return Err(asap_physical_operators::Error::MemoryLimit.into()); } diff --git a/data_plane/src/precompute_engine/output_sink.rs b/data_plane/src/precompute_engine/output_sink.rs index 2a294ed9f..756e0cc6e 100644 --- a/data_plane/src/precompute_engine/output_sink.rs +++ b/data_plane/src/precompute_engine/output_sink.rs @@ -106,10 +106,8 @@ impl SketchStoreSink { /// Missing configuration or incompatible state must surface as failure: /// a finite-input completion barrier cannot acknowledge dropped outputs. /// - /// PR-6 follow-up: resolves the source `PrecomputeMaterialization` via - /// `PolicyRegistry::get(output.policy_fp)`. The legacy - /// `aggregation_id` fallback branch (PR 4) is gone — `policy_fp` - /// is the only identity handle on `PrecomputedOutput`. Outputs + /// Resolves the stored output from the installed plan by + /// `output.policy_fp`, the only identity handle on `PrecomputedOutput`. Outputs /// emitted with the `PolicyFingerprint::UNSET` sentinel (e.g. /// raw-mode fast-path that has no source config) are skipped /// rather than routed by a parallel id. @@ -122,9 +120,11 @@ impl SketchStoreSink { return false; } let cfg = self.hot_reload.snapshot(); - let registry = cfg.policy_registry(); - let agg_cfg = match registry.get(output.policy_fp) { - Some(c) => c.clone(), + let installed = cfg + .get_aggregation_config(output.policy_fp.into()) + .zip(cfg.agg_kind(output.policy_fp.into())); + let (agg_cfg, kind) = match installed { + Some(installed) => installed, None => { // CQ-6 — policy-miss drop: a content-addressed policy_fp // that the running streaming-config registry doesn't know @@ -138,7 +138,6 @@ impl SketchStoreSink { return false; } }; - let agg_cfg = &agg_cfg; let Some(reference) = cfg.stored_output_reference(output.policy_fp.into()) else { return false; }; @@ -175,6 +174,7 @@ impl SketchStoreSink { &address, &self.series_resolver, agg_cfg, + &kind, output, accumulator, ) @@ -372,50 +372,33 @@ mod tests { ); } - fn sum_agg_config(_id: u64, metric: &str, grouping_keys: &[&str]) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Callers obtain the id - // via `config.policy_fp_u64()`. - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new( - grouping_keys.iter().map(|s| s.to_string()).collect(), - ) - .into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + fn sum_agg_config( + _id: u64, + metric: &str, + grouping_keys: &[&str], + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + metric, + KeyByLabelNames::new(grouping_keys.iter().map(|s| s.to_string()).collect()), + 1, + 1, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + crate::tests::test_utilities::outputs::allocated( + config, + AggregationType::Sum.planner_exact_family().unwrap(), + ) } #[test] fn sketch_index_sink_writes_to_index() { let cfg = sum_agg_config(7, "cpu_seconds", &["zone"]); - let agg_id = cfg.policy_fp_u64(); - let mut configs = HashMap::new(); - configs.insert(agg_id, cfg); - let streaming = InstalledPrecomputePlan::from_raw_ids(configs); + let agg_id = cfg.0.policy_fp_u64(); + let streaming = InstalledPrecomputePlan::new([cfg]); let hot_reload = InstalledPrecomputePlanHandle::new(streaming.clone()); let summary_store = Arc::new(SketchStore::new()); @@ -469,11 +452,11 @@ mod tests { #[test] fn sink_reactivates_catalog_series_without_reusing_retired_payload() { let cfg = sum_agg_config(7, "cpu_seconds", &[]); - let fingerprint = cfg.policy_fingerprint(); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let fingerprint = cfg.0.policy_fingerprint(); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[cfg.clone()], + vec![(&cfg.0, &cfg.1, String::new())], ) .unwrap(); let store = Arc::new(SketchStore::new()); @@ -485,9 +468,7 @@ mod tests { Arc::new(SeriesIdResolver::open(temporary.path().join("resolver.wal")).unwrap()); let sink = SketchStoreSink::new( store.clone(), - InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::from_raw_ids( - HashMap::from([(fingerprint.0, cfg)]), - )), + InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new([cfg])), resolver, ); let original_generation = Arc::new(catalog.reference().unwrap()); @@ -572,14 +553,16 @@ mod tests { #[test] fn sketch_policy_is_registered_and_stored_as_sketch_state() { - let mut cfg = sum_agg_config(8, "latency", &[]); - cfg.aggregation_type = AggregationType::DDSketch; - cfg.parameters - .insert("alpha".into(), serde_json::json!(0.01)); - let policy_fp = cfg.policy_fp_u64(); - let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::from_raw_ids( - HashMap::from([(policy_fp, cfg)]), - )); + let (cfg, _) = sum_agg_config(8, "latency", &[]); + let cfg = crate::tests::test_utilities::outputs::allocated( + cfg, + crate::tests::test_utilities::outputs::family( + AggregationType::DDSketch, + &serde_json::json!({"alpha": 0.01}), + ), + ); + let policy_fp = cfg.0.policy_fp_u64(); + let hot_reload = InstalledPrecomputePlanHandle::new(InstalledPrecomputePlan::new([cfg])); let summary_store = Arc::new(SketchStore::new()); let sink = SketchStoreSink::new( summary_store.clone(), diff --git a/data_plane/src/precompute_engine/raw_dag.rs b/data_plane/src/precompute_engine/raw_dag.rs index fb46b81b2..15d9999e4 100644 --- a/data_plane/src/precompute_engine/raw_dag.rs +++ b/data_plane/src/precompute_engine/raw_dag.rs @@ -89,18 +89,16 @@ impl RawDagProgram { source: Source::TimeSeries { metric }, .. } if metric == &config.metric => { - let (metric, window, filter) = - control_plane::physical::compiler::raw_time_series_input_contract( + let (metric, window, _) = + asap_types::precompute_plan::raw_time_series_input_contract( expression, matches!(family, SummaryFamilyType::ExactAggregate(..)), )?; if metric != config.metric || window.is_some_and(|seconds| seconds != config.window_size) - || asap_types::utils::normalize_spatial_filter(&filter) - != config.spatial_filter_normalized { return Err( - "raw DAG source filter/window differs from physical binding".into(), + "raw DAG source window differs from physical binding".into() ); } } @@ -151,35 +149,6 @@ impl RawDagProgram { return Err("raw exact update differs from stored source projection".into()); } } - if &config.accumulator_spec().map_err(|e| e.to_string())?.family != family { - return Err( - "materialization storage family differs from selected Planner node".into(), - ); - } - // The stored descriptor must name the same update semantics; its - // content identity cannot be reused for an unrelated DAG program. - let update_matches = match (&input.weight, config.sample_update_rule()) { - ( - SummaryInputExpr::Column(_), - asap_types::SampleUpdateRule::Value { scale }, - ) => scale == 1.0, - (SummaryInputExpr::Constant(value), asap_types::SampleUpdateRule::Count) => { - *value == 1.0 - } - _ => { - asap_types::accumulator_spec::is_unit_sample_frequency(input) - || (matches!( - family, - SummaryFamilyType::ExactAggregate( - planner_types::post_asap::ExactKind::Count, - _ - ) - ) && input.weight == SummaryInputExpr::Constant(1.0)) - } - }; - if !update_matches { - return Err("DAG update differs from stored summary identity".into()); - } let compiled = installed .native_program(node.id)? .ok_or("raw materialization lacks its Planner precompute graph")?; @@ -220,6 +189,21 @@ impl RawDagProgram { selected.ok_or_else(|| "raw materialization has no selected post-ASAP DAG producer".into()) } + /// A program with the given node semantics and no executable graph, for + /// tests of consumers that only inspect the node. + #[cfg(test)] + pub(crate) fn fixture(family: SummaryFamilyType, input: SummaryUpdate) -> Self { + Self { + node: PostAsapNodeId(1), + family, + input, + grouping: GroupingStrategy::PerSubpopulationInstance, + reduction: planner_types::pre_asap::Reduction::PerEntity, + program: std::sync::Arc::from(Vec::new()), + source: 0, + } + } + /// Execute the Planner graph over one pane's samples as one typed batch. /// Returns `None` when the graph admits no population from them. pub fn build<'a>( diff --git a/data_plane/src/precompute_engine/revisions.rs b/data_plane/src/precompute_engine/revisions.rs index 9aecb674a..14ef43c4a 100644 --- a/data_plane/src/precompute_engine/revisions.rs +++ b/data_plane/src/precompute_engine/revisions.rs @@ -458,10 +458,13 @@ pub(crate) fn encode_state( pub(crate) fn decode_state( record: &RevisionRecord, - config: &asap_types::PrecomputeMaterialization, + family: &SummaryFamilyType, ) -> Result, RevisionError> { - let family = config.accumulator_spec()?.family; - let batch = native::decode_batch(&record.payload, state_schema(family), record.payload.len())?; + let batch = native::decode_batch( + &record.payload, + state_schema(family.clone()), + record.payload.len(), + )?; match batch.rows() { [row] => match row.as_slice() { [Value::Summary { state, .. }] => { @@ -632,7 +635,12 @@ impl RevisionRuntime { if decode_native_record(&plan, record, self.policy.max_checkpoint_bytes)? .is_none() { - decode_state(record, config)?; + decode_state( + record, + plan.precompute_plan + .state_family(config.stored_output_id) + .ok_or("recovered revision has no state schema")?, + )?; } } } diff --git a/data_plane/src/precompute_engine/worker.rs b/data_plane/src/precompute_engine/worker.rs index d1b1591bb..a367da198 100644 --- a/data_plane/src/precompute_engine/worker.rs +++ b/data_plane/src/precompute_engine/worker.rs @@ -14,7 +14,6 @@ use asap_summary_state::codec::KeyCodec; use asap_summary_state::StoredState; use asap_types::aggregation_config::PrecomputeMaterialization; use asap_types::PolicyFingerprint; -use asap_types::SampleUpdateRule; use std::collections::{BTreeMap, HashMap}; // (PolicyFingerprint is used for both `PolicyFingerprint::from_config(...)` // on the emit path and the `policy_fp` field of GroupState below.) @@ -39,6 +38,9 @@ use tracing::{debug, debug_span, info, warn}; /// `GROUP BY window, key`. struct GroupState { program: Option>, + /// Stored state family for isolated kernel tests without a Planner program. + #[cfg(test)] + fixture_family: Option, series_id: u64, stored_output_reference: Option, catalog_generation: Option>, @@ -60,10 +62,6 @@ struct GroupState { /// Samples admitted to each open pane, keyed by pane_start_ms, in arrival /// order. The installed Planner graph builds the pane's state at close. active_panes: BTreeMap, i64, f64)>>, - /// Last cumulative counter sample per source series for heap membership - /// materializations. This is bounded O(series) derivative state, not a - /// raw-sample history, and deliberately survives pane rotation. - counter_previous: HashMap, /// Active panes for pre-built accumulator inputs (e.g. OTLP-delivered /// sketches), keyed by pane_start_ms. Each entry is the running merge /// of every accumulator that landed in that pane's time range. Kept @@ -90,6 +88,16 @@ struct PaneWallClock { } impl GroupState { + #[cfg(test)] + fn fixture_family(&self) -> Option<&planner_types::post_asap::SummaryFamilyType> { + self.fixture_family.as_ref() + } + + #[cfg(not(test))] + fn fixture_family(&self) -> Option<&planner_types::post_asap::SummaryFamilyType> { + None + } + fn stores_full_windows(&self) -> bool { matches!( self.config.window_layout, @@ -522,6 +530,8 @@ impl Worker { let config = Arc::new(cfg.clone()); let gs = GroupState { program, + #[cfg(test)] + fixture_family: snap.state_family(policy_fp.into()).cloned(), series_id: sid, stored_output_reference: snap.stored_output_reference(policy_fp.into()), catalog_generation: self.current_catalog_generation.clone(), @@ -536,7 +546,6 @@ impl Worker { policy_fp, group_key: Arc::clone(group_key), active_panes: BTreeMap::new(), - counter_previous: HashMap::new(), sketch_panes: BTreeMap::new(), max_event_time_ms: i64::MIN, closure_watermark_ms: i64::MIN, @@ -588,12 +597,7 @@ impl Worker { // Keep original timestamps inside accumulators (notably rate/increase), // shifting only pane membership and closure watermark for PromQL (a,b]. - let right_closed = state - .config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = state.program.is_some(); let pane_timestamp = |ts: i64| { if right_closed { ts.saturating_sub(1) @@ -635,11 +639,7 @@ impl Worker { let too_late = previous_event_time != i64::MIN && pane_timestamp(*ts) < watermark_for_event_time(previous_event_time, allowed_lateness_ms); - let value = if legacy_counter_delta(state) { - reset_aware_counter_delta(&mut state.counter_previous, series_key, *val, *ts) - } else { - Some(*val) - }; + let value = Some(*val); for bucket_start in state.bucket_starts_for(pane_timestamp(*ts)) { if let Some(revision) = &input_revision { state @@ -686,25 +686,18 @@ impl Worker { // Never feed the raw counter value into a membership // heap; the authoritative ExactCounter branch remains // responsible for the visible result. - if legacy_counter_delta(state) { - if let Some(input) = state.input_revisions.get_mut(&bucket_start) { - Arc::make_mut(input).first_revision = 0; - } - record_late_input("drop", "counter_delta_membership"); - continue; - } record_late_input("append_correction", "raw_sample"); let Some(correction) = build_pane( state.program.as_deref(), - &state.config, + state.fixture_family(), &[(Arc::from(series_key.as_str()), *ts, *val)], (bucket_start, bucket_end), )? else { continue; }; - if let (Some(observer), Some(revision)) = - (&self.erp_observer, &input_revision) + if let (Some(observer), Some(revision), Some(program)) = + (&self.erp_observer, &input_revision, &state.program) { observer.observe( &revision.generation, @@ -716,7 +709,7 @@ impl Worker { }, group_values: group_key.as_population_labels(), }, - &state.config, + program, *ts, *val, ); @@ -751,7 +744,8 @@ impl Worker { let pane = state.active_panes.entry(bucket_start).or_default(); if let Some(value) = value { pane.push((Arc::clone(&series), *ts, value)); - if let (Some(observer), Some(revision)) = (&self.erp_observer, &input_revision) + if let (Some(observer), Some(revision), Some(program)) = + (&self.erp_observer, &input_revision, &state.program) { observer.observe( &revision.generation, @@ -763,7 +757,7 @@ impl Worker { }, group_values: group_key.as_population_labels(), }, - &state.config, + program, *ts, value, ); @@ -1594,7 +1588,7 @@ pub fn decode_label_value(s: &str) -> std::borrow::Cow<'_, str> { /// Build a pane's state from its samples with the installed Planner graph. fn build_pane( program: Option<&super::raw_dag::RawDagProgram>, - config: &PrecomputeMaterialization, + fixture_family: Option<&planner_types::post_asap::SummaryFamilyType>, samples: &[(Arc, i64, f64)], pane: (i64, i64), ) -> Result>, String> { @@ -1612,15 +1606,21 @@ fn build_pane( #[cfg(test)] { let _ = pane; - let mut updater = create_fixture_accumulator(config); + let family = fixture_family.ok_or("fixture output has no state family")?; + let mut updater = create_fixture_accumulator( + family, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + ); for (series, time, value) in samples { - apply_sample(&mut *updater, series, *value, *time, config); + apply_sample(&mut *updater, series, *value, *time, family); } Ok(Some(updater.take_accumulator())) } #[cfg(not(test))] { - let _ = (config, samples, pane); + let _ = (fixture_family, samples, pane); Err("missing installed Planner producer".into()) } } @@ -1635,7 +1635,7 @@ fn close_pane( }; let built = build_pane( state.program.as_deref(), - &state.config, + state.fixture_family(), samples, state.bucket_bounds(start), )?; @@ -1643,34 +1643,24 @@ fn close_pane( Ok(built) } -/// Route a single sample to `updater`, dispatching keyed vs. non-keyed based on config. -/// -/// For keyed accumulators (MultipleSum, CMS, HydraKLL), the key is extracted -/// from the series' **aggregated_labels** — these are the labels that become -/// the key dimension *inside* the sketch (e.g., which bucket in a CMS, which -/// entry in a KeyedSumCountAccumulator's HashMap). This matches the Arroyo SQL -/// pattern: `udf(concat_ws(';', aggregated_labels), value)`. +/// Route a single fixture sample to `updater`. Planner's PromQL Top-K item +/// is the series identity; other keyed fixtures project no item labels. #[cfg(test)] pub(crate) fn apply_sample( updater: &mut dyn AccumulatorUpdater, series_key: &str, val: f64, ts: i64, - config: &PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, ) { + use planner_types::post_asap::{SketchAlgorithm, SummaryFamilyType}; if updater.is_keyed() { - // Planner's PromQL Top-K item is the series identity. When no - // explicit aggregated labels are projected, retain the canonical - // series key instead of collapsing every series onto an empty item. - let key = if config.aggregated_labels.labels.is_empty() - && matches!( - config.aggregation_type, - crate::storage_engines::types::AggregationType::CountMinSketchWithHeap - | crate::storage_engines::types::AggregationType::CountSketchWithHeap - ) { + let key = if matches!(family, SummaryFamilyType::Sketch(kind, _) + if matches!(kind.algorithm(), SketchAlgorithm::CmsWithHeap | SketchAlgorithm::CountSketchWithHeap)) + { KeyByLabelValues::new_with_labels(vec![series_key.to_string()]) } else { - extract_aggregated_key_from_series(series_key, config) + KeyByLabelValues::new_with_labels(Vec::new()) }; updater.update_keyed(&key, val, ts); } else { @@ -1678,71 +1668,6 @@ pub(crate) fn apply_sample( } } -/// Convert a cumulative counter sample into a non-negative, reset-aware -/// increment. Only the immediately preceding sample per series is retained; -/// pane rotation therefore cannot lose the boundary increment. -/// Whether a sample must be converted to a counter delta before it reaches the -/// accumulator. -/// -/// Only the legacy configured update rule does this. A selected Planner program -/// never does: rate is an explicit upstream operator in the DAG, so the sample -/// reaches the accumulator unchanged. Both call sites used to ask the program -/// and were always told `false`. -fn legacy_counter_delta(state: &GroupState) -> bool { - state.program.is_none() - && matches!( - state.config.sample_update_rule(), - SampleUpdateRule::CounterDelta { .. } - ) -} - -pub(crate) fn reset_aware_counter_delta( - previous: &mut HashMap, - series_key: &str, - value: f64, - timestamp_ms: i64, -) -> Option { - if !value.is_finite() { - return None; - } - match previous.get(series_key).copied() { - Some((previous_ts, _)) if timestamp_ms <= previous_ts => None, - Some((_, previous_value)) => { - previous.insert(series_key.to_owned(), (timestamp_ms, value)); - Some(if value >= previous_value { - value - previous_value - } else { - value.max(0.0) - }) - } - None => { - previous.insert(series_key.to_owned(), (timestamp_ms, value)); - None - } - } -} - -/// Extract aggregated label values from a series key string. -/// These are the labels that form the key dimension *inside* keyed accumulators -/// (MultipleSum, CMS, HydraKLL), matching Arroyo's `agg_columns`. -fn extract_aggregated_key_from_series( - series_key: &str, - config: &PrecomputeMaterialization, -) -> KeyByLabelValues { - let labels = parse_labels_from_series_key(series_key); - let mut values = Vec::new(); - - for label_name in &config.aggregated_labels.labels { - if let Some(val) = labels.get(label_name.as_str()) { - values.push(val.to_string()); - } else { - values.push(String::new()); - } - } - - KeyByLabelValues::new_with_labels(values) -} - #[cfg(test)] mod tests { use super::*; @@ -1795,30 +1720,6 @@ mod tests { crate::precompute_engine::group_key::intern_pairs([("group", value)]) } - #[test] - fn counter_delta_is_reset_aware_series_local_and_cross_pane_safe() { - let mut previous = HashMap::new(); - assert_eq!(reset_aware_counter_delta(&mut previous, "a", 10.0, 1), None); - assert_eq!(reset_aware_counter_delta(&mut previous, "b", 40.0, 1), None); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 15.0, 2), - Some(5.0) - ); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 3.0, 3), - Some(3.0) - ); - assert_eq!( - reset_aware_counter_delta(&mut previous, "b", 44.0, 4), - Some(4.0) - ); - assert_eq!(reset_aware_counter_delta(&mut previous, "a", 99.0, 2), None); - assert_eq!( - reset_aware_counter_delta(&mut previous, "a", 5.0, 5), - Some(2.0) - ); - } - use flate2::{write::GzEncoder, Compression}; use serde_json::json; use std::io::Write; @@ -1905,64 +1806,52 @@ mod tests { use asap_types::sds::StoredOutputId; use asap_types::AggregationType; - fn make_agg_config( - id: u64, - metric: &str, - agg_type: AggregationType, - agg_sub_type: &str, - window_secs: u64, - slide_secs: u64, - grouping: Vec<&str>, - ) -> PrecomputeMaterialization { - make_agg_config_full( - id, - metric, - agg_type, - agg_sub_type, - window_secs, - slide_secs, - grouping, - vec![], - ) + thread_local! { + /// Stored state family of each fixture output, keyed by its allocated id. + static FIXTURE_FAMILIES: std::cell::RefCell< + HashMap, + > = Default::default(); } - #[allow(clippy::too_many_arguments)] - fn make_agg_config_full( + /// Give `config` the stored state `family`, allocating its id from it. + fn set_fixture_family( + config: &mut PrecomputeMaterialization, + family: planner_types::post_asap::SummaryFamilyType, + ) { + config.allocate_stored_output_id(&family); + FIXTURE_FAMILIES.with(|families| { + families + .borrow_mut() + .insert(config.stored_output_id, family) + }); + } + + fn make_agg_config( _id: u64, metric: &str, agg_type: AggregationType, - agg_sub_type: &str, + _agg_sub_type: &str, window_secs: u64, slide_secs: u64, grouping: Vec<&str>, - aggregated: Vec<&str>, ) -> PrecomputeMaterialization { - // `_id` is unused after PR 5 — identity is content-addressed - // via `PolicyFingerprint::from_config`. Callers below build the - // streaming-config map by reading `config.policy_fp_u64()` - // from the returned value. let window_type = if slide_secs == 0 || slide_secs == window_secs { WindowKind::Tumbling } else { WindowKind::Sliding }; - PrecomputeMaterialization::new( - agg_type, - agg_sub_type.to_string(), - HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(grouping.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(aggregated.iter().map(|s| s.to_string()).collect()), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), window_secs, slide_secs, window_type, - metric.to_string(), - metric.to_string(), - None, - None, - None, - ) + ); + set_fixture_family( + &mut config, + crate::tests::test_utilities::outputs::family(agg_type, &serde_json::json!({})), + ); + config } fn make_worker( @@ -2012,8 +1901,14 @@ mod tests { fn make_hot_reload( configs: HashMap, ) -> crate::storage_engines::types::InstalledPrecomputePlanHandle { + let outputs = configs.into_iter().map(|(id, mut config)| { + let family = FIXTURE_FAMILIES + .with(|families| families.borrow()[&config.stored_output_id].clone()); + config.stored_output_id = asap_types::sds::StoredOutputId(id); + (config, family) + }); crate::storage_engines::types::InstalledPrecomputePlanHandle::new( - crate::storage_engines::types::InstalledPrecomputePlan::from_raw_ids(configs), + crate::storage_engines::types::InstalledPrecomputePlan::new(outputs), ) } @@ -2029,57 +1924,6 @@ mod tests { // Test: raw mode — each sample forwarded as an exact Sum with sum==value // ----------------------------------------------------------------------- - #[test] - fn initial_counter_observation_publishes_an_empty_delta_window() { - let mut config = make_agg_config(1, "counter", AggregationType::Sum, "sum", 1, 1, vec![]); - config - .parameters - .insert("weight_mode".into(), serde_json::json!("counter_delta")); - let fingerprint = config.policy_fingerprint(); - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker( - HashMap::from([(fingerprint.0, config)]), - sink.clone(), - false, - 0, - LateDataPolicy::Drop, - ); - worker.current_input_revision = Some(Arc::new( - crate::storage_engines::types::SummaryInputRevision { - generation: Arc::new(asap_types::sds::CatalogGeneration { - schema_version: 1, - plan_id: 1, - plan_version: 1, - snapshot_sha256: "test".into(), - }), - first_revision: 1, - revision: 1, - }, - )); - worker - .process_group_samples( - 1, - fingerprint, - &test_group_key(""), - vec![("counter".into(), 1000, 7.0)], - ) - .unwrap(); - worker.force_close_all().unwrap(); - let captured = sink.drain(); - assert_eq!(captured.len(), 1); - assert_eq!(captured[0].0.input_revision.as_ref().unwrap().revision, 1); - assert_eq!( - captured[0] - .1 - .as_any() - .downcast_ref::() - .map(crate::tests::accumulator_fixture::sum_view) - .unwrap() - .sum, - 0.0 - ); - } - #[test] fn test_raw_mode_forwarding() { let sink = Arc::new(CapturingOutputSink::new()); @@ -2120,15 +1964,7 @@ mod tests { #[test] fn test_tumbling_window_correctness() { // 10s tumbling window - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2203,17 +2039,9 @@ mod tests { #[test] fn test_group_by_merges_series() { - // SingleSubpopulation Sum with no grouping labels + // Sum with no grouping labels // Two different series in the same group → both feed same accumulator - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2275,15 +2103,7 @@ mod tests { #[test] fn test_different_groups_separate_outputs() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec!["pattern"], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec!["pattern"]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2364,9 +2184,13 @@ mod tests { 0, vec!["pattern"], ); - config - .parameters - .insert("K".to_string(), serde_json::Value::from(20_u64)); + set_fixture_family( + &mut config, + crate::tests::test_utilities::outputs::family( + AggregationType::DatasketchesKLL, + &serde_json::json!({"K": 20}), + ), + ); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); @@ -2439,15 +2263,7 @@ mod tests { #[test] fn test_sliding_window_pane_sharing() { // 30s window, 10s slide → W=3 panes per window - let config = make_agg_config( - 2, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 30, - 10, - vec![], - ); + let config = make_agg_config(2, "cpu", AggregationType::Sum, "", 30, 10, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(2, config); @@ -2499,15 +2315,7 @@ mod tests { #[test] fn full_window_layout_materializes_each_overlapping_slide() { - let mut config = make_agg_config( - 2, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 30, - 10, - vec![], - ); + let mut config = make_agg_config(2, "cpu", AggregationType::Sum, "", 30, 10, vec![]); config.window_layout = asap_types::WindowMaterializationLayout::FullWindow; let policy = config.policy_fingerprint(); let sink = Arc::new(CapturingOutputSink::new()); @@ -2564,91 +2372,9 @@ mod tests { // All series go to one group, host is the key dimension INSIDE the sketch // ----------------------------------------------------------------------- - #[test] - fn test_keyed_accumulator_aggregated_labels() { - // Like planner output for `sum by (host) (cpu)`: - // grouping=[] (empty), aggregated=[host] (key inside KeyedSumCountAccumulator) - let config = make_agg_config_full( - 3, - "cpu", - AggregationType::MultipleSubpopulation, - "Sum", - 10, - 0, - vec![], // grouping: empty — one output group - vec!["host"], // aggregated: host is the key INSIDE the sketch - ); - let mut agg_configs = HashMap::new(); - agg_configs.insert(3, config); - - let sink = Arc::new(CapturingOutputSink::new()); - let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); - - // Both series go to the SAME bucket (group_key="" since grouping is empty). - // The host label is extracted as the aggregated key inside the accumulator. - let pf = PolicyFingerprint(3); - worker - .process_group_samples( - 3, - pf, - &test_group_key(""), - vec![ - ("cpu{host=\"A\"}".to_string(), 1000, 10.0), - ("cpu{host=\"B\"}".to_string(), 2000, 20.0), - ], - ) - .unwrap(); - - // Close the single bucket's window - worker - .process_group_samples( - 3, - pf, - &test_group_key(""), - group_samples("cpu{host=\"A\"}", vec![(10000, 0.0)]), - ) - .unwrap(); - - let captured = sink.drain(); - assert_eq!( - captured.len(), - 1, - "one group → one output (both hosts inside)" - ); - - let (_output, acc) = &captured[0]; - let ms_acc = acc - .as_any() - .downcast_ref::() - .expect("should be a keyed exact Sum"); - assert!(ms_acc.is_keyed(), "both host keys inside one accumulator"); - let sum = |host: &str| { - ms_acc - .readout( - asap_types::Statistic::Sum, - None, - Some(&KeyByLabelValues::new_with_labels(vec![host.into()])), - ) - .unwrap() - .unwrap() - }; - let found_a = (sum("A") - 10.0).abs() < 1e-10; - let found_b = (sum("B") - 20.0).abs() < 1e-10; - assert!(found_a, "expected key A inside accumulator"); - assert!(found_b, "expected key B inside accumulator"); - } - #[test] fn test_late_data_drop() { - let config = make_agg_config( - 4, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(4, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(4, config); @@ -2704,15 +2430,7 @@ mod tests { #[test] fn test_late_data_forward_to_store() { - let config = make_agg_config( - 5, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(5, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(5, config); @@ -2804,21 +2522,11 @@ mod tests { #[test] fn test_extract_key_from_series() { let config = PrecomputeMaterialization::new( - AggregationType::SingleSubpopulation, - "Sum".to_string(), - HashMap::new(), + "http_requests_total", asap_types::KeyByLabelNames::new(vec!["method".to_string(), "status".to_string()]), - asap_types::KeyByLabelNames::new(vec![]), - asap_types::KeyByLabelNames::new(vec![]), - String::new(), 60, 0, WindowKind::Tumbling, - "http_requests_total".to_string(), - "http_requests_total".to_string(), - Some(60), - None, - None, ); let key = extract_key_from_series( @@ -2881,15 +2589,7 @@ mod tests { // Two groups on the same worker. Group A advances to t=100s while // group B remains active at t=5s. Group A is not evidence that group // B's source has completed its earlier window. - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let agg_configs = HashMap::from([(1, config)]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker(agg_configs, sink.clone(), false, 0, LateDataPolicy::Drop); @@ -2980,15 +2680,7 @@ mod tests { #[test] fn repeated_flushes_do_not_make_fixed_timestamp_input_late() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3033,15 +2725,7 @@ mod tests { #[test] fn allowed_lateness_delays_event_time_window_close() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3091,15 +2775,7 @@ mod tests { #[test] fn first_catch_up_batch_closes_every_complete_window() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 5, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 5, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_lateness( HashMap::from([(1, config)]), @@ -3138,15 +2814,7 @@ mod tests { #[test] fn test_flush_publishes_worker_watermark() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let agg_configs = HashMap::from([(1, config)]); let sink = Arc::new(CapturingOutputSink::new()); let wm = Arc::new(AtomicI64::new(i64::MIN)); @@ -3590,15 +3258,7 @@ mod tests { #[test] fn wall_clock_fallback_does_not_close_active_raw_ingest() { - let cfg = make_agg_config( - 7, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(7, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(7, cfg)]), @@ -3659,15 +3319,7 @@ mod tests { #[test] fn absolute_wall_clock_deadline_closes_active_raw_ingest() { - let cfg = make_agg_config( - 9, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(9, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(9, cfg)]), @@ -3734,15 +3386,7 @@ mod tests { #[test] fn absolute_deadline_is_disabled_for_drop_policy() { - let cfg = make_agg_config( - 11, - "netflow_bytes", - AggregationType::SingleSubpopulation, - "Sum", - 1, - 0, - vec![], - ); + let cfg = make_agg_config(11, "netflow_bytes", AggregationType::Sum, "", 1, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker_with_wall_clock_policy( HashMap::from([(11, cfg)]), @@ -3969,15 +3613,7 @@ mod tests { // A pooled Sum is correct only for an explicit cross-entity reduction. #[test] fn pooled_sum_does_not_preserve_per_entity_output_rows() { - let config = make_agg_config( - 1, - "gauge", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "gauge", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker( HashMap::from([(1, config)]), @@ -4026,8 +3662,8 @@ mod tests { let config = make_agg_config( 1, "requests_total", - AggregationType::SingleSubpopulation, - "Increase", + AggregationType::Increase, + "", 10, 0, vec![], @@ -4072,15 +3708,7 @@ mod tests { // Acknowledgement proves FIFO input processing and trailing-window publication. #[tokio::test] async fn finite_input_drain_publishes_before_acknowledging() { - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let sink = Arc::new(CapturingOutputSink::new()); let mut worker = make_worker( HashMap::from([(1, config)]), @@ -4137,15 +3765,7 @@ mod tests { Err("deliberate sink failure".into()) } } - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut worker = make_worker( HashMap::from([(1, config)]), Arc::new(CapturingOutputSink::new()), @@ -4207,15 +3827,7 @@ mod tests { Err(Box::new(asap_physical_operators::dag::Error::MemoryLimit)) } } - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut worker = make_worker( HashMap::from([(1, config)]), Arc::new(CapturingOutputSink::new()), @@ -4252,15 +3864,7 @@ mod tests { fn shutdown_force_close_emits_trailing_sample_window() { // 10s tumbling window; make_worker uses grace=0, isolating the // force-close from the wall-clock fallback. - let config = make_agg_config( - 1, - "cpu", - AggregationType::SingleSubpopulation, - "Sum", - 10, - 0, - vec![], - ); + let config = make_agg_config(1, "cpu", AggregationType::Sum, "", 10, 0, vec![]); let mut agg_configs = HashMap::new(); agg_configs.insert(1, config); let sink = Arc::new(CapturingOutputSink::new()); @@ -4547,6 +4151,12 @@ mod dag_execution_tests { .expect("ASAP producer required") .clone(); let fp = config.policy_fingerprint(); + let agg_type = asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + ) + .unwrap(); let streaming = InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan).unwrap(); let sink = Arc::new(CapturingOutputSink::new()); @@ -4615,7 +4225,7 @@ mod dag_execution_tests { let group = crate::query_engines::asap_query_engine::summary_executor::GroupState::ExactAgg { entries: vec![std::rc::Rc::new(states)], - agg_type: config.aggregation_type, + agg_type, }; assert_eq!( group.exact_value_for(readout, &None, 0, 5000), @@ -4715,11 +4325,15 @@ mod dag_execution_tests { == Some(asap_types::sds::PopulationPartitioning::PerEntity) || (config.partitioning.is_none() && matches!( - config.aggregation_type, - asap_types::AggregationType::Increase - | asap_types::AggregationType::Rate - | asap_types::AggregationType::Min - | asap_types::AggregationType::Max + installed + .state_family(config.stored_output_id) + .and_then(asap_types::aggregation_type_for_family), + Some( + asap_types::AggregationType::Increase + | asap_types::AggregationType::Rate + | asap_types::AggregationType::Min + | asap_types::AggregationType::Max + ) )); // Routing identity as Remote Write assigns it. let mut groups = BTreeMap::, Vec<(String, i64, f64)>>::new(); @@ -4776,11 +4390,7 @@ mod dag_execution_tests { config.pane_origin_ms, &config.window_layout, ); - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + let right_closed = installed.right_closed_panes(config.stored_output_id); let mut expected = BTreeMap::new(); let mut backfill = BTreeMap::new(); for (key, samples) in &groups { @@ -4862,7 +4472,8 @@ mod dag_execution_tests { "{}", config.metric ); - assert_eq!(actual, expected, "{:?}", config.aggregation_type); + let family = installed.state_family(config.stored_output_id).unwrap(); + assert_eq!(actual, expected, "{family:?}"); // Backfill builds each window with the same installed graph. let backfilled = backfill .into_iter() @@ -4877,14 +4488,10 @@ mod dag_execution_tests { ((key, start, end), state.serialize_to_bytes()) }) .collect::>(); - assert_eq!( - backfilled, expected, - "backfill {:?}", - config.aggregation_type - ); + assert_eq!(backfilled, expected, "backfill {family:?}"); outputs += 1; families.insert( - format!("{:?}", config.accumulator_spec().unwrap().family) + format!("{family:?}") .split(['(', ' ', ',']) .find(|part| { [ @@ -4989,11 +4596,8 @@ mod dag_execution_tests { config.pane_origin_ms, &config.window_layout, ); - let right_closed = config - .parameters - .get("promql_right_closed") - .and_then(serde_json::Value::as_bool) - .unwrap_or(false); + // Raw PromQL programs close panes on the right, like `(start, end]` ranges. + let right_closed = true; let mut panes = BTreeMap::<(i64, i64), Vec<(&str, i64, f64)>>::new(); for (time, value) in [(1100, 10.0), (1200, 20.0), (1300, 30.0), (1400, 40.0)] { let pane_time = if right_closed { time - 1 } else { time }; diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs index ee644c84a..63f5853f0 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/accelerator.rs @@ -811,25 +811,22 @@ mod tests { "SELECT sum(value) FROM requests".into() }; let mut config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), + "requests", KeyByLabelNames::empty(), - String::new(), 1, 1, WindowKind::Tumbling, - String::new(), - "requests".into(), - None, - Some("asap_e2e.samples".into()), - Some("value".into()), ); + config.table_name = Some("asap_e2e.samples".into()); + config.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); config.pane_origin_ms = Some(0); config.table_timestamp_column = Some("timestamp_ms".into()); - let sds = SummaryCatalog::from_materializations(41, 1, &[config.clone()]).unwrap(); + let family = AggregationType::Sum.planner_exact_family().unwrap(); + config.allocate_stored_output_id(&family); + let sds = + SummaryCatalog::from_outputs(41, 1, vec![(&config, &family, String::new())]).unwrap(); let materialization = *sds.outputs.keys().next().unwrap(); let read = QueryNodeId(0); let readout = QueryNodeId(1); @@ -1022,7 +1019,7 @@ mod tests { }; let mut precompute = asap_types::precompute_plan::PrecomputePlan::build( envelope.clone(), - vec![config], + vec![(config, family)], &["fixture".into()], ) .unwrap(); @@ -1271,21 +1268,11 @@ mod tests { assert!(request.send().await.unwrap().status().is_success()); } let mut cfg = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), - KeyByLabelNames::empty(), + "requests", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 1, 1, WindowKind::Tumbling, - String::new(), - "requests".into(), - None, - None, - None, ); cfg.pane_origin_ms = Some(0); cfg.table_name = Some("asap_e2e.samples".into()); @@ -1293,10 +1280,12 @@ mod tests { cfg.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { name: "value".into(), }); + let (cfg, family) = crate::tests::test_utilities::outputs::allocated( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + ); let hot = crate::storage_engines::types::InstalledPrecomputePlanHandle::from_arc(Arc::new( - crate::storage_engines::types::InstalledPrecomputePlan::from_raw_ids(HashMap::from([ - (cfg.policy_fp_u64(), cfg.clone()), - ])), + crate::storage_engines::types::InstalledPrecomputePlan::new([(cfg.clone(), family)]), )); let registry = Arc::new(crate::storage_engines::sketch_db::backfill::BackfillRegistry::new()); diff --git a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs index 1ae3909a4..338a10651 100644 --- a/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs +++ b/data_plane/src/query_engines/asap_clickhouse_query_engine/execution.rs @@ -629,7 +629,7 @@ mod tests { let result = execute_sql_dag_with_external( &SketchStore::new(), &entry, - &SummaryCatalog::from_materializations(1, 1, &[]).unwrap(), + &SummaryCatalog::from_outputs(1, 1, Vec::new()).unwrap(), &[(QueryNodeId(0), relation)].into(), 0, 1000, @@ -821,7 +821,7 @@ mod tests { (QueryNodeId(2), relation), ]); let index = SketchStore::new(); - let catalog = SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); + let catalog = SummaryCatalog::from_outputs(1, 1, Vec::new()).unwrap(); let run = || execute_sql_dag_with_external(&index, &entry, &catalog, &prepared, 0, 1000, false); assert!(matches!(run(), ClickHouseDagOutcome::Accelerated(_))); diff --git a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs index c227e5eb0..212e2da3e 100644 --- a/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs +++ b/data_plane/src/query_engines/asap_query_engine/catalog_resolver.rs @@ -179,24 +179,16 @@ mod tests { } fn catalog_fixture() -> SummaryCatalog { - let config = PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - Default::default(), + let family = AggregationType::Sum.planner_exact_family().unwrap(); + let mut config = PrecomputeMaterialization::new( + "m", KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - "m".into(), - None, - None, - None, ); - SummaryCatalog::from_materializations(1, 1, &[config]).unwrap() + config.allocate_stored_output_id(&family); + SummaryCatalog::from_outputs(1, 1, vec![(&config, &family, String::new())]).unwrap() } // A readout cannot relabel a valid sum materialization as a rate capability. diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 7c079838b..3b6f748e3 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -1548,26 +1548,29 @@ mod tests { #[test] fn metricsql_quantile_preserves_catalog_metric_name_only_per_entity() { use control_plane::query_plan::*; - let config: asap_types::PrecomputeMaterialization = + let mut config: asap_types::PrecomputeMaterialization = serde_json::from_value(serde_json::json!({ - "aggregation_type": "DDSketch", "aggregation_sub_type": "", + "stored_output_id": 0, "metric": "latency_ms", "window_size": 1, "slide_interval": 1, "window_type": "tumbling", "num_aggregates_to_retain": 3, - "parameters": {"alpha": 0.01}, "pane_origin_ms": 0, + "pane_origin_ms": 0, "partitioning": "per_entity", "window_layout": {"kind": "pane", "pane_secs": 1}, - "grouping_labels": {"labels": []}, "aggregated_labels": {"labels": []}, - "rollup_labels": {"labels": []}, "spatial_filter": "", - "spatial_filter_normalized": "", "original_yaml": "" + "grouping_labels": {"labels": []}, "table_name": null })) .unwrap(); + let family = crate::tests::test_utilities::outputs::family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"alpha": 0.01}), + ); + config.allocate_stored_output_id(&family); let idx = ddsketch_fixture(); let mut metadata = (*idx.instance(1).unwrap()).clone(); metadata.policy_fp = config.policy_fingerprint(); idx.install_summary_catalog(std::sync::Arc::new( - asap_types::summary_catalog::SummaryCatalog::from_materializations( + asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[config.clone()], + vec![(&config, &family, String::new())], ) .unwrap(), )) @@ -1848,8 +1851,11 @@ mod tests { metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config( - config, + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + &plan.precompute_plan.population_filter(config).unwrap(), ), accuracy: None, first_seen_unix_ms: 0, @@ -1917,7 +1923,12 @@ mod tests { metric_name: "a".into(), group_by_keys: Default::default(), capability: Some(Capability::ExactAgg(asap_types::AggregationType::Sum)), - agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_config(config), + agg_kind: crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap(), + &plan.precompute_plan.population_filter(config).unwrap(), + ), accuracy: None, first_seen_unix_ms: 0, retired_at_ms: None, diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index fa3ccfcc4..026a85afe 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -13,6 +13,24 @@ use asap_types::PrecomputeMaterialization; use control_plane::physical::compiler::PLANNER_REVISION; use std::{collections::BTreeMap, sync::Arc}; +thread_local! { + /// Stored state family of each fixture output, keyed by its allocated id. + static FAMILIES: std::cell::RefCell< + std::collections::HashMap< + asap_types::sds::StoredOutputId, + planner_types::post_asap::SummaryFamilyType, + >, + > = Default::default(); +} + +/// The stored state family a fixture output was allocated with. +pub(super) fn family_of( + config: &PrecomputeMaterialization, +) -> planner_types::post_asap::SummaryFamilyType { + FAMILIES.with(|families| families.borrow()[&config.stored_output_id].clone()) +} + +/// A per-entity output of the `aggregation` kernel with these dimensions. pub(super) fn materialization( metric: &str, aggregation: &str, @@ -21,18 +39,26 @@ pub(super) fn materialization( pane_ms: u64, ) -> PrecomputeMaterialization { assert_eq!(pane_ms % 1000, 0); - serde_json::from_value(serde_json::json!({ - "aggregation_type": aggregation, "aggregation_sub_type": "", + let mut config: PrecomputeMaterialization = serde_json::from_value(serde_json::json!({ + "stored_output_id": 0, "metric": metric, "window_size": pane_ms / 1000, "slide_interval": pane_ms / 1000, "window_type": "tumbling", "num_aggregates_to_retain": 100, - "parameters": parameters, "pane_origin_ms": 0, + "pane_origin_ms": 0, "partitioning": "per_entity", "window_layout": {"kind": "pane", "pane_secs": pane_ms / 1000}, - "grouping_labels": {"labels": labels}, "aggregated_labels": {"labels": []}, - "rollup_labels": {"labels": []}, "spatial_filter": "", - "spatial_filter_normalized": "", "original_yaml": "" + "grouping_labels": {"labels": labels}, + "table_name": null })) - .unwrap() + .unwrap(); + let family = + crate::tests::test_utilities::outputs::family(aggregation.parse().unwrap(), ¶meters); + config.allocate_stored_output_id(&family); + FAMILIES.with(|families| { + families + .borrow_mut() + .insert(config.stored_output_id, family) + }); + config } pub(super) fn entry( @@ -62,16 +88,16 @@ pub(super) fn entry( .then_some(config.slide_interval * 1000), materialization: config.policy_fingerprint().into(), stored_output_reference: - asap_types::summary_catalog::SummaryCatalog::from_materializations( + asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[config.clone()], + vec![(config, &family_of(config), String::new())], ) .unwrap() .output_reference(config.policy_fingerprint().into()) .unwrap(), output_grouping: grouping, - item_labels: config.aggregated_labels.labels.clone(), + item_labels: Vec::new(), window_ms: config.stored_window_ms(), pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(lookback_ms), @@ -104,12 +130,13 @@ pub(super) fn install( planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "runtime-fixture".into(), }; - let materializations: Vec<_> = configs.iter().map(|(c, _)| c.clone()).collect(); - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &materializations) - .unwrap(); + let outputs: Vec<_> = configs + .iter() + .map(|(c, _)| (c.clone(), family_of(c))) + .collect(); let mut precompute = - PrecomputePlan::build(envelope.clone(), materializations, &["fixture".into()]).unwrap(); + PrecomputePlan::build(envelope.clone(), outputs, &["fixture".into()]).unwrap(); + let catalog = asap_types::summary_catalog::SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, diff --git a/data_plane/src/storage_engines/sketch_db/accuracy.rs b/data_plane/src/storage_engines/sketch_db/accuracy.rs index 084c87cc6..c432139c5 100644 --- a/data_plane/src/storage_engines/sketch_db/accuracy.rs +++ b/data_plane/src/storage_engines/sketch_db/accuracy.rs @@ -3,19 +3,17 @@ //! //! Implements backend accuracy metadata consumed through SummaryCatalog and QueryPlan. Logical //! guarantees are owned by ASAPPlanner and family bounds by summary libraries. -//! Given the `aggregation_type` + `parameters` pinned on an -//! `AggSchema`, the registry can expose the theoretical accuracy -//! bound of every query answer computed from it — so users and -//! control planes see "this quantile is within ε relative error with -//! probability 1 - δ" as a first-class part of the schema, not a -//! number they have to rederive from the sketch literature. +//! Given the Planner state family of a stored output, this exposes the +//! theoretical accuracy bound of every query answer computed from it — so +//! users and control planes see "this quantile is within ε relative error +//! with probability 1 - δ" as a first-class part of the schema, not a number +//! they have to rederive from the sketch literature. //! //! ## Scope of this module //! -//! Pure derivation: `derive(&PrecomputeMaterialization)` -//! looks at `aggregation_type` and the relevant entries in -//! `config.parameters` and returns an `AccuracyProfile`. No -//! runtime measurement, no sampling — just the textbook bound. +//! Pure derivation: [`derive`] maps a `SummaryFamilyType` to an +//! `AccuracyProfile`. No runtime measurement, no sampling — just the +//! textbook bound. //! //! These bounds are asymptotic / probabilistic worst-case //! guarantees from the original sketch papers. Real error @@ -27,83 +25,29 @@ //! CMS, HLL, KLL, and DDSketch profiles come from ASAPPlanner's //! `DefaultAccuracyModel`. HLL reports relative standard error with unknown //! failure probability (`delta: null`), rather than a deterministic guarantee. -//! CountSketch, heap-retention extensions, and GOS augmentation remain local. +//! CountSketch and heap-retention extensions remain local. use serde::{Deserialize, Serialize}; -use asap_types::aggregation_config::PrecomputeMaterialization; -use asap_types::AggregationType; - pub use asap_types::accuracy::{AccuracyKind, AccuracyProfile}; -use planner_types::post_asap::SketchParams as PlannerParams; - -/// Derive an [`AccuracyProfile`] from a pinned -/// [`PrecomputeMaterialization`]. Reads `aggregation_type` and any -/// necessary entries in `parameters`; falls back to exact for -/// unknown / legacy variants (harmless — the caller just gets -/// "0 error" rather than a panic). -/// -/// GOS continuous-query envelope (design-gos-unified-edge-telemetry.md §4, -/// Theorem 1): when the edge gates delta transmission by the GOS relative -/// threshold (`parameters["gos_delta_epsilon"] = ε_st > 0`), the warm -/// sketch answered from delta-applied state carries an extra DETERMINISTIC -/// staleness term of at most `ε_st` (relative) at any query time — it adds -/// linearly to the sketch's own probabilistic bound (`ε_total = ε_sk + -/// ε_st`; the random parts compose in quadrature but the staleness part is -/// adversarial, so linear addition is the honest envelope). δ is -/// unchanged (staleness is not probabilistic). -pub fn derive(config: &PrecomputeMaterialization) -> AccuracyProfile { - let mut profile = derive_sketch_only(config); - let eps_st = config - .parameters - .get("gos_delta_epsilon") - .and_then(|v| v.as_f64()) - .unwrap_or(0.0); - if eps_st > 0.0 && profile.kind != AccuracyKind::Exact { - profile.epsilon += eps_st; - } - profile -} - -/// Source adapter for installed aggregation configs. -pub trait BackendAccuracyProfile { - fn derive(config: &PrecomputeMaterialization) -> Self; - fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self; -} -impl BackendAccuracyProfile for AccuracyProfile { - fn derive(config: &PrecomputeMaterialization) -> Self { - derive(config) - } - fn derive_sketch_only(config: &PrecomputeMaterialization) -> Self { - derive_sketch_only(config) - } -} - -/// The sketch's own theoretical bound, without the GOS staleness term. -fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { - match config.aggregation_type { - AggregationType::UnivMon => AccuracyProfile { +use planner_types::post_asap::{SketchParams as PlannerParams, SummaryFamilyType}; + +/// Derive the [`AccuracyProfile`] of state with the Planner `family`. +/// Exact families have zero error. +pub fn derive(family: &SummaryFamilyType) -> AccuracyProfile { + let SummaryFamilyType::Sketch(kind, _) = family else { + return AccuracyProfile::exact(); + }; + match kind.params() { + PlannerParams::UnivMon { .. } => AccuracyProfile { epsilon: f64::MAX, delta: Some(1.0), kind: AccuracyKind::Uncalibrated, }, - // Exact aggregates. (The `SetAggregator` / - // `DeltaSetAggregator` exact-set-membership family lived - // here too before its retirement.) - AggregationType::Sum - | AggregationType::Count - | AggregationType::Increase - | AggregationType::Rate - | AggregationType::Min - | AggregationType::Max => AccuracyProfile::exact(), - - AggregationType::CountMinSketch => { - let (rows, cols) = cms_params(config); - shared_profile(PlannerParams::Cms { - depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), - width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), - }) - } + params @ (PlannerParams::Cms { .. } + | PlannerParams::Hll { .. } + | PlannerParams::Kll { .. } + | PlannerParams::DDSketch { .. }) => shared_profile(params.clone()), // CountMinSketchWithHeap: CMS frequency estimator // coupled with a heap of the top-`k` heaviest items @@ -115,36 +59,26 @@ fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { // ≥ N/heap_size is guaranteed to be in the top-K // output; each retained count is within N/heap_size // of the true value (SpaceSaving guarantee). - // We report the **tighter** of the two as the - // user-facing ε — typically the heap bound - // `1/heap_size` dominates when heap_size ≪ w, and the - // CMS bound `e/w` dominates when the heap is generously - // sized. `kind = TopK` signals that ε is the - // combined frequency + retention guarantee. - // δ stays `exp(-d)` from the CMS half; retention itself - // is deterministic given an adversarial-free stream, - // but the count estimate remains probabilistic at - // depth d. + // We report the worse of the two as the user-facing ε. + // `kind = TopK` signals that ε is the combined frequency + + // retention guarantee. δ stays `exp(-d)` from the CMS half. // Sources: // - Cormode & Muthukrishnan 2005 (CMS bound) // - Metwally, Agrawal, El Abbadi. "Efficient // computation of frequent and top-k elements in // data streams." ICDT 2005. (top-K retention) - AggregationType::CountMinSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); + PlannerParams::CmsWithHeap { + width, + depth, + heap_size, + } => { let cms = shared_profile(PlannerParams::Cms { - depth: u32::try_from(rows).unwrap_or(u32::MAX).max(1), - width: u32::try_from(cols).unwrap_or(u32::MAX).max(1), + depth: (*depth).max(1), + width: (*width).max(1), }); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - // Worst of the two — a user should expect errors - // no bigger than `ε · N`. - let epsilon = cms.epsilon.max(heap_epsilon); - let delta = cms.delta; AccuracyProfile { - epsilon, - delta, + epsilon: cms.epsilon.max(1.0 / f64::from(*heap_size).max(1.0)), + delta: cms.delta, kind: AccuracyKind::TopK, } } @@ -152,58 +86,29 @@ fn derive_sketch_only(config: &PrecomputeMaterialization) -> AccuracyProfile { // CountSketch: ε = 1/√w, δ = 1/2^d (Charikar-Chen- // Farach-Colton). Signed counters → tighter epsilon // than CMS but same confidence ramp with depth. - AggregationType::CountSketch => { - let (rows, cols) = cms_params(config); - let epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let delta = Some(0.5_f64.powi(rows as i32)); - AccuracyProfile { - epsilon, - delta, - kind: AccuracyKind::AdditiveFrequency, - } - } - - // CountSketchWithHeap: CountSketch frequency estimator - // paired with a top-k heap. Mirrors the - // CountMinSketchWithHeap branch above — the CountSketch - // half gives ε_point = 1/√w; the heap half gives - // ε_heap = 1/heap_size for retention. Report the - // tighter (max) of the two. - AggregationType::CountSketchWithHeap => { - let (rows, cols) = cms_params(config); - let heap = cms_heap_size(config); - let cs_epsilon = 1.0 / (cols as f64).max(1.0).sqrt(); - let heap_epsilon = 1.0 / (heap as f64).max(1.0); - let epsilon = cs_epsilon.max(heap_epsilon); - let delta = Some(0.5_f64.powi(rows as i32)); - AccuracyProfile { - epsilon, - delta, - kind: AccuracyKind::TopK, - } - } - - AggregationType::HLL => shared_profile(PlannerParams::Hll { - precision: u8::try_from(hll_precision(config)).unwrap_or(14), - }), - AggregationType::DatasketchesKLL | AggregationType::HydraKLL => { - shared_profile(PlannerParams::Kll { - k: kll_k(config).max(1), - }) - } - AggregationType::DDSketch => shared_profile(PlannerParams::DDSketch { - alpha: ddsketch_alpha(config), - }), + PlannerParams::CountSketch { width, depth } => AccuracyProfile { + epsilon: 1.0 / f64::from(*width).max(1.0).sqrt(), + delta: Some(0.5_f64.powi(*depth as i32)), + kind: AccuracyKind::AdditiveFrequency, + }, - // Legacy / wrapper variants. Return exact — they are - // config-shape placeholders that dispatch to concrete - // aggregator types elsewhere; their accuracy profile - // depends on the sub_type, which the factory resolves - // at updater-construction time. Phase 6.4 v2 can walk - // sub_type to give a tighter answer. - AggregationType::SingleSubpopulation | AggregationType::MultipleSubpopulation => { - AccuracyProfile::exact() - } + // CountSketchWithHeap: the CountSketch half gives ε_point = 1/√w; + // the heap half gives ε_heap = 1/heap_size for retention. + PlannerParams::CountSketchWithHeap { + width, + depth, + heap_size, + } => AccuracyProfile { + epsilon: (1.0 / f64::from(*width).max(1.0).sqrt()) + .max(1.0 / f64::from(*heap_size).max(1.0)), + delta: Some(0.5_f64.powi(*depth as i32)), + kind: AccuracyKind::TopK, + }, + _ => AccuracyProfile { + epsilon: f64::MAX, + delta: Some(1.0), + kind: AccuracyKind::Uncalibrated, + }, } } @@ -211,69 +116,6 @@ fn shared_profile(params: PlannerParams) -> AccuracyProfile { AccuracyProfile::from_sketch_params(¶ms).expect("shared family has a numeric planner bound") } -// Parameter extraction helpers. Kept file-local (not pub) because -// they duplicate tiny bits of `precompute_engine::accumulator_factory` -// and the backfill-vs-live separation rule (see that module's doc) -// says it's OK for them to drift — this module is the single -// authority on *accuracy*, not on *construction*. - -/// Read canonical depth `d` and width `w` parameters. -fn cms_params(config: &PrecomputeMaterialization) -> (u64, u64) { - let rows = config - .parameters - .get("d") - .and_then(|v| v.as_u64()) - .unwrap_or(4); - let cols = config - .parameters - .get("w") - .and_then(|v| v.as_u64()) - .unwrap_or(1000); - (rows, cols) -} - -fn hll_precision(config: &PrecomputeMaterialization) -> u32 { - config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(|v| v.as_u64()) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(14) -} - -fn kll_k(config: &PrecomputeMaterialization) -> u32 { - config - .parameters - .get("K") - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .and_then(|v| u32::try_from(v).ok()) - .unwrap_or(200) -} - -fn ddsketch_alpha(config: &PrecomputeMaterialization) -> f64 { - config - .parameters - .get("alpha") - .and_then(|v| v.as_f64()) - .unwrap_or(0.01) -} - -/// CMS-with-heap heap size (the `k` in "top-k retention"). Read -/// from `parameters["heap_size"]` with a default of 100 — -/// matches the default the control plane's planner uses when the -/// caller didn't override. -fn cms_heap_size(config: &PrecomputeMaterialization) -> u64 { - config - .parameters - .get("heap_size") - .or_else(|| config.parameters.get("topk")) - .or_else(|| config.parameters.get("k")) - .and_then(|v| v.as_u64()) - .unwrap_or(100) -} - /// Per-segment accuracy record. Attached to a multi-segment /// [`AccuracyEnvelope`] so clients can see the error bound for /// each piece of the schema-timeline-crossing query. @@ -366,104 +208,60 @@ impl AccuracyEnvelope { #[cfg(test)] mod tests { use super::*; - use asap_types::enums::WindowKind; - use asap_types::KeyByLabelNames; - use serde_json::{json, Value}; - use std::collections::HashMap; - - fn base_config( - agg_type: AggregationType, - params: HashMap, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - params, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, + use planner_types::post_asap::{ + ExactKind, ExactParams, GroupingStrategy, SketchAlgorithm, SketchKind, + }; + + fn sketch(algorithm: SketchAlgorithm, params: PlannerParams) -> SummaryFamilyType { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, ) } - /// Both source adapters must agree for the same normalized family parameters. + fn cms(depth: u32, width: u32) -> SummaryFamilyType { + sketch(SketchAlgorithm::Cms, PlannerParams::Cms { width, depth }) + } + + fn hll(precision: u8) -> SummaryFamilyType { + sketch(SketchAlgorithm::Hll, PlannerParams::Hll { precision }) + } + + /// Planner and backend derivations agree for the same shared family parameters. #[test] fn planner_and_backend_shared_family_parity() { use control_plane::types::SketchParams; let cases = vec![ - ( - SketchParams::CountMinSketch { - rows: 4, - cols: 1000, - metric_name: "m".into(), - }, - AggregationType::CountMinSketch, - json!({"d": 4, "w": 1000}), - ), ( SketchParams::CountMinSketch { rows: 7, cols: 4096, metric_name: "m".into(), }, - AggregationType::CountMinSketch, - json!({"d": 7, "w": 4096}), - ), - ( - SketchParams::HLL { precision: 14 }, - AggregationType::HLL, - json!({"precision": 14}), - ), - ( - SketchParams::HLL { precision: 10 }, - AggregationType::HLL, - json!({"p": 10}), - ), - ( - SketchParams::KLL { - k: 200, - quantiles: vec![], - }, - AggregationType::DatasketchesKLL, - json!({"K": 200}), + cms(7, 4096), ), + (SketchParams::HLL { precision: 10 }, hll(10)), ( SketchParams::KLL { k: 512, quantiles: vec![], }, - AggregationType::HydraKLL, - json!({"k": 512}), - ), - ( - SketchParams::DDSketch { - relative_accuracy: 0.01, - quantiles: vec![], - }, - AggregationType::DDSketch, - json!({"alpha": 0.01}), + sketch(SketchAlgorithm::Kll, PlannerParams::Kll { k: 512 }), ), ( SketchParams::DDSketch { relative_accuracy: 0.05, quantiles: vec![], }, - AggregationType::DDSketch, - json!({"alpha": 0.05}), + sketch( + SketchAlgorithm::DDSketch, + PlannerParams::DDSketch { alpha: 0.05 }, + ), ), ]; - for (params, family, wire) in cases { - let config = base_config(family, serde_json::from_value(wire).unwrap()); + for (params, family) in cases { let planner = control_plane::accuracy::derive(¶ms); - let backend = derive(&config); + let backend = derive(&family); assert_eq!(planner, backend, "parameters: {params:?}"); assert_eq!( serde_json::to_value(planner).unwrap(), @@ -472,30 +270,10 @@ mod tests { } } - /// GOS widens epsilon without inventing confidence for HLL. - #[test] - fn shared_family_gos_preserves_planner_confidence() { - for family in [ - AggregationType::CountMinSketch, - AggregationType::HLL, - AggregationType::DatasketchesKLL, - AggregationType::DDSketch, - ] { - let base = derive(&base_config(family, HashMap::new())); - let widened = derive(&base_config( - family, - HashMap::from([("gos_delta_epsilon".into(), json!(0.05))]), - )); - assert_eq!(widened.epsilon, base.epsilon + 0.05); - assert_eq!(widened.delta, base.delta); - assert_eq!(widened.kind, base.kind); - } - } - /// Unknown confidence stays unknown regardless of segment order. #[test] fn envelope_preserves_unknown_failure_probability() { - let hll = derive(&base_config(AggregationType::HLL, HashMap::new())); + let hll = derive(&hll(14)); assert_eq!(hll.delta, None); for profiles in [ [hll, AccuracyProfile::exact()], @@ -519,230 +297,78 @@ mod tests { /// Backend-only UnivMon must not acquire a calibrated shared-family bound. #[test] fn univmon_remains_uncalibrated() { - let profile = derive(&base_config(AggregationType::UnivMon, HashMap::new())); - assert_eq!(profile.kind, AccuracyKind::Uncalibrated); - assert_eq!(profile.epsilon, f64::MAX); - assert_eq!(profile.delta, Some(1.0)); - } - - #[test] - fn sum_is_exact() { - let p = derive(&base_config(AggregationType::Sum, HashMap::new())); - assert_eq!(p.kind, AccuracyKind::Exact); - assert_eq!(p.epsilon, 0.0); - assert_eq!(p.delta, Some(0.0)); + let p = derive(&sketch( + SketchAlgorithm::UnivMon, + PlannerParams::UnivMon { + heap_size: 32, + sketch_rows: 5, + sketch_cols: 1024, + layers: 4, + }, + )); + assert_eq!(p.kind, AccuracyKind::Uncalibrated); } + /// Every exact family has zero error. #[test] - fn min_max_increase_are_exact() { - for t in [ - AggregationType::Min, - AggregationType::Max, - AggregationType::Increase, + fn exact_families_are_exact() { + for (kind, params) in [ + (ExactKind::Sum, ExactParams::Sum), + (ExactKind::Min, ExactParams::Min), + (ExactKind::Max, ExactParams::Max), + (ExactKind::Increase, ExactParams::Increase), ] { - let p = derive(&base_config(t, HashMap::new())); + let p = derive(&SummaryFamilyType::ExactAggregate(kind, params)); + assert_eq!(p.epsilon, 0.0); assert_eq!(p.kind, AccuracyKind::Exact); } } - #[test] - fn gos_staleness_widens_epsilon_linearly() { - // A CountSketch (ε_sk = 1/√w) whose edge gates deltas at ε_st carries - // ε_total = ε_sk + ε_st in the continuous-query envelope (Theorem 1); - // δ is unchanged (staleness is deterministic). - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(256)); - let base = derive(&base_config(AggregationType::CountSketch, params.clone())); - params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let widened = derive(&base_config(AggregationType::CountSketch, params)); - assert!((widened.epsilon - (base.epsilon + 0.05)).abs() < 1e-12); - assert_eq!(widened.delta, base.delta); - assert_eq!(widened.kind, base.kind); - } - - #[test] - fn gos_staleness_does_not_touch_exact() { - // Exact aggregates are not GOS-gated (Count-Sketch families only), so a - // stray parameter must not fabricate an ε>0 "exact" answer. - let mut params = HashMap::new(); - params.insert("gos_delta_epsilon".to_string(), json!(0.05)); - let p = derive(&base_config(AggregationType::Sum, params)); - assert_eq!(p.epsilon, 0.0); - assert_eq!(p.kind, AccuracyKind::Exact); - } - + /// CMS reports ε = e/w and δ = e^-d. #[test] fn cms_epsilon_is_e_over_w() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(2718)); - let p = derive(&base_config(AggregationType::CountMinSketch, params)); - // e / 2718 ≈ 0.0010001 — very close to 0.001. + let p = derive(&cms(5, 2718)); assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); assert!((p.epsilon - std::f64::consts::E / 2718.0).abs() < 1e-12); - // δ = 1/2^5 = 0.03125 assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); } + /// A heap reports the worse of the CMS point bound and the retention bound. #[test] - fn cms_uses_defaults_when_params_absent() { - let p = derive(&base_config( - AggregationType::CountMinSketch, - HashMap::new(), - )); - // Defaults rows=4, cols=1000 per accumulator_factory. - assert!((p.epsilon - std::f64::consts::E / 1000.0).abs() < 1e-12); - assert!((p.delta.unwrap() - (-4.0_f64).exp()).abs() < 1e-12); // 1/16 - } - - #[test] - fn cms_with_heap_carries_top_k_kind_and_heap_bound() { - // Large heap: 1/heap_size (= 1e-4) dominates the e/w CMS - // bound (e/1e6 ≈ 2.72e-6). Expect ε = 1/heap. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(5)); - params.insert("w".to_string(), json!(1_000_000)); - params.insert("heap_size".to_string(), json!(10_000)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - 1.0 / 10_000.0).abs() < 1e-12); - assert!((p.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); // 1/2^5 - } - - #[test] - fn cms_with_heap_cms_bound_dominates_when_heap_is_generous() { - // Generously-sized heap (1e6) + narrow CMS (w=100) → - // 1/heap (1e-6) ≪ e/w (2.7e-2), so the CMS bound dominates. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(100)); - params.insert("heap_size".to_string(), json!(1_000_000)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - std::f64::consts::E / 100.0).abs() < 1e-12); - } - - #[test] - fn cms_with_heap_uses_default_heap_size_100() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(1000)); - // heap_size absent → default 100 → 1/100 = 0.01 dominates - // e/1000 ≈ 0.00272. - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert_eq!(p.kind, AccuracyKind::TopK); - assert!((p.epsilon - 0.01).abs() < 1e-12); - } - - #[test] - fn cms_with_heap_accepts_alternative_param_names() { - // Config may name the heap "k" or "topk" instead of - // "heap_size" — all three should work. - for alias in ["heap_size", "topk", "k"] { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(1_000_000)); - params.insert(alias.to_string(), json!(500)); - let p = derive(&base_config( - AggregationType::CountMinSketchWithHeap, - params, - )); - assert!( - (p.epsilon - 1.0 / 500.0).abs() < 1e-12, - "alias '{}' should produce heap-driven ε", - alias - ); - } + fn cms_with_heap_reports_the_worse_of_point_and_retention_bounds() { + let heap = |depth, width, heap_size| { + derive(&sketch( + SketchAlgorithm::CmsWithHeap, + PlannerParams::CmsWithHeap { + width, + depth, + heap_size, + }, + )) + }; + let retention = heap(5, 1_000_000, 10_000); + assert_eq!(retention.kind, AccuracyKind::TopK); + assert!((retention.epsilon - 1.0 / 10_000.0).abs() < 1e-12); + assert!((retention.delta.unwrap() - (-5.0_f64).exp()).abs() < 1e-12); + let point = heap(4, 100, 1_000_000); + assert!((point.epsilon - std::f64::consts::E / 100.0).abs() < 1e-12); } + /// CountSketch reports ε = 1/√w, which differs from the CMS bound. #[test] fn countsketch_epsilon_is_one_over_sqrt_w() { - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(100)); - let p = derive(&base_config(AggregationType::CountSketch, params)); - assert_eq!(p.kind, AccuracyKind::AdditiveFrequency); - assert!((p.epsilon - 0.1).abs() < 1e-9); // 1/√100 = 0.1 - assert!((p.delta.unwrap() - 0.0625).abs() < 1e-12); // Backend-only CountSketch convention. - } - - #[test] - fn hll_epsilon_matches_flajolet_bound() { - let mut params = HashMap::new(); - params.insert("precision".to_string(), json!(14)); - let p = derive(&base_config(AggregationType::HLL, params)); - assert_eq!(p.kind, AccuracyKind::RelativeCardinality); - // 1.04 / √16384 = 1.04 / 128 = 0.008125 - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn hll_uses_default_precision_14() { - let p = derive(&base_config(AggregationType::HLL, HashMap::new())); - assert!((p.epsilon - 0.008125).abs() < 1e-9); - } - - #[test] - fn kll_epsilon_matches_karnin_lang_liberty_bound() { - let mut params = HashMap::new(); - params.insert("K".to_string(), json!(200)); - let p = derive(&base_config(AggregationType::DatasketchesKLL, params)); - assert_eq!(p.kind, AccuracyKind::RankQuantile); - // Planner uses the DataSketches empirical 99th-percentile rank-error fit. - assert!((p.epsilon - 2.296 / 200.0_f64.powf(0.9723)).abs() < 1e-12); - assert!((p.delta.unwrap() - 0.01).abs() < 1e-12); - } - - #[test] - fn hydra_kll_follows_the_same_kll_bound() { - let mut params = HashMap::new(); - params.insert("k".to_string(), json!(400)); - let p = derive(&base_config(AggregationType::HydraKLL, params)); - // Planner uses the DataSketches empirical 99th-percentile rank-error fit. - assert!((p.epsilon - 2.296 / 400.0_f64.powf(0.9723)).abs() < 1e-12); - } - - #[test] - fn ddsketch_epsilon_is_alpha_directly() { - let mut params = HashMap::new(); - params.insert("alpha".to_string(), json!(0.02)); - let p = derive(&base_config(AggregationType::DDSketch, params)); - assert_eq!(p.kind, AccuracyKind::RelativeQuantile); - assert_eq!(p.epsilon, 0.02); - assert_eq!(p.delta, Some(0.0)); - } - - #[test] - fn ddsketch_uses_default_alpha_0_01() { - let p = derive(&base_config(AggregationType::DDSketch, HashMap::new())); - assert_eq!(p.epsilon, 0.01); - } - - #[test] - fn legacy_wrapper_types_fall_back_to_exact() { - // `SingleSubpopulation` / `MultipleSubpopulation` are - // config-shape wrappers whose real type is in sub_type. - // Without resolving sub_type we return exact (harmless - // lower bound). Phase 6.4 v2 may tighten this. - for t in [ - AggregationType::SingleSubpopulation, - AggregationType::MultipleSubpopulation, - ] { - let p = derive(&base_config(t, HashMap::new())); - assert_eq!(p.kind, AccuracyKind::Exact); - } + let cs = derive(&sketch( + SketchAlgorithm::CountSketch, + PlannerParams::CountSketch { + width: 10_000, + depth: 4, + }, + )); + assert!((cs.epsilon - 0.01).abs() < 1e-12); + assert!(derive(&cms(4, 10_000)).epsilon < cs.epsilon); } + /// Profiles round-trip through their wire form. #[test] fn accuracy_profile_roundtrips_through_serde() { let input = AccuracyProfile { @@ -755,25 +381,4 @@ mod tests { let back: AccuracyProfile = serde_json::from_str(&json).unwrap(); assert_eq!(back, input); } - - #[test] - fn cms_and_countsketch_differ_by_sqrt_e() { - // Sanity check: for the same (w, d), CountSketch's ε is - // smaller by a factor of √e / √w × 1/√w = 1/(√e) — - // i.e. CountSketch is a factor ~1.65 tighter than CMS on - // epsilon alone. Confirms the bounds are not copy-pasted. - let mut params = HashMap::new(); - params.insert("d".to_string(), json!(4)); - params.insert("w".to_string(), json!(10000)); - let cms = derive(&base_config( - AggregationType::CountMinSketch, - params.clone(), - )); - let cs = derive(&base_config(AggregationType::CountSketch, params)); - // cms.epsilon = e/10000 ≈ 2.718e-4 - // cs.epsilon = 1/√10000 = 0.01 = 1e-2 - // So cms < cs (for w=10000). They cross at w = e. - assert!(cms.epsilon < cs.epsilon); - assert!(cs.epsilon > 0.0); - } } diff --git a/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs b/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs index acea64985..1970da66c 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/clickhouse_reader.rs @@ -242,7 +242,7 @@ pub fn clickhouse_reader_factory(config: ClickHouseReaderConfig) -> ReaderFactor return Err("table materialization has no explicit value projection".into()) } } - materialization.population_filter_canonical()?; + materialization.table_population_canonical()?; let mut reader = ClickHouseReader::new(source_config)?; reader.population = Some(materialization.table_population.clone().unwrap_or_default()); reader.value_projection = Some(materialization.effective_value_projection().clone()); @@ -548,22 +548,16 @@ mod tests { #[test] fn typed_source_enters_clickhouse_backfill_lifecycle() { let mut materialization = asap_types::PrecomputeMaterialization::new( - asap_types::AggregationType::Sum, - String::new(), - Default::default(), + "samples.value", asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 1, 1, asap_types::WindowKind::Tumbling, - String::new(), - "samples.value".into(), - None, - Some("another_table".into()), - Some("value".into()), ); + materialization.table_name = Some("another_table".into()); + materialization.value_projection = Some(asap_types::sds::ValueProjectionIdentity::Column { + name: "value".into(), + }); let factory = clickhouse_reader_factory(config("samples")); materialization.table_timestamp_column = Some("timestamp_ms".into()); let reader = factory( diff --git a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs index ee6c355ee..a82e8ecfa 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/processor.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/processor.rs @@ -216,6 +216,9 @@ impl WindowProcessor for BackfillWindowProcessor { .cloned() .ok_or_else(|| format!("agg_id {agg_id} not in current InstalledPrecomputePlan"))?; let program = snapshot.raw_programs.get(&agg_id).cloned(); + let kind = snapshot + .agg_kind(asap_types::sds::StoredOutputId(agg_id)) + .ok_or_else(|| format!("agg_id {agg_id} has no installed state schema"))?; #[cfg(not(test))] if program.is_none() { return Err("backfill requires a post-ASAP DAG installation".into()); @@ -299,7 +302,15 @@ impl WindowProcessor for BackfillWindowProcessor { } else { #[cfg(test)] { - build_backfilled_accumulator(&config, &samples) + build_backfilled_accumulator( + snapshot + .state_family(config.stored_output_id) + .ok_or("backfill fixture has no state family")?, + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &samples, + ) } #[cfg(not(test))] { @@ -372,6 +383,7 @@ impl WindowProcessor for BackfillWindowProcessor { &address, _resolver, &config, + &kind, output, accumulator.as_ref(), ) @@ -428,29 +440,18 @@ mod tests { } else { KeyByLabelNames::from_names(grouping.into_iter().map(String::from).collect()) }; - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - std::collections::HashMap::new(), - grouping_labels, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + let mut config = + PrecomputeMaterialization::new(metric, grouping_labels, 60, 60, WindowKind::Tumbling); + config.allocate_stored_output_id(&sum_family()); + config + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } fn streaming_config_with(config: PrecomputeMaterialization) -> Arc { - let mut map = std::collections::HashMap::new(); - map.insert(config.policy_fp_u64(), config); - Arc::new(InstalledPrecomputePlan::from_raw_ids(map)) + Arc::new(InstalledPrecomputePlan::new([(config, sum_family())])) } #[tokio::test] @@ -640,7 +641,12 @@ mod tests { // Live path: factory + update_single per sample in order. let live_bytes = { - let mut updater = create_fixture_accumulator(&cfg); + let mut updater = create_fixture_accumulator( + &sum_family(), + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + ); for s in &samples { updater.update_single(s.value, s.timestamp_ms); } @@ -650,7 +656,13 @@ mod tests { // Backfill path: build_backfilled_accumulator. let backfill_bytes = { - let acc = build_backfilled_accumulator(&cfg, &samples); + let acc = build_backfilled_accumulator( + &sum_family(), + &planner_types::post_asap::SummaryUpdate::column( + planner_types::pre_asap::ColumnRef::SampleValue, + ), + &samples, + ); acc.serialize_to_bytes() }; @@ -843,10 +855,10 @@ mod tests { let hot = InstalledPrecomputePlanHandle::from_arc(streaming.clone()); let registry = Arc::new(BackfillRegistry::new()); let summary_store = Arc::new(SketchStore::new()); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let catalog = asap_types::summary_catalog::SummaryCatalog::from_outputs( 1, 1, - &[cfg.clone()], + vec![(&cfg, &sum_family(), String::new())], ) .unwrap(); summary_store @@ -1000,6 +1012,7 @@ mod tests { .ok() }, &cfg, + &crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), ""), &output, &acc, ) diff --git a/data_plane/src/storage_engines/sketch_db/backfill/service.rs b/data_plane/src/storage_engines/sketch_db/backfill/service.rs index 8e5639424..fb6674010 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/service.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/service.rs @@ -352,29 +352,22 @@ mod tests { fn sum_config(_agg_id: u64, metric: &str) -> PrecomputeMaterialization { // `_agg_id` is unused after PR 5 — identity is content-addressed. - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - std::collections::HashMap::new(), + let mut config = PrecomputeMaterialization::new( + metric, KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), 60, 60, WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, - ) + ); + config.allocate_stored_output_id(&AggregationType::Sum.planner_exact_family().unwrap()); + config } fn streaming_with(cfg: PrecomputeMaterialization) -> Arc { - let mut m = std::collections::HashMap::new(); - m.insert(cfg.policy_fp_u64(), cfg); - Arc::new(InstalledPrecomputePlan::from_raw_ids(m)) + Arc::new(InstalledPrecomputePlan::new([( + cfg, + AggregationType::Sum.planner_exact_family().unwrap(), + )])) } async fn wait_for_status( diff --git a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs index d419cb1fc..3b9705028 100644 --- a/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs +++ b/data_plane/src/storage_engines/sketch_db/backfill/window_builder.rs @@ -12,8 +12,6 @@ use crate::storage_engines::types::AggregateCore; use crate::tests::accumulator_fixture::create_fixture_accumulator; #[cfg(test)] use asap_physical_operators::factory::AccumulatorUpdater; -#[cfg(test)] -use asap_types::aggregation_config::PrecomputeMaterialization; /// Construct the accumulator for one `(agg_id, window)` pair by /// feeding `samples` in order into a fresh `AccumulatorUpdater`. @@ -30,17 +28,18 @@ use asap_types::aggregation_config::PrecomputeMaterialization; /// implementation without worrying about the async runtime. #[cfg(test)] pub fn build_backfilled_accumulator( - config: &PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, + update: &planner_types::post_asap::SummaryUpdate, samples: &[RawSample], ) -> Box { - let mut updater: Box = create_fixture_accumulator(config); + let mut updater: Box = create_fixture_accumulator(family, update); for sample in samples { apply_sample( &mut *updater, &sample.labels, sample.value, sample.timestamp_ms, - config, + family, ); } updater.take_accumulator() @@ -49,30 +48,16 @@ pub fn build_backfilled_accumulator( #[cfg(test)] mod tests { use super::*; - use asap_types::aggregation_config::PrecomputeMaterialization; - use asap_types::enums::WindowKind; use asap_types::AggregationType; - use asap_types::KeyByLabelNames; - use std::collections::HashMap; + use planner_types::post_asap::{SummaryInputExpr, SummaryUpdate}; + use planner_types::pre_asap::ColumnRef; - fn sum_config() -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) + fn value_update() -> SummaryUpdate { + SummaryUpdate::column(ColumnRef::SampleValue) + } + + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + AggregationType::Sum.planner_exact_family().unwrap() } fn raw(labels: &str, ts: i64, v: f64) -> RawSample { @@ -94,18 +79,20 @@ mod tests { AggregationType::CountSketchWithHeap, ] { for mode in ["count", "value"] { - let mut config = sum_config(); - config.aggregation_type = kind; - config.parameters = serde_json::from_value(serde_json::json!({ - "d": 4, "w": 1024, "heap_size": 10, "weight_mode": mode - })) - .unwrap(); + let family = crate::tests::test_utilities::outputs::family( + kind, + &serde_json::json!({"d": 4, "w": 1024, "heap_size": 10}), + ); + let mut update = value_update(); + if mode == "count" { + update.weight = SummaryInputExpr::Constant(1.0); + } let samples = vec![ raw("m{svc=\"a\"}", 10, 100.0), raw("m{svc=\"b\"}", 20, 2.0), raw("m{svc=\"b\"}", 30, 3.0), ]; - let acc = build_backfilled_accumulator(&config, &samples); + let acc = build_backfilled_accumulator(&family, &update, &samples); let mut ranked: Vec<(String, f64)> = if let Some(heap) = acc.as_any() .downcast_ref::() @@ -138,21 +125,18 @@ mod tests { #[test] fn sum_accumulator_sums_all_samples_in_order() { - let config = sum_config(); let samples = vec![ raw("m{svc=\"a\"}", 10, 1.0), raw("m{svc=\"a\"}", 20, 2.0), raw("m{svc=\"a\"}", 30, 3.0), ]; - let acc = build_backfilled_accumulator(&config, &samples); + let acc = build_backfilled_accumulator(&sum_family(), &value_update(), &samples); assert_eq!(crate::tests::accumulator_fixture::sum_of(acc.as_ref()), 6.0); } #[test] fn empty_samples_produce_empty_accumulator() { - let config = sum_config(); - let acc = build_backfilled_accumulator(&config, &[]); - // An empty exact Sum is the additive identity. + let acc = build_backfilled_accumulator(&sum_family(), &value_update(), &[]); assert_eq!(crate::tests::accumulator_fixture::sum_of(acc.as_ref()), 0.0); } } diff --git a/data_plane/src/storage_engines/sketch_db/data/mod.rs b/data_plane/src/storage_engines/sketch_db/data/mod.rs index f515c9e14..f12ac1c55 100644 --- a/data_plane/src/storage_engines/sketch_db/data/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/data/mod.rs @@ -33,9 +33,6 @@ //! (in `drivers::ingest::series_resolver`) is the single mint //! authority for every sid in the system. `AggKind::canonical_string()` //! here produces the third element of the resolver's cache key. -//! - [`canonical_parameters`] — helper that renders a parameters -//! `HashMap` into the canonical string form -//! `AggKind::ExactAgg::parameters_canonical` expects. //! //! ## Re-exports for callers //! @@ -122,11 +119,9 @@ pub enum AggKind { /// accumulator serializer. ExactAgg { agg_type: AggregationType, - /// Stable canonical encoding of the agg's - /// `parameters: HashMap` — sorted keys, each - /// value rendered via serde_json. Held as a single owned - /// string so equality / hashing stay cheap and independent of - /// the original HashMap's iteration order. + /// Canonical tuning parameters of the exact state. Planner exact + /// families have none, so installed outputs register it empty; + /// recovered legacy records may carry a non-empty encoding. parameters_canonical: String, /// Canonical form of the policy's spatial-filter predicate; /// see the `Sketch` variant's field doc. @@ -135,103 +130,84 @@ pub enum AggKind { } /// Complete resolver identity for a configured materialization. All live and -/// replay paths must include policy semantics, not just the sketch family. -pub(crate) fn materialization_kind_for_config( +/// replay paths must include the stored output, not just the sketch family. +pub(crate) fn materialization_kind( config: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, ) -> String { format!( "{}|{}", - agg_kind_for_config(config).canonical_string(), + kind.canonical_string(), config.policy_fingerprint() ) } -/// Resolve the physical state family produced by a precompute policy. This is -/// shared by SID minting and store registration so a sketch policy can never -/// be minted as `ExactAgg` and later registered as `Sketch` (or vice versa). -pub fn agg_kind_for_config( - config: &asap_types::aggregation_config::PrecomputeMaterialization, +/// The physical state kind of a stored output of the Planner `family` whose +/// input has the canonical predicate `filter`. This is shared by SID minting +/// and store registration so a sketch output can never be minted as `ExactAgg` +/// and later registered as `Sketch` (or vice versa). +pub fn agg_kind_for_family( + family: &planner_types::post_asap::SummaryFamilyType, + filter: &str, ) -> AggKind { use planner_types::post_asap::{SketchAlgorithm as Algorithm, SketchParams, SummaryFamilyType}; - - // HLL is intentionally absent from raw-value accumulator dispatch because - // it arrives through SketchEnvelope ingest. It is still a sketch for SID - // identity and store registration, so classify it before consulting the - // accumulator factory contract. - let sketch = (config.aggregation_type == AggregationType::HLL) - .then(|| { - let precision = config - .parameters - .get("precision") - .or_else(|| config.parameters.get("p")) - .and_then(serde_json::Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) - .unwrap_or(14); - AggKind::Sketch { - algorithm: SketchAlgorithm::Hll, - config: SketchConfig::Hll { precision }, - spatial_filter_canonical: config.spatial_filter_normalized.clone(), - } - }) - .or_else(|| { - config.accumulator_spec().ok().and_then(|spec| { - let SummaryFamilyType::Sketch(kind, _) = spec.family else { - return None; - }; - let algorithm = kind.algorithm().clone(); - let physical = match (kind.algorithm(), kind.params()) { - ( - Algorithm::UnivMon, - SketchParams::UnivMon { - heap_size, - sketch_rows, - sketch_cols, - layers, - }, - ) => SketchConfig::UnivMon { - heap_size: *heap_size, - sketch_rows: *sketch_rows, - sketch_cols: *sketch_cols, - layers: *layers, + let sketch = match family { + SummaryFamilyType::Sketch(kind, _) => { + let physical = match (kind.algorithm(), kind.params()) { + ( + Algorithm::UnivMon, + SketchParams::UnivMon { + heap_size, + sketch_rows, + sketch_cols, + layers, }, - (Algorithm::DDSketch, SketchParams::DDSketch { alpha }) => { - SketchConfig::DDSketch { - relative_accuracy: *alpha, - } - } - (Algorithm::Kll, SketchParams::Kll { k }) => SketchConfig::Kll { k: *k }, - (Algorithm::Hll, SketchParams::Hll { precision }) => SketchConfig::Hll { - precision: (*precision).into(), - }, - (Algorithm::Cms, SketchParams::Cms { width, depth }) - | (Algorithm::CmsWithHeap, SketchParams::CmsWithHeap { width, depth, .. }) => { - SketchConfig::CountMin { - rows: *depth as i32, - cols: *width as i32, - } - } - (Algorithm::CountSketch, SketchParams::CountSketch { width, depth }) - | ( - Algorithm::CountSketchWithHeap, - SketchParams::CountSketchWithHeap { width, depth, .. }, - ) => SketchConfig::CountSketch { + ) => Some(SketchConfig::UnivMon { + heap_size: *heap_size, + sketch_rows: *sketch_rows, + sketch_cols: *sketch_cols, + layers: *layers, + }), + (Algorithm::DDSketch, SketchParams::DDSketch { alpha }) => { + Some(SketchConfig::DDSketch { + relative_accuracy: *alpha, + }) + } + (Algorithm::Kll, SketchParams::Kll { k }) => Some(SketchConfig::Kll { k: *k }), + (Algorithm::Hll, SketchParams::Hll { precision }) => Some(SketchConfig::Hll { + precision: (*precision).into(), + }), + (Algorithm::Cms, SketchParams::Cms { width, depth }) + | (Algorithm::CmsWithHeap, SketchParams::CmsWithHeap { width, depth, .. }) => { + Some(SketchConfig::CountMin { rows: *depth as i32, cols: *width as i32, - }, - _ => return None, - }; - Some(AggKind::Sketch { - algorithm, - config: physical, - spatial_filter_canonical: config.spatial_filter_normalized.clone(), - }) + }) + } + (Algorithm::CountSketch, SketchParams::CountSketch { width, depth }) + | ( + Algorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { width, depth, .. }, + ) => Some(SketchConfig::CountSketch { + rows: *depth as i32, + cols: *width as i32, + }), + _ => None, + }; + physical.map(|config| AggKind::Sketch { + algorithm: kind.algorithm().clone(), + config, + spatial_filter_canonical: filter.to_owned(), }) - }); - + } + _ => None, + }; sketch.unwrap_or_else(|| AggKind::ExactAgg { - agg_type: config.aggregation_type, - parameters_canonical: canonical_parameters(&config.parameters), - spatial_filter_canonical: config.spatial_filter_normalized.clone(), + agg_type: asap_types::aggregation_type_for_family(family) + .expect("installed plans only carry families with a stored-state kernel"), + // Exact families have no tuning parameters. + parameters_canonical: String::new(), + spatial_filter_canonical: filter.to_owned(), }) } @@ -296,24 +272,6 @@ impl AggKind { } } -/// Render a `HashMap` of parameters into the canonical -/// string form `AggKind::ExactAgg::parameters_canonical` expects. -/// Keys sorted lexicographically; each value via `serde_json`. -pub fn canonical_parameters( - parameters: &std::collections::HashMap, -) -> String { - let sorted: std::collections::BTreeMap<&String, &serde_json::Value> = - parameters.iter().collect(); - let mut buf = String::new(); - for (k, v) in sorted { - buf.push_str(k); - buf.push('='); - buf.push_str(&serde_json::to_string(v).unwrap_or_default()); - buf.push(';'); - } - buf -} - impl AggKind { /// Stable string form of this `AggKind`, used as the third element /// of the `SeriesIdResolver` cache key and as the `agg_kind_canonical` @@ -355,9 +313,7 @@ impl AggKind { spatial_filter_canonical, } => { // `AggregationType`'s `Display` impl is stable - // (matches the snake-case form on the wire) and - // `parameters_canonical` is already canonicalized - // upstream (see [`canonical_parameters`]). + // (matches the snake-case form on the wire). format!( "exact_agg:{}:{}:filter={}", agg_type, parameters_canonical, spatial_filter_canonical, @@ -579,31 +535,19 @@ impl AggPayload { #[cfg(test)] mod tests { use super::*; - use asap_types::{enums::WindowKind, KeyByLabelNames}; - use std::collections::HashMap; + // HLL state registers as a sketch kind with its precision. #[test] - fn hll_envelope_config_is_registered_as_a_sketch() { - let config = asap_types::aggregation_config::PrecomputeMaterialization::new( - AggregationType::HLL, - String::new(), - HashMap::from([("precision".to_string(), serde_json::json!(12))]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "unique_users".to_string(), - None, - None, - None, + fn hll_family_is_registered_as_a_sketch() { + let family = planner_types::post_asap::SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + SketchAlgorithm::Hll, + planner_types::post_asap::SketchParams::Hll { precision: 12 }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, ); - assert!(matches!( - agg_kind_for_config(&config), + agg_kind_for_family(&family, ""), AggKind::Sketch { algorithm: SketchAlgorithm::Hll, config: SketchConfig::Hll { precision: 12 }, diff --git a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs index b57e679de..9a3b3002c 100644 --- a/data_plane/src/storage_engines/sketch_db/index/maintenance.rs +++ b/data_plane/src/storage_engines/sketch_db/index/maintenance.rs @@ -638,6 +638,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, sources: &[FrozenExactWindows], @@ -646,6 +647,7 @@ impl SketchStore { self.publish_frozen_output( sid, config, + kind, output, state, FrozenPublicationInputs::Requested(sources), @@ -657,6 +659,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, cohort: &CompleteRawMaintenanceCohort, @@ -665,6 +668,7 @@ impl SketchStore { self.publish_frozen_output( sid, config, + kind, output, state, FrozenPublicationInputs::Complete(cohort), @@ -676,6 +680,7 @@ impl SketchStore { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn AggregateCore, proof: FrozenPublicationInputs<'_>, @@ -711,7 +716,7 @@ impl SketchStore { return Err("derived publication differs from its installed input identity".into()); } let labels = self - .register_precompute_output_with_instances(sid, config, output, &mut instances) + .register_precompute_output_with_instances(sid, config, kind, output, &mut instances) .ok_or("derived output registration failed")?; let _mutation = self.begin_state_mutation(); let binding = instances.get(&sid).ok_or("derived output was removed")?; @@ -799,6 +804,32 @@ mod tests { use super::*; use crate::storage_engines::types::PrecomputedOutput; + fn sum_family() -> planner_types::post_asap::SummaryFamilyType { + asap_types::AggregationType::Sum + .planner_exact_family() + .unwrap() + } + + fn sum_kind() -> AggKind { + crate::storage_engines::sketch_db::data::agg_kind_for_family(&sum_family(), "") + } + + fn sum_catalog( + plan_version: u64, + configs: &[asap_types::PrecomputeMaterialization], + ) -> asap_types::summary_catalog::SummaryCatalog { + let family = sum_family(); + asap_types::summary_catalog::SummaryCatalog::from_outputs( + 1, + plan_version, + configs + .iter() + .map(|c| (c, &family, String::new())) + .collect(), + ) + .unwrap() + } + #[test] fn complete_population_keeps_every_sid_and_rejects_missing_live_binding() { // Multiple SIDs are inventory entries, never an implicit singleton; @@ -812,18 +843,16 @@ mod tests { .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); - first.aggregation_type = asap_types::AggregationType::Sum; - first.aggregation_sub_type = "sum".into(); first.grouping_labels = ["instance".to_string()].into_iter().collect(); + first.allocate_stored_output_id(&"sum"); let mut target = first.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([first.policy_fingerprint().into()]), program_sha256: "0".repeat(64), }); + target.allocate_stored_output_id(&"sum"); let configs = [first.clone(), first, target]; - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &configs) - .unwrap(); + let catalog = sum_catalog(1, &configs); let store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(catalog.clone())) @@ -864,7 +893,15 @@ mod tests { revision, revision, 2000, - |writer| writer.ingest_precompute_with_series_id(sid, config, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + sid, + config, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); } @@ -923,11 +960,11 @@ mod tests { .compile_promql() .unwrap(); let mut first = plan.precompute_plan.materializations[0].clone(); - first.aggregation_type = asap_types::AggregationType::Sum; - first.aggregation_sub_type = "sum".into(); first.grouping_labels = std::iter::empty::().collect(); + first.allocate_stored_output_id(&"sum"); let mut second = first.clone(); second.metric = "cohort_second".into(); + second.allocate_stored_output_id(&"sum"); let mut target = first.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([ @@ -936,10 +973,9 @@ mod tests { ]), program_sha256: "0".repeat(64), }); + target.allocate_stored_output_id(&"sum"); let configs = [first, second, target]; - let catalog = - asap_types::summary_catalog::SummaryCatalog::from_materializations(1, 1, &configs) - .unwrap(); + let catalog = sum_catalog(1, &configs); let mut store = Arc::new(SketchStore::new()); store .install_summary_catalog(Arc::new(catalog.clone())) @@ -979,7 +1015,15 @@ mod tests { revision, revision, 2000, - |writer| writer.ingest_precompute_with_series_id(sid, config, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + sid, + config, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); requests.push(( @@ -1018,6 +1062,7 @@ mod tests { writer.ingest_precompute_with_series_id( 900, &configs[0], + &sum_kind(), &extra_window, &extra_sum, ) @@ -1056,7 +1101,13 @@ mod tests { sum.update(None, 11.0, 0); assert!(store .publish_complete_raw_maintenance_output( - 902, target, &output, &sum, &complete, [42; 32] + 902, + target, + &sum_kind(), + &output, + &sum, + &complete, + [42; 32] ) .unwrap()); assert!(store @@ -1093,11 +1144,17 @@ mod tests { let mut extra = PrecomputedOutput::new(0, 1000, None, configs[0].policy_fingerprint()); extra.catalog_generation = Some(Arc::clone(&generation)); assert!(store - .register_precompute_output(904, &configs[0], &extra) + .register_precompute_output(904, &configs[0], &sum_kind(), &extra) .is_some()); assert!(store .publish_complete_raw_maintenance_output( - 905, target, &output, &sum, &complete, [42; 32] + 905, + target, + &sum_kind(), + &output, + &sum, + &complete, + [42; 32] ) .is_err()); assert!(!store.instances.read().unwrap().contains_key(&905)); @@ -1111,7 +1168,15 @@ mod tests { .any(|record| record.storage_handle == 905)); store.force_expire(901).unwrap(); assert!(store - .publish_frozen_maintenance_output(903, target, &output, &sum, &cohort, [42; 32]) + .publish_frozen_maintenance_output( + 903, + target, + &sum_kind(), + &output, + &sum, + &cohort, + [42; 32] + ) .is_err()); assert!(store .recover_frozen_maintenance_output(902, target, &cohort, [42; 32], (0, 1000)) @@ -1204,7 +1269,15 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(700, source, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id( + 700, + source, + &sum_kind(), + &output, + &sum, + ) + }, ) .unwrap(); } @@ -1409,7 +1482,9 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(702, source, &output, &sum), + |writer| { + writer.ingest_precompute_with_series_id(702, source, &sum_kind(), &output, &sum) + }, ) .unwrap(); let deadline = @@ -1479,21 +1554,15 @@ mod tests { let plan = crate::tests::test_utilities::planning::quoted_snapshot(snapshot, false) .compile_promql() .unwrap(); - let mut source_config = plan.precompute_plan.materializations[0].clone(); - source_config.aggregation_type = asap_types::AggregationType::Sum; - source_config.aggregation_sub_type = "sum".into(); + let source_config = plan.precompute_plan.materializations[0].clone(); let source_id = source_config.policy_fingerprint().into(); let mut target = source_config.clone(); target.derived_input = Some(asap_types::derived_input::DerivedInputIdentity { inputs: BTreeSet::from([source_id]), program_sha256: "0".repeat(64), }); - let catalog = asap_types::summary_catalog::SummaryCatalog::from_materializations( - 1, - 1, - &[source_config.clone(), target.clone()], - ) - .unwrap(); + target.allocate_stored_output_id(&"sum"); + let catalog = sum_catalog(1, &[source_config.clone(), target.clone()]); let store = Arc::new(SketchStore::new()); store.install_summary_catalog(Arc::new(catalog)).unwrap(); let directory = tempfile::tempdir().unwrap(); @@ -1510,10 +1579,10 @@ mod tests { let mut source_output = output.clone(); source_output.policy_fp = source_config.policy_fingerprint(); store - .register_precompute_output(600, &source_config, &source_output) + .register_precompute_output(600, &source_config, &sum_kind(), &source_output) .unwrap(); store - .register_precompute_output(601, &target, &output) + .register_precompute_output(601, &target, &sum_kind(), &output) .unwrap(); let record = store .metadata_record(&store.instances.read().unwrap()[&601]) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index bb47a012d..d86f35714 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -40,8 +40,8 @@ use crate::storage_engines::sketch_db::sds::{ }; pub use crate::storage_engines::sketch_db::data::{ - canonical_parameters, AccuracyBound, AggKind, AggPayload, AggregationType, Capability, - SketchAlgorithm, SketchConfig, SketchEncoding, SketchSampleState, SketchTimeSeries, + AccuracyBound, AggKind, AggPayload, AggregationType, Capability, SketchAlgorithm, SketchConfig, + SketchEncoding, SketchSampleState, SketchTimeSeries, }; fn now_ms() -> u64 { @@ -764,6 +764,7 @@ impl SummaryPublicationWriter<'_> { address: &asap_types::sds::StoredSummaryKey, resolver: &crate::drivers::ingest::series_resolver::SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -813,7 +814,7 @@ impl SummaryPublicationWriter<'_> { return None; } self.0 - .ingest_precompute_with_admission(handle, config, output, state) + .ingest_precompute_with_admission(handle, config, kind, output, state) } #[cfg(test)] @@ -821,11 +822,12 @@ impl SummaryPublicationWriter<'_> { &self, sid: u64, config: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, state: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { self.0 - .ingest_precompute_with_admission(sid, config, output, state) + .ingest_precompute_with_admission(sid, config, kind, output, state) } #[cfg(test)] @@ -3180,6 +3182,7 @@ impl SketchStore { &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3187,7 +3190,7 @@ impl SketchStore { if admission.is_finite_closed() { return None; } - self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, output, accumulator) + self.ingest_precompute_config_with_admission(mint_sid, agg_cfg, kind, output, accumulator) } #[cfg(test)] @@ -3195,6 +3198,7 @@ impl SketchStore { &self, mint_sid: impl FnOnce(&str, &str, &str) -> R, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3206,31 +3210,33 @@ impl SketchStore { // invoking the sid-direct sibling. let (attrs_fp, _label_values_map) = build_attrs_fp_and_label_map(agg_cfg, output).ok()?; let agg_kind_canonical = - crate::storage_engines::sketch_db::data::materialization_kind_for_config(agg_cfg); + crate::storage_engines::sketch_db::data::materialization_kind(agg_cfg, kind); let sid = mint_sid(&agg_cfg.metric, &attrs_fp, &agg_kind_canonical).into()?; - self.ingest_precompute_with_admission(sid, agg_cfg, output, accumulator) + self.ingest_precompute_with_admission(sid, agg_cfg, kind, output, accumulator) } fn register_precompute_output( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, ) -> Option> { let mut instances = self.instances.write().ok()?; - self.register_precompute_output_with_instances(sid, agg_cfg, output, &mut instances) + self.register_precompute_output_with_instances(sid, agg_cfg, kind, output, &mut instances) } fn register_precompute_output_with_instances( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, instances: &mut HashMap, ) -> Option> { let (_attrs_fp, label_values_map) = build_attrs_fp_and_label_map(agg_cfg, output).ok()?; let key_names = &agg_cfg.grouping_labels.names(); - let agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg); + let agg_kind = kind.clone(); let (capability, accuracy) = agg_kind.capability_and_accuracy(); match instances.get(&sid) { @@ -3314,6 +3320,7 @@ impl SketchStore { &self, sid: u64, agg_cfg: &asap_types::aggregation_config::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3321,13 +3328,14 @@ impl SketchStore { if admission.is_finite_closed() { return None; } - self.ingest_precompute_with_admission(sid, agg_cfg, output, accumulator) + self.ingest_precompute_with_admission(sid, agg_cfg, kind, output, accumulator) } fn ingest_precompute_with_admission( &self, sid: u64, agg_cfg: &asap_types::PrecomputeMaterialization, + kind: &AggKind, output: &crate::storage_engines::types::PrecomputedOutput, accumulator: &dyn crate::storage_engines::types::AggregateCore, ) -> Option { @@ -3335,19 +3343,14 @@ impl SketchStore { tracing::warn!(sid, %error, "rejecting summary state without a stored codec"); return None; } - let expected = agg_cfg.accumulator_spec().ok()?.family; - if matches!( - expected, - planner_types::post_asap::SummaryFamilyType::ExactAggregate(..) - ) && accumulator - .get_accumulator_type() - .planner_exact_family() - .as_ref() - != Some(&expected) - { - return None; + if let AggKind::ExactAgg { agg_type, .. } = kind { + if accumulator.get_accumulator_type().planner_exact_family() + != agg_type.planner_exact_family() + { + return None; + } } - let label_values_map = self.register_precompute_output(sid, agg_cfg, output)?; + let label_values_map = self.register_precompute_output(sid, agg_cfg, kind, output)?; // Keep the physical lifetime alive through publication. Removal takes // this same lock exclusively, so it cannot race metadata validation and @@ -3398,7 +3401,7 @@ impl SketchStore { } let window = (output.start_timestamp, output.end_timestamp); - let accepted = match crate::storage_engines::sketch_db::data::agg_kind_for_config(agg_cfg) { + let accepted = match kind { AggKind::Sketch { .. } => self.append_sample_with_binding( sid, label_values_map, @@ -3933,9 +3936,13 @@ mod tests { fn meta_for_config( sid: u64, config: &asap_types::PrecomputeMaterialization, + plan: &asap_types::precompute_plan::PrecomputePlan, ) -> SummarySeriesMetadata { let mut metadata = meta_with_policy(sid, config.policy_fingerprint()); - metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_config(config); + metadata.agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_family( + plan.state_family(config.stored_output_id).unwrap(), + &plan.population_filter(config).unwrap(), + ); metadata } @@ -4049,7 +4056,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -4060,7 +4070,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(41, &state_config)); + store.register(meta_for_config(41, &state_config, &plan.precompute_plan)); store.append_sample( 41, BTreeMap::from([("job".to_string(), "api".to_string())]), @@ -4143,7 +4153,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -4154,7 +4167,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog)) .unwrap(); - store.register(meta_for_config(42, &state_config)); + store.register(meta_for_config(42, &state_config, &plan.precompute_plan)); let producers = BTreeMap::from([( StoredOutputId::from(fingerprint), ( @@ -4234,7 +4247,7 @@ mod tests { identity_version: 1, plan_id: 7, plan_version: 2, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), materialization: PolicyFingerprint(41).into(), series_identity: "service=checkout,zone=a".into(), schema_id: "schema-41".into(), @@ -4628,16 +4641,6 @@ mod tests { // sids by construction. #[test] - fn canonical_parameters_is_insertion_order_independent() { - let mut p_ab = std::collections::HashMap::new(); - p_ab.insert("alpha".to_string(), serde_json::json!(1)); - p_ab.insert("beta".to_string(), serde_json::json!(2)); - let mut p_ba = std::collections::HashMap::new(); - p_ba.insert("beta".to_string(), serde_json::json!(2)); - p_ba.insert("alpha".to_string(), serde_json::json!(1)); - assert_eq!(canonical_parameters(&p_ab), canonical_parameters(&p_ba)); - } - // Unkeyed Planner exact MIN panes feed the MIN rollup. #[test] fn planner_exact_min_panes_feed_the_min_rollup() { @@ -5349,14 +5352,17 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) .unwrap() .clone(); let fingerprint = state_config.policy_fingerprint(); - let metadata = meta_for_config(507, &state_config); + let metadata = meta_for_config(507, &state_config, &plan.precompute_plan); let record = StoredOutputMetadataRecord::new( metadata.storage_handle, metadata.metric_name.clone(), @@ -5403,7 +5409,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5414,7 +5423,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(509, &state_config)); + store.register(meta_for_config(509, &state_config, &plan.precompute_plan)); store.append_sample(509, BTreeMap::new(), (0, 10_000), sample(1)); let mut next = plan.summary_catalog; next.plan_version += 1; @@ -5445,13 +5454,13 @@ mod tests { ) .unwrap(); assert!(inventory.instances.is_empty()); - store.register(meta_for_config(510, &state_config)); + store.register(meta_for_config(510, &state_config, &plan.precompute_plan)); store.append_sample(510, BTreeMap::new(), (0, 10_000), sample(2)); assert_eq!(store.series_ids_for_policy(fingerprint), vec![510]); - let incompatible = asap_types::summary_catalog::SummaryCatalog::from_materializations( + let incompatible = asap_types::summary_catalog::SummaryCatalog::from_outputs( plan.precompute_plan.envelope.plan_id, plan.precompute_plan.envelope.plan_version + 2, - &[], + Vec::new(), ) .unwrap(); store @@ -5477,7 +5486,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5500,7 +5512,11 @@ mod tests { .unwrap(); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); old_sid = resolver.resolve("metric", "group", "family"); - store.register(meta_for_config(old_sid, &state_config)); + store.register(meta_for_config( + old_sid, + &state_config, + &plan.precompute_plan, + )); for pane in 0..4 { store.append_sample( old_sid, @@ -5527,7 +5543,11 @@ mod tests { }) .unwrap(); assert_ne!(new_sid, old_sid); - store.register(meta_for_config(new_sid, &state_config)); + store.register(meta_for_config( + new_sid, + &state_config, + &plan.precompute_plan, + )); for pane in 0..4 { store.append_sample( new_sid, @@ -5602,7 +5622,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5614,7 +5637,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(850, &state_config)); + store.register(meta_for_config(850, &state_config, &plan.precompute_plan)); let generation = store.active_catalog_generation().unwrap(); let writer = Arc::new(persistence::metadata::StoredOutputMetadataFile::new( directory.path(), @@ -5711,7 +5734,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5724,7 +5750,7 @@ mod tests { store .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); - store.register(meta_for_config(851, &state_config)); + store.register(meta_for_config(851, &state_config, &plan.precompute_plan)); let mut config = durable_cfg(directory.path().to_path_buf()); config.hot_window_ms = None; config.seal_window_count = 100; @@ -5833,7 +5859,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5849,7 +5878,7 @@ mod tests { .install_summary_catalog(Arc::new(plan.summary_catalog.clone())) .unwrap(); for sid in [801, 802, 803] { - store.register(meta_for_config(sid, &state_config)); + store.register(meta_for_config(sid, &state_config, &plan.precompute_plan)); } let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for sid in [801, 802, 803] { @@ -5876,7 +5905,7 @@ mod tests { expected_retirement = store.force_retire(801, Duration::from_secs(3600)).unwrap(); assert!(store.force_expire(802).is_some()); assert!(store.remove_instance(803).is_some()); - store.register(meta_for_config(803, &state_config)); + store.register(meta_for_config(803, &state_config, &plan.precompute_plan)); assert!( store.instance(803).is_none(), "removed SID reused before restart" @@ -5906,7 +5935,7 @@ mod tests { recovered.instance(803).is_none(), "removed state resurrected" ); - recovered.register(meta_for_config(803, &state_config)); + recovered.register(meta_for_config(803, &state_config, &plan.precompute_plan)); assert!( recovered.instance(803).is_none(), "removed SID reused after restart" @@ -5929,7 +5958,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -5946,7 +5978,7 @@ mod tests { )]); let tmp = tempfile::TempDir::new().unwrap(); let disk = tmp.path().to_path_buf(); - let mut metadata = meta_for_config(506, &state_config); + let mut metadata = meta_for_config(506, &state_config, &plan.precompute_plan); metadata.group_by_keys = ["job".to_string()].into_iter().collect(); { @@ -6715,7 +6747,10 @@ mod tests { .iter() .find(|config| { matches!( - config.accumulator_spec().unwrap().family, + plan.precompute_plan + .state_family(config.stored_output_id) + .unwrap() + .clone(), planner_types::post_asap::SummaryFamilyType::Sketch(..) ) }) @@ -6756,7 +6791,7 @@ mod tests { ) .unwrap(); assert_ne!(handle, other_population); - store.register(meta_for_config(handle, &config)); + store.register(meta_for_config(handle, &config, &plan.precompute_plan)); let mut persistence = store.start_persistence(durable_cfg(disk.clone())).unwrap(); for pane in 0..4 { store.append_sample( @@ -6843,6 +6878,7 @@ mod tests { .iter() .find(|config| config.policy_fingerprint() == output.stored_output_id.fingerprint()) .unwrap(), + &plan.precompute_plan, )); assert_eq!(store.storage_handles_for_output(&output), vec![100]); let mut next = plan.summary_catalog; @@ -6942,31 +6978,38 @@ impl SketchStore { output.population_labels = Some(record.group.clone()); output.catalog_generation = Some(Arc::clone(&generation)); output.stored_output_reference = Some(record.reference.clone()); - let state = crate::precompute_engine::revisions::decode_state(record, config)?; + let family = plan + .precompute_plan + .state_family(definition) + .ok_or("revision view lacks output state schema")?; + let kind = plan + .installed_precompute_plan + .agg_kind(definition) + .ok_or("revision view lacks output state schema")?; + let state = crate::precompute_engine::revisions::decode_state(record, family)?; let labels = view - .register_precompute_output(sid, config, &output) + .register_precompute_output(sid, config, &kind, &output) .ok_or("revision view rejected output metadata")?; // This is an immutable read view of already committed revisions, // not additive producer admission (which correctly rejects derived // writes). Every payload was decoded against its installed family. - let accepted = - match crate::storage_engines::sketch_db::data::agg_kind_for_config(config) { - AggKind::Sketch { .. } => view.append_sample_with_binding( - sid, - labels, - (record.start_ms, record.end_ms), - SketchSampleState { - bytes: state.serialize_to_bytes(), - encoding: SketchEncoding::full_frame_for(state.as_ref()), - }, - ), - AggKind::ExactAgg { .. } => view.append_precompute_with_binding( - sid, - labels, - (record.start_ms, record.end_ms), - state.clone_boxed_core(), - ), - }; + let accepted = match kind { + AggKind::Sketch { .. } => view.append_sample_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + SketchSampleState { + bytes: state.serialize_to_bytes(), + encoding: SketchEncoding::full_frame_for(state.as_ref()), + }, + ), + AggKind::ExactAgg { .. } => view.append_precompute_with_binding( + sid, + labels, + (record.start_ms, record.end_ms), + state.clone_boxed_core(), + ), + }; if !accepted { return Err("revision view rejected committed payload".into()); } diff --git a/data_plane/src/storage_engines/sketch_db/index/native.rs b/data_plane/src/storage_engines/sketch_db/index/native.rs index 636be0961..bbf70853b 100644 --- a/data_plane/src/storage_engines/sketch_db/index/native.rs +++ b/data_plane/src/storage_engines/sketch_db/index/native.rs @@ -39,6 +39,7 @@ impl SketchStore { &self, resolver: &SeriesIdResolver, config: &asap_types::PrecomputeMaterialization, + family: &planner_types::post_asap::SummaryFamilyType, output: &crate::storage_engines::types::PrecomputedOutput, batch: Batch, max_bytes: usize, @@ -57,7 +58,6 @@ impl SketchStore { self.read_complete_raw_maintenance_cohort(generation, &program.inputs, window)?; let (population_key, group) = build_attrs_fp_and_label_map(config, output)?; let state = NativeSummaryOutput::new(batch, max_bytes)?; - let family = config.accumulator_spec().map_err(|e| e.to_string())?.family; if state .batch() .schema() @@ -69,13 +69,13 @@ impl SketchStore { planner_types::post_asap::SummaryFamilyType::Plain(_) ) }) - .any(|field| field.dtype != family) + .any(|field| &field.dtype != family) { return Err("native output schema differs from installed definition".into()); } for value in state.batch().rows().iter().flatten() { if let Value::Summary { family: actual, .. } = value { - if actual != &family { + if actual != family { return Err("native output family differs from installed definition".into()); } } @@ -104,6 +104,8 @@ impl SketchStore { self.publish_complete_raw_maintenance_output( sid, config, + // Derived outputs read stored states, so their input has no predicate. + &crate::storage_engines::sketch_db::data::agg_kind_for_family(family, ""), output, &state, &cohort, @@ -378,6 +380,14 @@ mod tests { .iter() .find(|c| c.derived_input.is_some()) .unwrap(); + let source_family = plan + .precompute_plan + .state_family(source.stored_output_id) + .unwrap(); + let target_family = plan + .precompute_plan + .state_family(target.stored_output_id) + .unwrap(); let directory = tempfile::tempdir().unwrap(); let resolver_path = directory.path().join("resolver.jsonl"); let resolver = SeriesIdResolver::open(resolver_path.clone()).unwrap(); @@ -421,7 +431,18 @@ mod tests { revision, revision, 120_000, - |writer| writer.ingest_precompute_with_series_id(700, source, &output, &state), + |writer| { + writer.ingest_precompute_with_series_id( + 700, + source, + &crate::storage_engines::sketch_db::data::agg_kind_for_family( + source_family, + "", + ), + &output, + &state, + ) + }, ) .unwrap(); } @@ -456,14 +477,8 @@ mod tests { time_index: None, }); let input = Batch::try_new(raw_schema.clone(), values).unwrap(); - let build = Operator::summary_build( - raw_schema, - target.accumulator_spec().unwrap().family, - 0, - None, - vec![], - ) - .unwrap(); + let build = + Operator::summary_build(raw_schema, target_family.clone(), 0, None, vec![]).unwrap(); let output_batch = run(input, build); let expected_schema = output_batch.schema().clone(); let mut output = PrecomputedOutput::new(0, 60_000, None, target.policy_fingerprint()); @@ -473,13 +488,21 @@ mod tests { .publish_native_summary_output( &resolver, target, + target_family, &output, output_batch.clone(), 1 << 20 ) .unwrap()); store - .publish_native_summary_output(&resolver, target, &output, output_batch, 1 << 20) + .publish_native_summary_output( + &resolver, + target, + target_family, + &output, + output_batch, + 1 << 20, + ) .unwrap(); assert_eq!(persistence.manifest.live_parts().len(), before + 1); let reference = plan diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs index 93b0ed5c8..176086452 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/eviction.rs @@ -199,35 +199,24 @@ mod tests { use asap_types::KeyByLabelNames; use std::collections::HashMap; - fn sum_agg_config(id: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: AggregationType::Sum, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::empty().into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: format!("metric_{id}"), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - } + fn sum_agg_config( + id: u64, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + let mut config = PrecomputeMaterialization::new( + format!("metric_{id}"), + KeyByLabelNames::empty(), + 1, + 1, + WindowKind::Tumbling, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + crate::tests::test_utilities::outputs::allocated( + config, + AggregationType::Sum.planner_exact_family().unwrap(), + ) } /// Build a streaming-config keyed on the policy-fingerprint u64 @@ -235,18 +224,14 @@ mod tests { /// marker_id → fingerprint mapping so callers can look up the /// right key. fn make_streaming_config(ids: &[u64]) -> (Arc, HashMap) { - let mut map = HashMap::new(); + let mut outputs = Vec::new(); let mut id_to_fp = HashMap::new(); for &id in ids { let cfg = sum_agg_config(id); - let fp = cfg.policy_fp_u64(); - id_to_fp.insert(id, fp); - map.insert(fp, cfg); + id_to_fp.insert(id, cfg.0.policy_fp_u64()); + outputs.push(cfg); } - ( - Arc::new(InstalledPrecomputePlan::from_raw_ids(map)), - id_to_fp, - ) + (Arc::new(InstalledPrecomputePlan::new(outputs)), id_to_fp) } fn write_one( @@ -280,6 +265,9 @@ mod tests { .ingest_precompute_for_agg_config( |m, fp, ak| resolver.resolve(m, fp, ak), agg_cfg, + &installed_precompute_plan + .agg_kind(agg_cfg.stored_output_id) + .unwrap(), &output, &acc, ) diff --git a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs index d6310b46f..e28518f81 100644 --- a/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs +++ b/data_plane/src/storage_engines/sketch_db/lifecycle/reconcile.rs @@ -13,7 +13,7 @@ use std::time::Duration; use crate::storage_engines::types::InstalledPrecomputePlan; use asap_types::aggregation_config::PrecomputeMaterialization; -use crate::storage_engines::sketch_db::data::{canonical_parameters, AggKind}; +use crate::storage_engines::sketch_db::data::AggKind; use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::sketch_db::lifecycle::AggStatus; @@ -166,21 +166,19 @@ fn signature_into( } } -fn signature_from_agg_config(cfg: &PrecomputeMaterialization) -> Vec { - let agg_kind = AggKind::ExactAgg { - agg_type: cfg.aggregation_type, - parameters_canonical: canonical_parameters(&cfg.parameters), - spatial_filter_canonical: cfg.spatial_filter_normalized.clone(), - }; +fn signature_from_agg_config(cfg: &PrecomputeMaterialization, agg_kind: &AggKind) -> Vec { let group_by_keys: BTreeSet = cfg.grouping_labels.iter().cloned().collect(); - signature_bytes(&cfg.metric, &agg_kind, &group_by_keys) + signature_bytes(&cfg.metric, agg_kind, &group_by_keys) } fn build_live_signature_set(config: &InstalledPrecomputePlan) -> HashSet> { config .materializations() .values() - .map(signature_from_agg_config) + .filter_map(|cfg| { + let kind = config.agg_kind(cfg.stored_output_id)?; + Some(signature_from_agg_config(cfg, &kind)) + }) .collect() } @@ -276,23 +274,15 @@ mod tests { metric: &str, agg_type: AggregationType, group_by: Vec<&str>, - ) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - HashMap::new(), - KeyByLabelNames::new(group_by.into_iter().map(|s| s.to_string()).collect()), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 1, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + crate::tests::test_utilities::outputs::output( + metric, + agg_type.planner_exact_family().unwrap(), + group_by, 1, - WindowKind::Tumbling, - String::new(), - metric.to_string(), - None, - None, - None, ) } @@ -321,12 +311,13 @@ mod tests { } } - fn streaming(configs: Vec) -> InstalledPrecomputePlan { - let mut map = HashMap::new(); - for (i, c) in configs.into_iter().enumerate() { - map.insert(i as u64 + 1, c); - } - InstalledPrecomputePlan::from_raw_ids(map) + fn streaming( + configs: Vec<( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + )>, + ) -> InstalledPrecomputePlan { + InstalledPrecomputePlan::new(configs) } #[test] diff --git a/data_plane/src/storage_engines/sketch_db/sds.rs b/data_plane/src/storage_engines/sketch_db/sds.rs index 58ea8eb62..3ffe43dcb 100644 --- a/data_plane/src/storage_engines/sketch_db/sds.rs +++ b/data_plane/src/storage_engines/sketch_db/sds.rs @@ -78,30 +78,8 @@ fn legacy_summary(kind: &AggKind) -> SummaryDescriptor { // A valid deployment reference does not authorize bytes in another state format. fn operator_matches(summary: &SummaryDescriptor, actual: &AggKind) -> bool { match &summary.operator { - SummaryOperator::Configured { - aggregation_type, - aggregation_sub_type, - parameters, - .. - } => { - let config = asap_types::PrecomputeMaterialization::new( - *aggregation_type, - aggregation_sub_type.clone(), - parameters.clone().into_iter().collect(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), - 1, - 1, - asap_types::WindowKind::Tumbling, - String::new(), - String::new(), - None, - None, - None, - ); - super::data::agg_kind_for_config(&config).operator_canonical_string() + SummaryOperator::Configured { family } => { + super::data::agg_kind_for_family(family, "").operator_canonical_string() == actual.operator_canonical_string() } SummaryOperator::ExactAgg { diff --git a/data_plane/src/storage_engines/types/hot_reload_config.rs b/data_plane/src/storage_engines/types/hot_reload_config.rs index b624d8164..c66e1b6b5 100644 --- a/data_plane/src/storage_engines/types/hot_reload_config.rs +++ b/data_plane/src/storage_engines/types/hot_reload_config.rs @@ -702,7 +702,7 @@ mod tests { generated_at_unix_ms: activation_unix_ms, activation_unix_ms, expiry_unix_ms, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; @@ -735,7 +735,7 @@ mod tests { generated_at_unix_ms: activation_unix_ms, activation_unix_ms, expiry_unix_ms, - backend_compat: "asap-query-backend.v1".into(), + backend_compat: "asap-query-backend.v2".into(), planner_revision: control_plane::physical::compiler::PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }, @@ -830,23 +830,17 @@ mod tests { ); } - fn dummy_agg(id: u64) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - AggregationType::Sum, - String::new(), - HashMap::new(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), + fn dummy_agg( + id: u64, + ) -> ( + PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ) { + crate::tests::test_utilities::outputs::output( + &format!("metric_{id}"), + AggregationType::Sum.planner_exact_family().unwrap(), + vec![], 60, - 60, - WindowKind::Tumbling, - String::new(), - format!("metric_{id}"), - None, - None, - None, ) } @@ -856,15 +850,14 @@ mod tests { /// dummy-id→fingerprint mapping so the assertions below can look /// up entries. fn cfg_with_ids(ids: &[u64]) -> (InstalledPrecomputePlan, std::collections::HashMap) { - let mut map = HashMap::new(); + let mut outputs = Vec::new(); let mut id_to_fp = std::collections::HashMap::new(); for &id in ids { let cfg = dummy_agg(id); - let fp = cfg.policy_fp_u64(); - id_to_fp.insert(id, fp); - map.insert(fp, cfg); + id_to_fp.insert(id, cfg.0.policy_fp_u64()); + outputs.push(cfg); } - (InstalledPrecomputePlan::from_raw_ids(map), id_to_fp) + (InstalledPrecomputePlan::new(outputs), id_to_fp) } #[test] diff --git a/data_plane/src/storage_engines/types/installed_precompute_plan.rs b/data_plane/src/storage_engines/types/installed_precompute_plan.rs index 1ec253024..e42c553d2 100644 --- a/data_plane/src/storage_engines/types/installed_precompute_plan.rs +++ b/data_plane/src/storage_engines/types/installed_precompute_plan.rs @@ -5,7 +5,8 @@ use std::ops::Index; use super::storage_backend::StorageBackend; use asap_types::sds::StoredOutputId; -use asap_types::{PolicyRegistry, PrecomputeMaterialization}; +use asap_types::PrecomputeMaterialization; +use planner_types::post_asap::SummaryFamilyType; #[derive(Debug, Clone)] pub struct InstalledPrecomputePlan { @@ -14,16 +15,33 @@ pub struct InstalledPrecomputePlan { HashMap>, pub(crate) precompute_plan: Option, pub(crate) materializations_by_output: HashMap, + /// Each output's stored state family, from its schema contract. + state_families: HashMap, + /// Each output's canonical input predicate, from its Planner DAG scan. + population_filters: HashMap, + /// The per-item dimension of outputs whose Planner update keys items by a label. + item_labels: HashMap, pub(crate) storage_backend: StorageBackend, } impl InstalledPrecomputePlan { - fn derived_view(materializations: HashMap) -> Self { + fn derived_view( + outputs: impl IntoIterator, + ) -> Self { + let mut materializations = HashMap::new(); + let mut state_families = HashMap::new(); + for (config, family) in outputs { + state_families.insert(config.stored_output_id, family); + materializations.insert(config.stored_output_id, config); + } Self { partitioning: Default::default(), raw_programs: HashMap::new(), precompute_plan: None, materializations_by_output: materializations, + state_families, + population_filters: HashMap::new(), + item_labels: HashMap::new(), storage_backend: StorageBackend::default(), } } @@ -31,6 +49,53 @@ impl InstalledPrecomputePlan { /// Production construction always validates the executable DAG and bindings. pub fn from_precompute_plan(plan: asap_types::precompute_plan::PrecomputePlan) -> Result { let materializations = plan.runtime_materializations()?; + let outputs = materializations + .into_values() + .map(|config| { + let family = plan + .state_family(config.stored_output_id) + .cloned() + .ok_or_else(|| anyhow::anyhow!("stored output has no state schema"))?; + Ok((config, family)) + }) + .collect::>>()?; + let population_filters = outputs + .iter() + .map(|(config, _)| { + plan.population_filter(config) + .map(|filter| (config.stored_output_id, filter)) + .map_err(anyhow::Error::msg) + }) + .collect::>>()?; + let mut item_labels = HashMap::new(); + for (config, _) in &outputs { + use planner_types::post_asap::{PostAsapOperatorPayload, SummaryInputExpr}; + use planner_types::pre_asap::ColumnRef; + let Some((node, _)) = plan + .summary_producer(config.stored_output_id) + .map_err(anyhow::Error::msg)? + else { + continue; + }; + if let PostAsapOperatorPayload::SummaryAgg { + input: + planner_types::post_asap::SummaryUpdate { + item: + Some(SummaryInputExpr::Column( + ColumnRef::Named(label) | ColumnRef::Qualified { name: label, .. }, + )), + .. + }, + .. + } = node.payload + { + item_labels.insert(config.stored_output_id, label); + } + } + let materializations: HashMap<_, _> = outputs + .iter() + .map(|(config, _)| (config.stored_output_id, config.clone())) + .collect(); let mut programs = HashMap::new(); for config in materializations.values().filter(|config| { config.derived_input.is_none() @@ -42,7 +107,9 @@ impl InstalledPrecomputePlan { .map_err(anyhow::Error::msg)?; programs.insert(config.policy_fp_u64(), std::sync::Arc::new(program)); } - let mut view = Self::derived_view(materializations); + let mut view = Self::derived_view(outputs); + view.population_filters = population_filters; + view.item_labels = item_labels; view.partitioning = crate::precompute_engine::partitioning::DagPartitioning::from_plan(&plan); view.precompute_plan = Some(plan); @@ -52,30 +119,20 @@ impl InstalledPrecomputePlan { // Isolated kernel/storage fixtures can omit a physical installation. This // constructor is absent from the production library and binary. + /// Each output is paired with its stored state family. #[cfg(test)] - pub fn new(materializations: HashMap) -> Self { - Self::derived_view(materializations) - } - - /// Fixtures hold raw output ids, so say that once here rather than - /// wrapping every literal. Production construction goes through - /// `from_precompute_plan`, which takes the ids from the plan itself. - #[cfg(test)] - pub fn from_raw_ids(materializations: HashMap) -> Self { - Self::new( - materializations - .into_iter() - .map(|(id, value)| (StoredOutputId(id), value)) - .collect(), - ) + pub fn new( + outputs: impl IntoIterator, + ) -> Self { + Self::derived_view(outputs) } #[cfg(test)] pub fn with_storage_backend( - materializations: HashMap, + outputs: impl IntoIterator, storage_backend: StorageBackend, ) -> Self { - let mut view = Self::derived_view(materializations); + let mut view = Self::derived_view(outputs); view.storage_backend = storage_backend; view } @@ -92,19 +149,21 @@ impl InstalledPrecomputePlan { }); #[cfg(test)] let selected = selected.or_else(|| { - let materializations = self + let outputs = self .materializations_by_output .values() - .cloned() - .collect::>(); - asap_types::summary_catalog::SummaryCatalog::from_materializations( - 0, - 0, - &materializations, - ) - .ok()? - .output_reference(definition) - .ok() + .map(|config| { + Some(( + config, + self.state_family(config.stored_output_id)?, + self.population_filter(config.stored_output_id).to_owned(), + )) + }) + .collect::>>()?; + asap_types::summary_catalog::SummaryCatalog::from_outputs(0, 0, outputs) + .ok()? + .output_reference(definition) + .ok() }); selected } @@ -134,8 +193,41 @@ impl InstalledPrecomputePlan { self.materializations_by_output.contains_key(&output) } - pub fn policy_registry(&self) -> PolicyRegistry { - PolicyRegistry::from_configs(self.materializations_by_output.values().cloned()) + /// The stored state family of `output`. + pub fn state_family(&self, output: StoredOutputId) -> Option<&SummaryFamilyType> { + self.state_families.get(&output) + } + + /// The physical stored-state kind of `output`. + pub fn agg_kind( + &self, + output: StoredOutputId, + ) -> Option { + self.state_family(output).map(|family| { + crate::storage_engines::sketch_db::data::agg_kind_for_family( + family, + self.population_filter(output), + ) + }) + } + + /// Whether `output`'s panes follow PromQL's `(start, end]` range + /// convention. Raw Planner programs read only time-series scans, whose + /// ranges are PromQL ranges. + pub fn right_closed_panes(&self, output: StoredOutputId) -> bool { + self.raw_programs.contains_key(&output.as_u64()) + } + + /// The label that keys `output`'s items, when its Planner update has one. + pub fn item_label(&self, output: StoredOutputId) -> Option<&str> { + self.item_labels.get(&output).map(String::as_str) + } + + /// The canonical input predicate of `output`; empty when unfiltered. + pub fn population_filter(&self, output: StoredOutputId) -> &str { + self.population_filters + .get(&output) + .map_or("", String::as_str) } } @@ -170,6 +262,40 @@ impl Default for InstalledPrecomputePlan { #[cfg(test)] mod tests { + // A raw output's input predicate, pane convention and stored state kind + // come from its installed Planner producer, not from the materialization. + #[test] + fn raw_output_computation_comes_from_its_planner_producer() { + let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + snapshot["query_workload"]["repeating_queries"][0]["query"] = + serde_json::json!("quantile_over_time(0.99, m{job=\"api\"}[1m])"); + let plan = crate::tests::test_utilities::planning::quoted_snapshot( + serde_json::from_value(snapshot).unwrap(), + false, + ) + .compile_promql() + .unwrap(); + let installed = + super::InstalledPrecomputePlan::from_precompute_plan(plan.precompute_plan.clone()) + .unwrap(); + let output = plan.precompute_plan.materializations[0].stored_output_id; + assert_eq!(installed.population_filter(output), "{job=\"api\"}"); + assert!(installed.right_closed_panes(output)); + let family = installed.state_family(output).unwrap(); + assert_eq!(Some(family), plan.precompute_plan.state_family(output)); + assert_eq!( + installed.agg_kind(output).unwrap().canonical_string(), + crate::storage_engines::sketch_db::data::agg_kind_for_family(family, "{job=\"api\"}") + .canonical_string() + ); + let data = &plan.summary_catalog.data_descriptors + [&plan.summary_catalog.outputs[&output].data_descriptor_id]; + assert_eq!(data.population_filter_canonical, "{job=\"api\"}"); + } + // Flat lists cannot enter through the authoritative physical-plan document. #[test] fn rejects_flat_aggregation_documents() { diff --git a/data_plane/src/tests/accumulator_fixture.rs b/data_plane/src/tests/accumulator_fixture.rs index 4df1d3018..4c8866a9d 100644 --- a/data_plane/src/tests/accumulator_fixture.rs +++ b/data_plane/src/tests/accumulator_fixture.rs @@ -1,34 +1,40 @@ -//! Config fixtures for backend integration tests; production binds Planner payloads. +//! Kernel fixtures for backend unit tests: a stored state family and the +//! Planner update it ingests. Production executes the bound Planner DAG. use asap_physical_operators::factory::*; use asap_physical_operators::summary_kernels::exact::ExactAccumulator; -use asap_summary_state::{AggregateCore, KeyByLabelValues}; -use asap_types::{accumulator_spec::cms_params, PrecomputeMaterialization}; -use planner_types::post_asap::{ExactKind, SketchAlgorithm, SketchParams, SummaryFamilyType}; -#[cfg(test)] -/// Return `true` if `config` produces a keyed (MultipleSubpopulation) updater, -/// without allocating an updater object. +use asap_summary_state::{AggregateCore, AggregationType, KeyByLabelValues}; +use planner_types::post_asap::{ + ExactKind, GroupingStrategy, HydraParams, SketchAlgorithm, SketchParams, SummaryFamilyType, + SummaryInputExpr, SummaryUpdate, +}; + +/// Whether `family` stores a keyed (multi-population) accumulator. /// /// **Contract:** this must agree with every concrete `AccumulatorUpdater::is_keyed()` /// implementation. When a new accumulator type is added, update both here and /// in the corresponding struct. -pub fn config_is_keyed(config: &PrecomputeMaterialization) -> bool { - config - .accumulator_spec() - .expect("valid fixture") - .grouping - .is_some() +#[cfg(test)] +pub fn family_is_keyed(family: &SummaryFamilyType) -> bool { + match family { + SummaryFamilyType::Sketch(_, GroupingStrategy::SharedMultiSubpopulation { .. }) => true, + SummaryFamilyType::Sketch(kind, _) => matches!( + kind.algorithm(), + SketchAlgorithm::Cms + | SketchAlgorithm::CmsWithHeap + | SketchAlgorithm::CountSketch + | SketchAlgorithm::CountSketchWithHeap + ), + _ => false, + } } -/// Top-k ranking quantity, selected by `weight_mode` or its alias `topk_weight`. -/// -/// * `value` / `sum`: sum values per key (default). -/// * `count` / `frequency` / `freq`: count occurrences per key. +/// Top-k ranking quantity: a unit weight counts occurrences per key, +/// otherwise values are summed. #[cfg(test)] -fn topk_weight_param(config: &PrecomputeMaterialization) -> TopkWeight { - match config.sample_update_rule() { - asap_types::SampleUpdateRule::Count => TopkWeight::Count, - asap_types::SampleUpdateRule::Value { .. } - | asap_types::SampleUpdateRule::CounterDelta { .. } => TopkWeight::Value, +fn topk_weight(update: &SummaryUpdate) -> TopkWeight { + match update.weight { + SummaryInputExpr::Constant(1.0) => TopkWeight::Count, + _ => TopkWeight::Value, } } @@ -101,17 +107,14 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { /// Production execution requires a validated Planner DAG program. #[cfg(test)] pub fn create_fixture_accumulator( - config: &PrecomputeMaterialization, + family: &SummaryFamilyType, + update: &SummaryUpdate, ) -> Box { - let spec = config - .accumulator_spec() - .expect("invalid isolated kernel fixture"); + let keyed = family_is_keyed(family); - let keyed = spec.grouping.is_some(); - - match (&spec.family, keyed) { + match (family, keyed) { (SummaryFamilyType::ExactAggregate(..), keyed) => Box::new(ExactUpdater { - acc: ExactAccumulator::new(spec.family.clone(), keyed).expect("exact fixture family"), + acc: ExactAccumulator::new(family.clone(), keyed).expect("exact fixture family"), }), (SummaryFamilyType::Sketch(kind, _), false) @@ -119,18 +122,20 @@ pub fn create_fixture_accumulator( { Box::new(KllAccumulatorUpdater::new(kll_k(kind.params()))) } - // HydraKLL: `k` comes off the typed params like the unkeyed case, - // but the `(row, col)` tiling grid has no `SketchParams::Kll` - // field to live in (see `asap_types::accumulator_spec`'s module - // doc) — read it the same way bare CMS does, via `cms_params`. - (SummaryFamilyType::Sketch(kind, _), true) if kind.algorithm() == &SketchAlgorithm::Kll => { - let (row_num, col_num) = cms_params(config); - Box::new(HydraKllAccumulatorUpdater::new( - row_num, - col_num, - kll_k(kind.params()), - )) - } + // HydraKLL: `k` comes off the typed params like the unkeyed case; + // the shared bucket count is the tiling width over four rows. + ( + SummaryFamilyType::Sketch( + kind, + GroupingStrategy::SharedMultiSubpopulation { + params: HydraParams::HydraKll { shared_buckets, .. }, + .. + }, + ), + true, + ) if kind.algorithm() == &SketchAlgorithm::Kll => Box::new( + HydraKllAccumulatorUpdater::new(4, *shared_buckets as usize, kll_k(kind.params())), + ), // Bare CMS: point-frequency only, min-of-rows estimator. `keyed=false` // can't actually arise here today (no `AggregationType` resolves to @@ -165,7 +170,7 @@ pub fn create_fixture_accumulator( row_num, col_num, heap_size, - topk_weight_param(config), + topk_weight(update), )) } @@ -178,7 +183,7 @@ pub fn create_fixture_accumulator( row_num, col_num, heap_size, - topk_weight_param(config), + topk_weight(update), )) } @@ -203,7 +208,7 @@ pub fn create_fixture_accumulator( unreachable!("validated UnivMon family parameters") }; create_planner_accumulator( - &spec.family, + family, &planner_types::post_asap::SummaryUpdate::column( planner_types::pre_asap::ColumnRef::SampleValue, ), @@ -219,7 +224,7 @@ pub fn create_fixture_accumulator( unreachable!("validated HLL family parameters") }; create_planner_accumulator( - &spec.family, + family, &planner_types::post_asap::SummaryUpdate::column( planner_types::pre_asap::ColumnRef::SampleValue, ), diff --git a/data_plane/src/tests/accuracy_empirical_validation_tests.rs b/data_plane/src/tests/accuracy_empirical_validation_tests.rs index 21cd2b115..a278e28a8 100644 --- a/data_plane/src/tests/accuracy_empirical_validation_tests.rs +++ b/data_plane/src/tests/accuracy_empirical_validation_tests.rs @@ -27,32 +27,17 @@ #[cfg(test)] use std::collections::HashMap; -use asap_types::aggregation_config::PrecomputeMaterialization; -use asap_types::enums::WindowKind; use asap_types::AggregationType; -use asap_types::KeyByLabelNames; use serde_json::{json, Value}; use crate::storage_engines::sketch_db::accuracy::{derive, AccuracyKind}; -fn cfg(agg_type: AggregationType, params: HashMap) -> PrecomputeMaterialization { - PrecomputeMaterialization::new( - agg_type, - String::new(), - params, - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 60, - 60, - WindowKind::Tumbling, - String::new(), - "m".to_string(), - None, - None, - None, - ) +/// The state family a kernel fixture with these dimensions stores. +fn cfg( + agg_type: AggregationType, + params: HashMap, +) -> planner_types::post_asap::SummaryFamilyType { + crate::tests::test_utilities::outputs::family(agg_type, &serde_json::to_value(params).unwrap()) } // -------- published-constant parity ---------- diff --git a/data_plane/src/tests/test_utilities/engine_factories.rs b/data_plane/src/tests/test_utilities/engine_factories.rs index 62a7621ec..720efc483 100644 --- a/data_plane/src/tests/test_utilities/engine_factories.rs +++ b/data_plane/src/tests/test_utilities/engine_factories.rs @@ -1,538 +1,158 @@ //! Engine factory helpers for integration tests //! //! Provides reusable construction helpers for ASAPQueryEngine + SketchStore -//! populated with various accumulator types. Unlike TestConfigBuilder which -//! hardcodes "SumAccumulator", these helpers build PrecomputeMaterialization with -//! the correct aggregation_type string. +//! populated with stored states of a given kernel family. use crate::drivers::ingest::series_resolver::SeriesIdResolver; use crate::query_engines::asap_query_engine::engine::ASAPQueryEngine; -use crate::query_engines::query_result::InstantVectorElement; +use crate::storage_engines::sketch_db::index::SketchStore; use crate::storage_engines::types::{ - AggregationType, InstalledPrecomputePlan, KeyByLabelValues, PrecomputeMaterialization, - PrecomputedOutput, QueryLanguage, WindowKind, + AggregationType, KeyByLabelValues, PrecomputeMaterialization, PrecomputedOutput, WindowKind, }; use crate::AggregateCore; -use asap_types::KeyByLabelNames; -use std::collections::HashMap; - -/// Helper for test factories — wraps the closure-mint call with a -/// fresh resolver and forwards to `SketchStore::ingest_precompute_for_agg_config`. -/// Each factory gets its own resolver instance; tests are isolated so -/// the `next_sid = 1, 2, ...` counter doesn't bleed between fixtures. -fn ingest_with_fresh_resolver( - summary_store: &crate::storage_engines::sketch_db::index::SketchStore, - resolver: &std::sync::Arc, - agg_cfg: &PrecomputeMaterialization, - output: &PrecomputedOutput, - accumulator: &dyn AggregateCore, -) -> Option { - let resolver = resolver.clone(); - summary_store.ingest_precompute_for_agg_config( - |m, fp, ak| resolver.resolve(m, fp, ak), - agg_cfg, - output, - accumulator, - ) -} use std::sync::Arc; /// Data to insert into a store: (label_values, accumulator) pub type AccumulatorData = Vec<(Option>, Box)>; -/// Creates a ASAPQueryEngine with a single aggregation populated with given data. -/// -/// # Arguments -/// * `metric` - Metric name -/// * `aggregation_type` - Accumulator type string (e.g. "SumAccumulator", "DatasketchesKLLAccumulator") -/// * `grouping_labels` - Label names for GROUP BY -/// * `data` - Vec of (label_values, accumulator) pairs to insert -/// * `promql_query` - The PromQL query string -pub fn create_engine_single_pop( +/// One stored output of `kind` state over `metric`, with its storage kind. +fn stored_output( metric: &str, - aggregation_type: AggregationType, + kind: AggregationType, grouping_labels: Vec<&str>, - data: AccumulatorData, - promql_query: &str, -) -> ASAPQueryEngine { - create_engine_single_pop_with_aggregated( + window_size: u64, + window_type: WindowKind, +) -> ( + PrecomputeMaterialization, + crate::storage_engines::sketch_db::index::AggKind, +) { + let family = super::outputs::family(kind, &serde_json::json!({})); + let mut config = PrecomputeMaterialization::new( metric, - aggregation_type, - grouping_labels, - vec![], - data, - promql_query, - ) + asap_types::KeyByLabelNames::new(grouping_labels.into_iter().map(str::to_owned).collect()), + window_size, + 1, + window_type, + ); + config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + config.allocate_stored_output_id(&family); + let agg_kind = crate::storage_engines::sketch_db::data::agg_kind_for_family(&family, ""); + (config, agg_kind) } -/// Creates a ASAPQueryEngine with aggregated labels (sub-key labels within the accumulator). -/// -/// Use for self-keyed multi-population accumulators (Multiple* types) where -/// `aggregated_labels` are the labels that key the accumulator internally -/// (e.g. "endpoint" within a MultipleIncrease accumulator). -pub fn create_engine_single_pop_with_aggregated( - metric: &str, - aggregation_type: AggregationType, - grouping_labels: Vec<&str>, - aggregated_labels: Vec<&str>, - data: AccumulatorData, - promql_query: &str, -) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - let aggregated_label_strings: Vec = - aggregated_labels.iter().map(|s| s.to_string()).collect(); - let all_schema_labels: Vec = grouping_label_strings - .iter() - .chain(aggregated_label_strings.iter()) - .cloned() - .collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - - // Insert data into SketchStore via the canonical helper (M2.3.6e). - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg config must be in installed_precompute_plan"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); +/// Ingest `(end_timestamp, labels, state)` rows of one output, each covering +/// `[end - span, end]`, through a fresh resolver. +fn ingest( + store: &SketchStore, + output: &( + PrecomputeMaterialization, + crate::storage_engines::sketch_db::index::AggKind, + ), + span: u64, + rows: impl IntoIterator>, Box)>, +) { + let resolver = Arc::new(SeriesIdResolver::new()); + let (config, kind) = output; + for (timestamp, labels, state) in rows { + let key = labels.map(|labels| KeyByLabelValues { labels }); let output = PrecomputedOutput::new( - timestamp, + timestamp - span, timestamp, key, - asap_types::PolicyFingerprint(agg_id), + config.policy_fingerprint(), + ); + store.ingest_precompute_for_agg_config( + |m, fp, ak| resolver.resolve(m, fp, ak), + config, + kind, + &output, + state.as_ref(), ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); } +} - ASAPQueryEngine::new(1).with_sketch_index(summary_store) +fn at_one_second(data: AccumulatorData) -> Vec<(u64, Option>, Box)> { + data.into_iter() + .map(|(labels, state)| (1_000_000, labels, state)) + .collect() } -/// Build a test engine with separate value and key aggregations. Each input -/// carries its own accumulator type, labels, and samples. -#[allow(clippy::too_many_arguments)] -#[allow(dead_code)] -pub fn create_engine_dual_input( +/// Creates a ASAPQueryEngine with a single aggregation populated with given data. +pub fn create_engine_single_pop( metric: &str, - value_agg_type: AggregationType, - key_agg_type: AggregationType, + aggregation_type: AggregationType, grouping_labels: Vec<&str>, - aggregated_labels: Vec<&str>, - value_data: AccumulatorData, - keys_data: AccumulatorData, - promql_query: &str, + data: AccumulatorData, + _promql_query: &str, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - let aggregated_label_strings: Vec = - aggregated_labels.iter().map(|s| s.to_string()).collect(); - let all_labels: Vec = grouping_label_strings - .iter() - .chain(aggregated_label_strings.iter()) - .cloned() - .collect(); - - let mut materializations_by_output = HashMap::new(); - - // Value aggregation - let value_agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: value_agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let value_id = value_agg_config.policy_fp_u64(); - materializations_by_output.insert(value_id, value_agg_config); - - // Keys aggregation - let keys_agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: key_agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::new(aggregated_label_strings), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let keys_id = keys_agg_config.policy_fp_u64(); - materializations_by_output.insert(keys_id, keys_agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - - let agg_cfg_1 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(value_id)) - .cloned() - .expect("value agg config"); - let agg_cfg_2 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(keys_id)) - .cloned() - .expect("keys agg config"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in value_data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(value_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); - } - for (label_values_opt, acc) in keys_data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(keys_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); - } - - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + let store = Arc::new(SketchStore::new()); + let output = stored_output( + metric, + aggregation_type, + grouping_labels, + 1, + WindowKind::Tumbling, + ); + ingest(&store, &output, 0, at_one_second(data)); + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a ASAPQueryEngine with two independent metrics, each with their own -/// aggregation config and query_config. -/// -/// agg_id=1 → metric_a, agg_id=2 → metric_b. -/// Both are registered as separate query_configs in the inference config. +/// stored output. #[allow(clippy::too_many_arguments)] pub fn create_engine_two_metrics( metric_a: &str, aggregation_type_a: AggregationType, grouping_labels_a: Vec<&str>, data_a: AccumulatorData, - query_a: &str, + _query_a: &str, metric_b: &str, aggregation_type_b: AggregationType, grouping_labels_b: Vec<&str>, data_b: AccumulatorData, - query_b: &str, + _query_b: &str, ) -> ASAPQueryEngine { - let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); - let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - - let agg_config_a = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: aggregation_type_a, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels_a.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric_a.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id_a = agg_config_a.policy_fp_u64(); - materializations_by_output.insert(id_a, agg_config_a); - - let agg_config_b = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: aggregation_type_b, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels_b.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric_b.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id_b = agg_config_b.policy_fp_u64(); - materializations_by_output.insert(id_b, agg_config_b); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg_1 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(id_a)) - .cloned() - .expect("agg a"); - let agg_cfg_2 = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(id_b)) - .cloned() - .expect("agg b"); - let timestamp = 1_000_000_u64; - for (label_values_opt, acc) in data_a { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(id_a), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_1, &output, acc.as_ref()); - } - for (label_values_opt, acc) in data_b { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(id_b), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg_2, &output, acc.as_ref()); + let store = Arc::new(SketchStore::new()); + for (metric, kind, labels, data) in [ + (metric_a, aggregation_type_a, grouping_labels_a, data_a), + (metric_b, aggregation_type_b, grouping_labels_b, data_b), + ] { + let output = stored_output(metric, kind, labels, 1, WindowKind::Tumbling); + ingest(&store, &output, 0, at_one_second(data)); } - let _ = (query_a, query_b); - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a ASAPQueryEngine with three independent metrics, each with their own -/// aggregation config and query_config. -/// -/// agg_id=1 → metric_a, agg_id=2 → metric_b, agg_id=3 → metric_c. +/// stored output. #[allow(clippy::too_many_arguments)] pub fn create_engine_three_metrics( metric_a: &str, aggregation_type_a: AggregationType, grouping_labels_a: Vec<&str>, data_a: AccumulatorData, - query_a: &str, + _query_a: &str, metric_b: &str, aggregation_type_b: AggregationType, grouping_labels_b: Vec<&str>, data_b: AccumulatorData, - query_b: &str, + _query_b: &str, metric_c: &str, aggregation_type_c: AggregationType, grouping_labels_c: Vec<&str>, data_c: AccumulatorData, - query_c: &str, + _query_c: &str, ) -> ASAPQueryEngine { - let labels_a: Vec = grouping_labels_a.iter().map(|s| s.to_string()).collect(); - let labels_b: Vec = grouping_labels_b.iter().map(|s| s.to_string()).collect(); - let labels_c: Vec = grouping_labels_c.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let mut ids: Vec = Vec::new(); - - for (agg_type, labels, metric) in [ - (aggregation_type_a, &labels_a, metric_a), - (aggregation_type_b, &labels_b, metric_b), - (aggregation_type_c, &labels_c, metric_c), + let store = Arc::new(SketchStore::new()); + for (metric, kind, labels, data) in [ + (metric_a, aggregation_type_a, grouping_labels_a, data_a), + (metric_b, aggregation_type_b, grouping_labels_b, data_b), + (metric_c, aggregation_type_c, grouping_labels_c, data_c), ] { - let cfg = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), - aggregation_type: agg_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(labels.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let id = cfg.policy_fp_u64(); - ids.push(id); - materializations_by_output.insert(id, cfg); + let output = stored_output(metric, kind, labels, 1, WindowKind::Tumbling); + ingest(&store, &output, 0, at_one_second(data)); } - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfgs: Vec<_> = ids - .iter() - .map(|id| { - installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(*id)) - .cloned() - .expect("agg present") - }) - .collect(); - let timestamp = 1_000_000_u64; - for (idx, data) in [(0, data_a), (1, data_b), (2, data_c)] { - let agg_cfg = &agg_cfgs[idx]; - let agg_id = ids[idx]; - for (label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, agg_cfg, &output, acc.as_ref()); - } - } - - let _ = (labels_a, labels_b, labels_c, query_a, query_b, query_c); - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ASAPQueryEngine::new(1).with_sketch_index(store) } /// Creates a single-pop engine with data at multiple timestamps for testing merge. @@ -544,76 +164,21 @@ pub fn create_engine_multi_timestamp( data: Vec<(u64, Option>, Box)>, promql_query: &str, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + create_engine_multi_timestamp_with_window( + metric, aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), - window_size: 1, - slide_interval: 1, - window_type: WindowKind::Tumbling, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg"); - for (timestamp, label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp - 1000, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); - } - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + grouping_labels, + data, + promql_query, + 1, + WindowKind::Tumbling, + ) } /// Creates a single-pop engine with data at multiple timestamps and configurable window. /// /// Like `create_engine_multi_timestamp` but allows setting `window_size` and `window_type` -/// on the PrecomputeMaterialization (needed for temporal queries like `sum_over_time(metric[5s])`). +/// on the stored output (needed for temporal queries like `sum_over_time(metric[5s])`). #[allow(clippy::too_many_arguments)] #[allow(clippy::type_complexity)] pub fn create_engine_multi_timestamp_with_window( @@ -621,72 +186,18 @@ pub fn create_engine_multi_timestamp_with_window( aggregation_type: AggregationType, grouping_labels: Vec<&str>, data: Vec<(u64, Option>, Box)>, - promql_query: &str, + _promql_query: &str, window_size: u64, window_type: WindowKind, ) -> ASAPQueryEngine { - let grouping_label_strings: Vec = - grouping_labels.iter().map(|s| s.to_string()).collect(); - - let mut materializations_by_output = HashMap::new(); - let agg_config = PrecomputeMaterialization { - stored_output_id: None, - semantic_fragment: None, - population_key_encoding: Default::default(), + let store = Arc::new(SketchStore::new()); + let output = stored_output( + metric, aggregation_type, - aggregation_sub_type: String::new(), - parameters: HashMap::new(), - grouping_labels: KeyByLabelNames::new(grouping_label_strings.clone()).into(), - aggregated_labels: KeyByLabelNames::empty(), - rollup_labels: KeyByLabelNames::empty(), - original_yaml: String::new(), + grouping_labels, window_size, - slide_interval: 1, window_type, - window_layout: asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }, - pane_origin_ms: None, - spatial_filter: String::new(), - spatial_filter_normalized: String::new(), - metric: metric.to_string(), - num_aggregates_to_retain: None, - table_name: None, - value_projection: None, - table_population: None, - derived_input: None, - table_timestamp_column: None, - partitioning: None, - value_source_column: None, - }; - let agg_id = agg_config.policy_fp_u64(); - materializations_by_output.insert(agg_id, agg_config); - - let installed_precompute_plan = Arc::new(InstalledPrecomputePlan { - partitioning: Default::default(), - raw_programs: Default::default(), - precompute_plan: None, - materializations_by_output: materializations_by_output - .into_iter() - .map(|(id, cfg)| (asap_types::sds::StoredOutputId(id), cfg)) - .collect(), - storage_backend: Default::default(), - }); - - let summary_store = - std::sync::Arc::new(crate::storage_engines::sketch_db::index::SketchStore::new()); - let resolver = std::sync::Arc::new(SeriesIdResolver::new()); - let agg_cfg = installed_precompute_plan - .get_aggregation_config(asap_types::sds::StoredOutputId(agg_id)) - .cloned() - .expect("agg"); - for (timestamp, label_values_opt, acc) in data { - let key = label_values_opt.map(|labels| KeyByLabelValues { labels }); - let output = PrecomputedOutput::new( - timestamp - 1000, - timestamp, - key, - asap_types::PolicyFingerprint(agg_id), - ); - ingest_with_fresh_resolver(&summary_store, &resolver, &agg_cfg, &output, acc.as_ref()); - } - ASAPQueryEngine::new(1).with_sketch_index(summary_store) + ); + ingest(&store, &output, 1000, data); + ASAPQueryEngine::new(1).with_sketch_index(store) } diff --git a/data_plane/src/tests/test_utilities/mod.rs b/data_plane/src/tests/test_utilities/mod.rs index 824a72884..2dffa3274 100644 --- a/data_plane/src/tests/test_utilities/mod.rs +++ b/data_plane/src/tests/test_utilities/mod.rs @@ -6,3 +6,5 @@ pub mod timing; pub use engine_factories::*; pub mod planning; + +pub mod outputs; diff --git a/data_plane/src/tests/test_utilities/outputs.rs b/data_plane/src/tests/test_utilities/outputs.rs new file mode 100644 index 000000000..917b7e36a --- /dev/null +++ b/data_plane/src/tests/test_utilities/outputs.rs @@ -0,0 +1,125 @@ +//! Stored-output fixtures: deployment fields paired with the stored state +//! family an installed plan's schema contract would carry. +use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; +use planner_types::post_asap::{ + GroupingStrategy, HydraKind, HydraParams, SketchAlgorithm, SketchKind, SketchParams, + SummaryFamilyType, +}; +use serde_json::Value; + +/// A tumbling output over `metric` grouped by `grouping`, with its id +/// allocated from `family`. +pub fn output( + metric: &str, + family: SummaryFamilyType, + grouping: Vec<&str>, + window: u64, +) -> (PrecomputeMaterialization, SummaryFamilyType) { + let config = PrecomputeMaterialization::new( + metric, + KeyByLabelNames::new(grouping.into_iter().map(str::to_owned).collect()), + window, + window, + WindowKind::Tumbling, + ); + allocated(config, family) +} + +/// Allocate `config`'s id from `family` after its deployment fields are final. +pub fn allocated( + mut config: PrecomputeMaterialization, + family: SummaryFamilyType, +) -> (PrecomputeMaterialization, SummaryFamilyType) { + config.allocate_stored_output_id(&family); + (config, family) +} + +/// The Planner family of a kernel fixture spelled as a backend kernel tag and +/// its sketch dimensions, with the kernels' default dimensions. +pub fn family(kind: AggregationType, parameters: &Value) -> SummaryFamilyType { + let u32_parameter = |names: &[&str], default: u32| { + names + .iter() + .find_map(|name| parameters.get(*name).and_then(Value::as_u64)) + .map_or(default, |value| value as u32) + }; + let sketch = |algorithm, params| { + SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + GroupingStrategy::PerSubpopulationInstance, + ) + }; + let (width, depth) = (u32_parameter(&["w"], 1000), u32_parameter(&["d"], 4)); + let heap_size = u32_parameter(&["heap_size", "k", "K"], 20); + match kind { + AggregationType::DatasketchesKLL => sketch( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: u32_parameter(&["K", "k"], 200), + }, + ), + AggregationType::HydraKLL => { + let k = u32_parameter(&["K", "k"], 200); + SummaryFamilyType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k }), + GroupingStrategy::SharedMultiSubpopulation { + kind: HydraKind::HydraKll, + params: HydraParams::HydraKll { + k, + shared_buckets: k, + }, + }, + ) + } + AggregationType::CountMinSketch => { + sketch(SketchAlgorithm::Cms, SketchParams::Cms { width, depth }) + } + AggregationType::CountMinSketchWithHeap => sketch( + SketchAlgorithm::CmsWithHeap, + SketchParams::CmsWithHeap { + width, + depth, + heap_size, + }, + ), + AggregationType::CountSketch => sketch( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width, depth }, + ), + AggregationType::CountSketchWithHeap => sketch( + SketchAlgorithm::CountSketchWithHeap, + SketchParams::CountSketchWithHeap { + width, + depth, + heap_size, + }, + ), + AggregationType::DDSketch => sketch( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { + alpha: ["relativeAccuracy", "relative_accuracy", "alpha"] + .iter() + .find_map(|name| parameters.get(*name).and_then(Value::as_f64)) + .unwrap_or(0.01), + }, + ), + AggregationType::HLL => sketch( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: u32_parameter(&["precision", "p"], 14) as u8, + }, + ), + AggregationType::UnivMon => sketch( + SketchAlgorithm::UnivMon, + SketchParams::UnivMon { + heap_size: u32_parameter(&["heap_size"], 32), + sketch_rows: u32_parameter(&["sketch_rows"], 5), + sketch_cols: u32_parameter(&["sketch_cols"], 1024), + layers: u32_parameter(&["layers"], 4) as u8, + }, + ), + exact => exact + .planner_exact_family() + .unwrap_or_else(|| panic!("{exact:?} names no Planner family")), + } +} diff --git a/data_plane/tests/all_sketches_process_oracle_e2e.rs b/data_plane/tests/all_sketches_process_oracle_e2e.rs index 4340e6154..41619306c 100644 --- a/data_plane/tests/all_sketches_process_oracle_e2e.rs +++ b/data_plane/tests/all_sketches_process_oracle_e2e.rs @@ -107,7 +107,12 @@ fn envelope(metric: &str, data: Data) -> ExportMetricsServiceRequest { } } -async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) -> Backend { +async fn start_backend( + materialization: &( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + ), +) -> Backend { let query_port = unused_port(); let otlp_http_port = unused_port(); let otlp_grpc_port = unused_port(); @@ -124,11 +129,12 @@ async fn start_backend(materialization: &asap_types::PrecomputeMaterialization) .find(|s| s.materialization == rule.materialization) .unwrap() .family, - control_plane::physical::compiler::StateFamilyContract::Sketch { - algorithm: planner_types::post_asap::SketchAlgorithm::Cms - | planner_types::post_asap::SketchAlgorithm::CountSketch, - .. - } + planner_types::post_asap::SummaryFamilyType::Sketch(ref kind, _) + if matches!( + kind.algorithm(), + planner_types::post_asap::SketchAlgorithm::Cms + | planner_types::post_asap::SketchAlgorithm::CountSketch + ) ) { rule.encoding = control_plane::physical::compiler::StateEncoding::SketchCoreMsgpackV1; } @@ -281,11 +287,18 @@ fn scalar_values(response: &Value) -> Vec<(HashMap, f64)> { .collect() } -fn config(metric: &str, kind: &str, parameters: &str) -> asap_types::PrecomputeMaterialization { +fn config( + metric: &str, + kind: &str, + parameters: &str, +) -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + let parameters: serde_json::Value = serde_yaml::from_str(parameters).unwrap(); physical_fixture::materialization( metric, - kind.parse().unwrap(), - serde_yaml::from_str(parameters).unwrap(), + physical_fixture::sketch_family(kind.parse().unwrap(), ¶meters), ) } diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 8e231adfd..05cdb4511 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -379,9 +379,16 @@ async fn certified_kll_state_to_query_oracle() { .compile_promql(request, environment) .unwrap(); assert_eq!(plan.precompute_plan.materializations.len(), 1); - let k = plan.precompute_plan.materializations[0].parameters["k"] - .as_u64() - .unwrap() as u32; + let family = plan + .precompute_plan + .state_family(plan.precompute_plan.materializations[0].stored_output_id) + .unwrap(); + let planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) = family else { + panic!("quantile output stores a sketch") + }; + let planner_types::post_asap::SketchParams::Kll { k } = *kind.params() else { + panic!("quantile output stores a KLL") + }; let guarantee = asap_aware_mapping::DefaultAccuracyModel::sketch_guarantee( &planner_types::post_asap::SketchAlgorithm::Kll, &planner_types::post_asap::SketchParams::Kll { k }, @@ -769,14 +776,24 @@ async fn registered_temporal_topk( } _ => panic!("fixture requires a heap implementation"), }; + let output = plan.precompute_plan.materializations[0].stored_output_id; assert_eq!( - plan.precompute_plan.materializations[0].aggregation_type, - expected_type - ); - assert_eq!( - plan.precompute_plan.materializations[0].parameters["weight_mode"], - "count" + plan.precompute_plan + .state_family(output) + .and_then(asap_types::aggregation_type_for_family), + Some(expected_type) ); + // The Planner update counts occurrences. + let (producer, _) = plan + .precompute_plan + .summary_producer(output) + .unwrap() + .unwrap(); + assert!(matches!( + producer.payload, + planner_types::post_asap::PostAsapOperatorPayload::SummaryAgg { ref input, .. } + if input.weight == planner_types::post_asap::SummaryInputExpr::Constant(1.0) + )); let artifact = data_plane::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, collector_plans: plan.collector_plans, @@ -1072,7 +1089,15 @@ async fn run_shared_dashboard(multi_pane: bool) { .precompute_plan .materializations .iter() - .map(|m| m.aggregation_type.as_str()) + .map(|m| { + asap_types::aggregation_type_for_family( + plan.precompute_plan + .state_family(m.stored_output_id) + .unwrap(), + ) + .unwrap() + .as_str() + }) .collect::>(); assert_eq!(families, std::collections::BTreeSet::from(["Sum", "Count"])); assert_eq!(plan.query_plan.entries.len(), 3); diff --git a/data_plane/tests/component_process_e2e.rs b/data_plane/tests/component_process_e2e.rs index 8a6d59269..198e88259 100644 --- a/data_plane/tests/component_process_e2e.rs +++ b/data_plane/tests/component_process_e2e.rs @@ -127,8 +127,10 @@ async fn production_binary_ingests_ddsketch_and_answers_promql() { let install = physical_fixture::artifact_from_materializations(vec![physical_fixture::materialization( "component_process_e2e_latency_ms", - asap_types::AggregationType::DDSketch, - [("relative_accuracy".into(), serde_json::json!(0.01))].into(), + physical_fixture::sketch_family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"relative_accuracy": 0.01}), + ), )]); let mut physical = tempfile::NamedTempFile::new().unwrap(); serde_json::to_writer(&mut physical, &install).unwrap(); diff --git a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs index ecc9d51aa..ce8e22b59 100644 --- a/data_plane/tests/e2e_controller_plans_and_backend_serves.rs +++ b/data_plane/tests/e2e_controller_plans_and_backend_serves.rs @@ -45,6 +45,7 @@ //! metric. use asap_types::PrecomputeMaterialization; +use planner_types::post_asap::SummaryFamilyType; use std::sync::Arc; use std::time::Duration; #[path = "support/physical_fixture.rs"] @@ -72,23 +73,27 @@ fn phase_aligned_now_ns() -> u64 { now - now % 5_000_000_000 + 3_000_000_000 } +/// A stored output with its state family. +type Output = (PrecomputeMaterialization, SummaryFamilyType); + async fn post_full_config( client: &reqwest::Client, stack: &FullStack, - materializations: &[PrecomputeMaterialization], + materializations: &[Output], ) { let mut configs = materializations.to_vec(); // The transport payloads below carry one-second states, so pin the // physical layout to match them. - for config in &mut configs { + for (config, family) in &mut configs { config.window_size = 1; config.slide_interval = 1; config.window_layout = asap_types::WindowMaterializationLayout::Pane { pane_secs: 1 }; + config.allocate_stored_output_id(family); } let mut artifact = physical_fixture::artifact_from_materializations(configs.clone()); if configs .iter() - .any(|c| c.metric == "http_requests_total_latency_ms") + .any(|(c, _)| c.metric == "http_requests_total_latency_ms") { for rule in &mut artifact.transmission_plan.rules { rule.mode = asap_types::producer_plan::TransmissionMode::Delta; @@ -170,7 +175,7 @@ use prost::Message; /// target and read back whichever family and parameters Planner committed to, /// rather than pinning a family. Family selection itself is covered by the /// control-plane compiler tests. -fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { +fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec { use control_plane::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; let mut fixture: JsonValue = serde_json::from_str(include_str!( @@ -204,32 +209,39 @@ fn plan_materializations(query: &str, accuracy: JsonValue) -> Vec Vec { - vec![PrecomputeMaterialization::new( - asap_types::AggregationType::CountMinSketch, - String::new(), - std::collections::HashMap::from([ - ("w".into(), serde_json::json!(512)), - ("d".into(), serde_json::json!(5)), - ]), +fn imported_cms_materializations(metric: &str) -> Vec { + let family = SummaryFamilyType::Sketch( + planner_types::post_asap::SketchKind::new( + planner_types::post_asap::SketchAlgorithm::Cms, + planner_types::post_asap::SketchParams::Cms { + width: 512, + depth: 5, + }, + ), + planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, + ); + let mut config = PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(vec!["service".into()]), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 5, 5, asap_types::enums::WindowKind::Tumbling, - String::new(), - metric.into(), - Some(12), - None, - None, - )] + ); + config.num_aggregates_to_retain = Some(12); + config.allocate_stored_output_id(&family); + vec![(config, family)] } /// Epsilon-delta accuracy target in the shape `QueryRequirements` expects. @@ -654,7 +666,7 @@ async fn controller_streaming_config_round_trips_through_backend_http() { 1, "expected exactly one materialization: {materializations:#?}" ); - let agg = &materializations[0]; + let agg = &materializations[0].0; assert_eq!(agg.metric, "http_latency_ms"); assert!(agg.window_size > 0, "window size must be > 0: {agg:#?}"); @@ -681,7 +693,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // The planner must thread the query's grouping into the materialization // the backend keys its per-population state by. - let agg = &materializations[0]; + let agg = &materializations[0].0; assert!( agg.grouping_labels .names() @@ -698,6 +710,7 @@ async fn controller_plans_with_grouping_and_backend_parses_grouping_labels() { // endpoint that reports only plan phase. assert!( materializations[0] + .0 .grouping_labels .names() .iter() @@ -905,9 +918,12 @@ async fn controller_plan_to_query_full_roundtrip_kll() { ); post_full_config(&client, &stack, &materializations).await; - let alpha = materializations[0].parameters["alpha"] - .as_f64() - .expect("planner sized a relative-accuracy quantile summary"); + let SummaryFamilyType::Sketch(kind, _) = &materializations[0].1 else { + panic!("planner selected a quantile sketch") + }; + let planner_types::post_asap::SketchParams::DDSketch { alpha } = kind.params().clone() else { + panic!("planner sized a relative-accuracy quantile summary") + }; let dd_state = build_dd_sketch_state(alpha, vec![5u64, 10, 15, 20], -1); let sketch_bytes = encode_dd_full_state(dd_state); @@ -1230,14 +1246,14 @@ async fn imported_cms_frequency_wire_roundtrip() { /// content match probes `parameters.w` and `parameters.d`). /// Sketch width/depth the planner sized this materialization to. The test /// payloads are built against these, never against pinned constants. -fn extract_w_d(agg: &PrecomputeMaterialization) -> (u32, u32) { - let w = agg.parameters["w"] - .as_u64() - .expect("materialization must carry parameters.w") as u32; - let d = agg.parameters["d"] - .as_u64() - .expect("materialization must carry parameters.d") as u32; - (w, d) +fn extract_w_d(agg: &Output) -> (u32, u32) { + let SummaryFamilyType::Sketch(kind, _) = &agg.1 else { + panic!("materialization stores a sketch") + }; + let planner_types::post_asap::SketchParams::Cms { width, depth } = kind.params() else { + panic!("materialization stores a CMS") + }; + (*width, *depth) } // Heap TopK serving acceptance lives in asapquery_compatibility_process_e2e: diff --git a/data_plane/tests/promql_differential_process_e2e.rs b/data_plane/tests/promql_differential_process_e2e.rs index a3c6359c5..50a4a18d2 100644 --- a/data_plane/tests/promql_differential_process_e2e.rs +++ b/data_plane/tests/promql_differential_process_e2e.rs @@ -335,24 +335,16 @@ async fn production_backend_matches_raw_oracle_and_range_endpoint() { } } -fn ddsketch_config() -> asap_types::PrecomputeMaterialization { - use asap_types::{AggregationType, KeyByLabelNames, PrecomputeMaterialization, WindowKind}; - PrecomputeMaterialization::new( - AggregationType::DDSketch, - String::new(), - std::collections::HashMap::from([("relative_accuracy".into(), serde_json::json!(ALPHA))]), - KeyByLabelNames::new(vec!["service".into()]), - KeyByLabelNames::empty(), - KeyByLabelNames::empty(), - String::new(), - 1, - 1, - WindowKind::Tumbling, - String::new(), - METRIC.into(), - None, - None, - None, +fn ddsketch_config() -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + physical_fixture::materialization( + METRIC, + physical_fixture::sketch_family( + asap_types::AggregationType::DDSketch, + &serde_json::json!({"relative_accuracy": ALPHA}), + ), ) } @@ -362,9 +354,9 @@ async fn bound_sds_keeps_hot_and_rebuild_outputs_and_groups_isolated() { use asap_types::query_plan::QueryPlanNode; use asap_types::sds::StoredOutputId; let mut hot = ddsketch_config(); - hot.stored_output_id = Some(StoredOutputId(1001)); + hot.0.stored_output_id = StoredOutputId(1001); let mut rebuild = hot.clone(); - rebuild.stored_output_id = Some(StoredOutputId(1002)); + rebuild.0.stored_output_id = StoredOutputId(1002); let mut install = physical_fixture::artifact_from_materializations(vec![hot, rebuild]); assert_eq!(install.summary_catalog.definitions.len(), 1); assert_eq!(install.summary_catalog.outputs.len(), 2); diff --git a/data_plane/tests/support/empty_physical_plan.rs b/data_plane/tests/support/empty_physical_plan.rs index 0e453d85b..339bd56a4 100644 --- a/data_plane/tests/support/empty_physical_plan.rs +++ b/data_plane/tests/support/empty_physical_plan.rs @@ -12,7 +12,7 @@ pub fn empty() -> asap_types::plan_publication::PhysicalPlanInstallRequest { capability_snapshot_id: "empty-bootstrap".into(), }; let catalog = - control_plane::physical::summary_catalog::SummaryCatalog::from_materializations(0, 0, &[]) + control_plane::physical::summary_catalog::SummaryCatalog::from_outputs(0, 0, Vec::new()) .unwrap(); let mut precompute_plan = PrecomputePlan::build(envelope.clone(), vec![], &[]).unwrap(); precompute_plan.summary_catalog = Some(catalog.reference().unwrap()); diff --git a/data_plane/tests/support/immutable_maintenance_process.rs b/data_plane/tests/support/immutable_maintenance_process.rs index 1aa1fefc5..d6b3d516e 100644 --- a/data_plane/tests/support/immutable_maintenance_process.rs +++ b/data_plane/tests/support/immutable_maintenance_process.rs @@ -66,10 +66,17 @@ async fn run_maintenance_process(multi_source: bool, distinct_groups: bool) { ); // The production cost model may choose DDSketch or KLL. Preserve that // choice and use its actual value contract for this singleton oracle. - let max_relative_error = match derived.aggregation_type { - asap_types::AggregationType::DDSketch => derived.parameters["alpha"].as_f64().unwrap(), - asap_types::AggregationType::DatasketchesKLL => 0.0, - ref other => panic!("singleton quantile oracle missing for {other:?}"), + let family = plan + .precompute_plan + .state_family(derived.stored_output_id) + .unwrap(); + let max_relative_error = match family { + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::DDSketch { alpha } => *alpha, + planner_types::post_asap::SketchParams::Kll { .. } => 0.0, + other => panic!("singleton oracle missing for {other:?}"), + }, + other => panic!("singleton quantile oracle missing for {other:?}"), }; eprintln!( "IMMUTABLE_SELECTED {}", diff --git a/data_plane/tests/support/physical_fixture.rs b/data_plane/tests/support/physical_fixture.rs index e86a100e9..80d6c6d39 100644 --- a/data_plane/tests/support/physical_fixture.rs +++ b/data_plane/tests/support/physical_fixture.rs @@ -8,40 +8,97 @@ use data_plane::{ }; use std::{collections::BTreeMap, sync::Arc}; -/// Imported-state fixtures bind explicit storage metadata, never a flat runtime config. +/// An imported-state output with its stored state family. Imported-state +/// fixtures bind explicit storage metadata, never a flat runtime config. #[allow(dead_code)] pub fn materialization( metric: &str, - family: asap_types::AggregationType, - parameters: std::collections::HashMap, -) -> asap_types::PrecomputeMaterialization { - asap_types::PrecomputeMaterialization::new( - family, - String::new(), - parameters, + family: planner_types::post_asap::SummaryFamilyType, +) -> ( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, +) { + let mut config = asap_types::PrecomputeMaterialization::new( + metric, asap_types::KeyByLabelNames::new(vec!["service".into()]), - asap_types::KeyByLabelNames::empty(), - asap_types::KeyByLabelNames::empty(), - String::new(), 1, 1, asap_types::enums::WindowKind::Tumbling, - String::new(), - metric.into(), - None, - None, - None, + ); + config.allocate_stored_output_id(&family); + (config, family) +} + +/// The sketch family an imported-state fixture's collector produces, from +/// the sketch kind and the dimensions it was spelled with. +#[allow(dead_code)] +pub fn sketch_family( + kind: asap_types::AggregationType, + parameters: &serde_json::Value, +) -> planner_types::post_asap::SummaryFamilyType { + use asap_types::AggregationType as A; + use planner_types::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; + let get = |names: &[&str]| { + names + .iter() + .find_map(|name| parameters.get(*name).and_then(serde_json::Value::as_f64)) + }; + let (width, depth) = ( + get(&["w"]).unwrap_or(1000.0) as u32, + get(&["d"]).unwrap_or(4.0) as u32, + ); + let (algorithm, params) = match kind { + A::DatasketchesKLL => ( + SketchAlgorithm::Kll, + SketchParams::Kll { + k: get(&["k", "K"]).unwrap_or(200.0) as u32, + }, + ), + A::HLL => ( + SketchAlgorithm::Hll, + SketchParams::Hll { + precision: get(&["precision", "p"]).unwrap_or(14.0) as u8, + }, + ), + A::CountMinSketch => (SketchAlgorithm::Cms, SketchParams::Cms { width, depth }), + A::CountSketch => ( + SketchAlgorithm::CountSketch, + SketchParams::CountSketch { width, depth }, + ), + A::DDSketch => ( + SketchAlgorithm::DDSketch, + SketchParams::DDSketch { + alpha: get(&["alpha", "relative_accuracy"]).unwrap_or(0.01), + }, + ), + other => panic!("no imported-state fixture for {other:?}"), + }; + planner_types::post_asap::SummaryFamilyType::Sketch( + SketchKind::new(algorithm, params), + Default::default(), ) } pub fn artifact(config: &InstalledPrecomputePlan) -> PhysicalPlanInstallRequest { - artifact_from_materializations(config.materializations().values().cloned().collect()) + artifact_from_materializations( + config + .materializations() + .values() + .map(|c| { + let family = config.state_family(c.stored_output_id).unwrap().clone(); + (c.clone(), family) + }) + .collect(), + ) } /// Same as [`artifact`], but from materializations the planner produced /// directly — no legacy `InstalledPrecomputePlan` document in between. pub fn artifact_from_materializations( - mut configs: Vec, + mut configs: Vec<( + asap_types::PrecomputeMaterialization, + planner_types::post_asap::SummaryFamilyType, + )>, ) -> PhysicalPlanInstallRequest { let envelope = PlanEnvelope { plan_id: 1, @@ -56,7 +113,7 @@ pub fn artifact_from_materializations( // Installed physical plans always carry an explicit pane phase. Inputs that // predate that contract bind to the Unix epoch grid before deriving catalog // identities and bindings. - for config in &mut configs { + for (config, _) in &mut configs { config.pane_origin_ms.get_or_insert(0); // These fixtures import synthetic states and replace the producer's // window layout. They do not install the original Planner DAG, so @@ -68,12 +125,10 @@ pub fn artifact_from_materializations( ); config.semantic_fragment = None; } - let catalog = control_plane::physical::summary_catalog::SummaryCatalog::from_materializations( - 1, 1, &configs, - ) - .unwrap(); let mut precompute = PrecomputePlan::build(envelope.clone(), configs, &["fixture".into()]).unwrap(); + let catalog = + control_plane::physical::summary_catalog::SummaryCatalog::from_plan(&precompute).unwrap(); precompute.bind_catalog(&catalog).unwrap(); let mut transmission = control_plane::physical::compiler::build_transmission_plan( envelope, @@ -93,7 +148,8 @@ pub fn artifact_from_materializations( use data_plane::storage_engines::types::AggregationType; let metric = &config.metric; let mut queries = Vec::new(); - match config.aggregation_type { + let family = precompute.state_family(config.stored_output_id).unwrap(); + match asap_types::aggregation_type_for_family(family).unwrap() { AggregationType::DatasketchesKLL | AggregationType::DDSketch => { for (window, ms) in [ ("1s", 1_000), @@ -165,7 +221,7 @@ pub fn artifact_from_materializations( config.policy_fingerprint().into(), ), output_grouping, - item_labels: config.aggregated_labels.labels.clone(), + item_labels: Vec::new(), window_ms: config.slide_interval * 1000, pane_origin_ms: config.pane_origin_ms, readout_lookback_ms: Some(lookback_ms), diff --git a/data_plane/tests/support/revisable_maintenance_process.rs b/data_plane/tests/support/revisable_maintenance_process.rs index eac0f8ca9..fef0e85d2 100644 --- a/data_plane/tests/support/revisable_maintenance_process.rs +++ b/data_plane/tests/support/revisable_maintenance_process.rs @@ -58,10 +58,17 @@ async fn run_revisions(multi_source: bool) { .iter() .find(|m| m.derived_input.is_some()) .unwrap(); - let relative_error = match derived.aggregation_type { - asap_types::AggregationType::DDSketch => derived.parameters["alpha"].as_f64().unwrap(), - asap_types::AggregationType::DatasketchesKLL => 0.0, - ref other => panic!("singleton revision oracle missing for {other:?}"), + let family = installed + .precompute_plan + .state_family(derived.stored_output_id) + .unwrap(); + let relative_error = match family { + planner_types::post_asap::SummaryFamilyType::Sketch(kind, _) => match kind.params() { + planner_types::post_asap::SketchParams::DDSketch { alpha } => *alpha, + planner_types::post_asap::SketchParams::Kll { .. } => 0.0, + other => panic!("singleton oracle missing for {other:?}"), + }, + other => panic!("singleton revision oracle missing for {other:?}"), }; std::fs::write(&artifact, serde_json::to_vec(&installed).unwrap()).unwrap(); let spawn = |port: u16| { diff --git a/docs/examples/asapquery-compatibility-demo-snapshot.json b/docs/examples/asapquery-compatibility-demo-snapshot.json index c24e976c4..ed36d55f3 100644 --- a/docs/examples/asapquery-compatibility-demo-snapshot.json +++ b/docs/examples/asapquery-compatibility-demo-snapshot.json @@ -120,6 +120,6 @@ "plan_version": 1, "activation_unix_ms": 10000, "expiry_unix_ms": null, - "backend_compat": "asap-query-backend.v1" + "backend_compat": "asap-query-backend.v2" } } diff --git a/docs/examples/asapquery-planning-snapshot.json b/docs/examples/asapquery-planning-snapshot.json index 736857c49..655aed5a1 100644 --- a/docs/examples/asapquery-planning-snapshot.json +++ b/docs/examples/asapquery-planning-snapshot.json @@ -71,6 +71,6 @@ "plan_version": 1, "activation_unix_ms": 10000, "expiry_unix_ms": null, - "backend_compat": "asap-query-backend.v1" + "backend_compat": "asap-query-backend.v2" } } From 81b0fddf8cfc3d55eb9f72f0fd60851c12f31527 Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 14:07:09 +0000 Subject: [PATCH 2/3] fix: reject stored outputs without one consistent Planner producer Materializations no longer carry their input filter, so an output's population comes only from the DAG node that produces it. Plan validation now rejects: - an output produced by several DAGs that disagree on the update or on the population (metric and canonical filter) they read; - a raw time-series output that has no Planner producer in a plan that carries DAGs, which would otherwise be read as unfiltered; - a bound raw producer whose input is not a source scan. Also keeps plain OTLP sketch envelopes out of shared (Hydra) states, restores the single-catalog descriptor test, and drops comments that still named removed fields. Co-Authored-By: Claude Opus 5.5 --- crates/asap_types/src/key_by_label_names.rs | 2 +- crates/asap_types/src/precompute_plan.rs | 81 +++++++++++++++++-- crates/asap_types/src/summary_catalog.rs | 22 +++-- data_plane/src/drivers/ingest/otel.rs | 8 ++ .../types/installed_precompute_plan.rs | 71 ++++++++++++++++ .../types/precomputed_output.rs | 2 +- data_plane/src/tests/accumulator_fixture.rs | 26 +++--- 7 files changed, 182 insertions(+), 30 deletions(-) diff --git a/crates/asap_types/src/key_by_label_names.rs b/crates/asap_types/src/key_by_label_names.rs index 5cd902b7d..134e961a4 100644 --- a/crates/asap_types/src/key_by_label_names.rs +++ b/crates/asap_types/src/key_by_label_names.rs @@ -3,7 +3,7 @@ //! Formerly `promql_utilities::data_model::key_by_label_names` — moved //! here for the same reason as [`crate::Statistic`]: `asap_types` //! (`PrecomputeMaterialization::grouping_labels`, `PolicyFingerprint`, -//! `PolicyRegistry`, `capability_matching`) is its real center of +//! `capability_matching`) is its real center of //! gravity and the shared foundation both `control_plane`'s ecosystem //! and `data_plane` can depend on without a cycle. Closer to a runtime //! index key than a planning IR node — ASAPController's diff --git a/crates/asap_types/src/precompute_plan.rs b/crates/asap_types/src/precompute_plan.rs index cefe2cb5c..25d96a267 100644 --- a/crates/asap_types/src/precompute_plan.rs +++ b/crates/asap_types/src/precompute_plan.rs @@ -314,6 +314,11 @@ pub enum PrecomputePlanError { InvalidSchema { schema_id: String }, #[error("materialization {0} uses a summary family unsupported by the runtime schema")] UnsupportedFamily(u64), + #[error("materialization {materialization} has no consistent Planner computation: {reason}")] + InvalidComputation { + materialization: u64, + reason: String, + }, #[error("materialization {materialization} has an invalid window layout: {reason}")] InvalidWindowLayout { materialization: u64, @@ -466,7 +471,9 @@ impl PrecomputePlan { /// The Planner SummaryAgg node that produces `output`, with the source /// expression feeding it when that input is a raw source. `None` when no - /// installed DAG produces the output with a SummaryAgg. + /// installed DAG produces the output with a SummaryAgg. Every DAG that + /// produces the output must agree on both, or the plan is rejected: the + /// runtime routes, filters and catalogs the output once. #[allow(clippy::type_complexity)] pub fn summary_producer( &self, @@ -479,6 +486,10 @@ impl PrecomputePlan { String, > { use planner_types::post_asap::PostAsapOperatorPayload; + let mut selected: Option<( + planner_types::post_asap::PostAsapDagNode, + Option, + )> = None; for installed in self.executable_dags.values() { let Some(node) = installed.binding.nodes.iter().find_map(|(node, binding)| { matches!(binding, crate::executable_plan::BackendNodeBinding::Materialization { stored_output } if *stored_output == output).then_some(*node) @@ -508,9 +519,39 @@ impl PrecomputePlan { }), _ => None, }; - return Ok(Some((producer.clone(), source))); + if let Some((first, first_source)) = &selected { + // Shared outputs may be read by scans that project different + // columns; what must agree is the update and its population. + let population = + |node: &planner_types::post_asap::PostAsapDagNode, + source: &Option| { + let PostAsapOperatorPayload::SummaryAgg { family, .. } = &node.payload + else { + return None; + }; + source.as_ref().map(|expression| { + raw_time_series_input_contract( + expression, + matches!(family, SummaryFamilyType::ExactAggregate(..)), + ) + .map(|(metric, _, filter)| { + (metric, crate::utils::normalize_spatial_filter(&filter)) + }) + }) + }; + if first.payload != producer.payload + || population(first, first_source) != population(producer, &source) + { + return Err(format!( + "stored output {} is produced by DAGs that disagree on its computation", + output.as_u64() + )); + } + } else { + selected = Some((producer.clone(), source)); + } } - Ok(None) + Ok(selected) } /// Canonical population predicate of `config`'s input: the typed table @@ -523,8 +564,16 @@ impl PrecomputePlan { if config.table_name.is_some() || config.derived_input.is_some() { return Ok(table); } - let Some((_, Some(expression))) = self.summary_producer(config.stored_output_id)? else { - return Ok(String::new()); + let expression = match self.summary_producer(config.stored_output_id)? { + Some((_, Some(expression))) => expression, + // A bound raw output whose input cannot be read must not be + // treated as unfiltered. + Some((_, None)) => { + return Err("raw output's Planner producer does not read a source scan".into()) + } + // Only plans without a Planner DAG for this output (imported or + // fixture outputs) have no predicate to read. + None => return Ok(String::new()), }; let family = self.state_family(config.stored_output_id); let (_, _, filter) = raw_time_series_input_contract( @@ -1033,6 +1082,28 @@ impl PrecomputePlan { schema.materialization.as_u64(), )); } + let invalid_computation = |reason: String| PrecomputePlanError::InvalidComputation { + materialization: schema.materialization.as_u64(), + reason, + }; + let raw = + materialization.table_name.is_none() && materialization.derived_input.is_none(); + // Once a plan carries Planner DAGs, a raw output's input predicate + // exists only on its producer; an unbound output would silently + // read every series of its metric. + if raw + && !self.executable_dags.is_empty() + && self + .summary_producer(schema.materialization) + .map_err(invalid_computation)? + .is_none() + { + return Err(invalid_computation( + "raw output has no Planner producer".into(), + )); + } + self.population_filter(materialization) + .map_err(invalid_computation)?; let source = materialization.table_name.as_ref().map_or_else( || Source::TimeSeries { metric: materialization.metric.clone(), diff --git a/crates/asap_types/src/summary_catalog.rs b/crates/asap_types/src/summary_catalog.rs index 614f09750..c83ef3ded 100644 --- a/crates/asap_types/src/summary_catalog.rs +++ b/crates/asap_types/src/summary_catalog.rs @@ -583,14 +583,22 @@ mod tests { GroupingStrategy::PerSubpopulationInstance, ) }; - let a = with_family(1, 1, &[config("requests", "", 60)], &kll(100)).unwrap(); - let b = with_family(1, 1, &[config("requests", "", 60)], &kll(200)).unwrap(); - assert_ne!(a.summary_descriptors, b.summary_descriptors); - assert_eq!(a.data_descriptors, b.data_descriptors); - let catalog = a; - assert_eq!(catalog.summary_descriptors.len(), 1); + let (small, large) = (kll(100), kll(200)); + let output = |family: &SummaryFamilyType| { + let (mut config, _) = config("requests", "", 60); + config.allocate_stored_output_id(family); + config + }; + let (a, b) = (output(&small), output(&large)); + let catalog = SummaryCatalog::from_outputs( + 1, + 1, + vec![(&a, &small, String::new()), (&b, &large, String::new())], + ) + .unwrap(); + assert_eq!(catalog.summary_descriptors.len(), 2); assert_eq!(catalog.data_descriptors.len(), 1); - assert_eq!(catalog.outputs.len(), 1); + assert_eq!(catalog.outputs.len(), 2); } // Construction order cannot affect the published snapshot bytes. diff --git a/data_plane/src/drivers/ingest/otel.rs b/data_plane/src/drivers/ingest/otel.rs index 5691b0ddc..72a58e84c 100644 --- a/data_plane/src/drivers/ingest/otel.rs +++ b/data_plane/src/drivers/ingest/otel.rs @@ -1919,6 +1919,14 @@ fn derive_sketch_policy_fp( if bound_output.is_some_and(|output| output != config.stored_output_id) || config.metric != metric || &keys != group_by_keys + // A plain sketch envelope never fills a shared (Hydra) state. + || matches!( + snap.state_family(config.stored_output_id), + Some(planner_types::post_asap::SummaryFamilyType::Sketch( + _, + planner_types::post_asap::GroupingStrategy::SharedMultiSubpopulation { .. } + )) + ) || snap .agg_kind(config.stored_output_id) .is_none_or(|installed| installed.canonical_string() != expected) diff --git a/data_plane/src/storage_engines/types/installed_precompute_plan.rs b/data_plane/src/storage_engines/types/installed_precompute_plan.rs index e42c553d2..d616a0307 100644 --- a/data_plane/src/storage_engines/types/installed_precompute_plan.rs +++ b/data_plane/src/storage_engines/types/installed_precompute_plan.rs @@ -262,6 +262,77 @@ impl Default for InstalledPrecomputePlan { #[cfg(test)] mod tests { + fn filtered_plan(job: &str) -> control_plane::physical::compiler::PrecomputePlan { + let mut snapshot: serde_json::Value = serde_json::from_str(include_str!( + "../../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + snapshot["query_workload"]["repeating_queries"][0]["query"] = + serde_json::json!(format!("quantile_over_time(0.99, m{{job=\"{job}\"}}[1m])")); + crate::tests::test_utilities::planning::quoted_snapshot( + serde_json::from_value(snapshot).unwrap(), + false, + ) + .compile_promql() + .unwrap() + .precompute_plan + } + + /// Rebind every materialization node of `plan`'s DAGs to `output`. + fn bind_all(plan: &mut control_plane::physical::compiler::PrecomputePlan, output: u64) { + for installed in plan.executable_dags.values_mut() { + for binding in installed.binding.nodes.values_mut() { + if let asap_types::executable_plan::BackendNodeBinding::Materialization { + stored_output, + } = binding + { + *stored_output = asap_types::sds::StoredOutputId(output); + } + } + } + } + + // Two DAGs that read different populations cannot share one stored output. + #[test] + fn producers_disagreeing_on_population_are_rejected() { + let mut plan = filtered_plan("a"); + let output = plan.materializations[0].stored_output_id.as_u64(); + let mut other = filtered_plan("b"); + bind_all(&mut other, output); + let (_, mut dag) = other.executable_dags.into_iter().next().unwrap(); + dag.document.query_id = "other-population".into(); + plan.executable_dags.insert("other-population".into(), dag); + let error = plan.validate().unwrap_err().to_string(); + assert!(error.contains("disagree on its computation"), "{error}"); + } + + // A raw output with no Planner producer is not read as unfiltered. + #[test] + fn unbound_raw_output_is_rejected() { + let mut plan = filtered_plan("a"); + let mut unbound = plan.materializations[0].clone(); + unbound.metric = "unbound".into(); + unbound.stored_output_id = asap_types::sds::StoredOutputId(1); + let mut schema = plan.schemas[0].clone(); + schema.materialization = unbound.stored_output_id; + schema.schema_id = schema.schema_id.replace( + &plan.materializations[0] + .stored_output_id + .as_u64() + .to_string(), + "1", + ); + schema.stored_output_reference = + asap_types::sds::StoredOutputReference::for_output(unbound.stored_output_id); + plan.materializations.push(unbound); + plan.schemas.push(schema); + let error = plan.validate().unwrap_err().to_string(); + assert!( + error.contains("raw output has no Planner producer"), + "{error}" + ); + } + // A raw output's input predicate, pane convention and stored state kind // come from its installed Planner producer, not from the materialization. #[test] diff --git a/data_plane/src/storage_engines/types/precomputed_output.rs b/data_plane/src/storage_engines/types/precomputed_output.rs index 8644f37da..fa6a147f3 100644 --- a/data_plane/src/storage_engines/types/precomputed_output.rs +++ b/data_plane/src/storage_engines/types/precomputed_output.rs @@ -78,7 +78,7 @@ impl PrecomputedOutput { /// /// `policy_fp` is the content-addressed handle on the source /// [`asap_types::PrecomputeMaterialization`]; sinks use it to look up the - /// config via `PolicyRegistry::get(policy_fp)`. Construction sites + /// installed output by it. Construction sites /// that lack a source config (raw-mode fast-path) pass /// [`PolicyFingerprint::UNSET`]; sinks then skip the output. pub fn new( diff --git a/data_plane/src/tests/accumulator_fixture.rs b/data_plane/src/tests/accumulator_fixture.rs index 4c8866a9d..d5e9a9c3e 100644 --- a/data_plane/src/tests/accumulator_fixture.rs +++ b/data_plane/src/tests/accumulator_fixture.rs @@ -42,20 +42,15 @@ fn topk_weight(update: &SummaryUpdate) -> TopkWeight { // Factory function // --------------------------------------------------------------------------- -/// Read the KLL `k` out of `SketchParams::Kll`. `accumulator_spec()` -/// always builds a `SketchKind` whose `SketchAlgorithm::Kll` is paired with -/// `SketchParams::Kll`, so the -/// other arm is unreachable from a `spec` this module builds itself. +/// Read the KLL `k` out of `SketchParams::Kll`; a valid `SketchKind` +/// pairs `SketchAlgorithm::Kll` with no other params. #[cfg(test)] fn kll_k(params: &SketchParams) -> u16 { match params { - // Lossless: `accumulator_spec()` only ever stores a value that - // already fit in `u16` (via `kll_k_param`'s own `u16::try_from` - // fallback) widened to `u32`. SketchParams::Kll { k } => *k as u16, - other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::Kll with non-Kll params: {other:?}" - ), + other => { + unreachable!("SketchKind paired SketchAlgorithm::Kll with non-Kll params: {other:?}") + } } } @@ -67,7 +62,7 @@ fn cms_dims(params: &SketchParams) -> (usize, usize) { (*depth as usize, *width as usize) } other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" + "SketchKind paired SketchAlgorithm::Cms/CountSketch with unexpected params: {other:?}" ), } } @@ -87,7 +82,7 @@ fn cms_heap_dims(params: &SketchParams) -> (usize, usize, usize) { heap_size, } => (*depth as usize, *width as usize, *heap_size as usize), other => unreachable!( - "accumulator_spec() paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" + "SketchKind paired a WithHeap SketchAlgorithm with unexpected params: {other:?}" ), } } @@ -98,7 +93,7 @@ fn ddsketch_alpha(params: &SketchParams) -> f64 { match params { SketchParams::DDSketch { alpha } => *alpha, other => unreachable!( - "accumulator_spec() paired SketchAlgorithm::DDSketch with non-DDSketch params: {other:?}" + "SketchKind paired SketchAlgorithm::DDSketch with non-DDSketch params: {other:?}" ), } } @@ -157,9 +152,8 @@ pub fn create_fixture_accumulator( // Heap-bearing top-k variant (raw-input ingest path): route to the // real `CmsHeapAccumulatorUpdater` so the per-policy top-k heap is // BUILT (heap-less CMS could not answer `topk(...)` — recall 0). - // Keyed by the configured group-by `aggregated_labels` (e.g. `host`), - // ranked by Σ value per key by default (`weight_mode: value`), or Σ - // count for genuine frequency-top-k (`weight_mode: count`). The OTLP + // Keyed by the update's item, ranked by Σ value per key, or Σ count + // when the update weight is the constant 1. The OTLP // modified-sketch path builds the heap agent-side and uses // `SketchEnvelope` ingest, not this raw arm. (SummaryFamilyType::Sketch(kind, _), _) From 57baf5d90aef6ddefd605df3198cb20783924348 Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 14:24:37 +0000 Subject: [PATCH 3/3] fix: retain one test attribute after storage stack integration Co-Authored-By: Claude Opus 5.5 --- data_plane/src/storage_engines/sketch_db/index/mod.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/data_plane/src/storage_engines/sketch_db/index/mod.rs b/data_plane/src/storage_engines/sketch_db/index/mod.rs index d86f35714..abccdb791 100644 --- a/data_plane/src/storage_engines/sketch_db/index/mod.rs +++ b/data_plane/src/storage_engines/sketch_db/index/mod.rs @@ -4640,7 +4640,6 @@ mod tests { // canonical strings → different resolver cache keys → different // sids by construction. - #[test] // Unkeyed Planner exact MIN panes feed the MIN rollup. #[test] fn planner_exact_min_panes_feed_the_min_rollup() {