From 4a900c59e27ad0094f75b72f29197476e02cdda5 Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 06:55:38 +0000 Subject: [PATCH 1/4] refactor: run query-time computation as Planner physical DAGs The backend re-lowered query-time computation itself: it re-parsed PromQL text to recover each operator (query_time::Lower, query_time_nodes and the selected_* witnesses), built QueryTimeOperator Binary/Aggregate/Sort/Limit/ Temporal/Subquery nodes and QueryPlanNode Scalar/Binary/ReduceSum, and translated them into label-map operators (physical_values.rs). The data plane evaluated subquery grids itself. Now the backend lowers only stored-state readouts. The query-time region above them is compiled once by physical_planner::compile, with the readouts as input contracts, into one PhysicalFragment. A root Planner cannot compile over readouts (per-series binaries, comparisons, histogram_quantile, exact selectors under computation) keeps no state and is forwarded whole as ExactFallback, as is any query Planner keeps pre-ASAP. Counter and range-maximum materializations are identified from the selected node, not from query text. The removed operator variants no longer deserialize. Co-Authored-By: Claude Opus 5.5 --- control_plane/src/physical/compiler.rs | 265 +-- .../src/physical/compiler/placement.rs | 25 +- control_plane/src/physical/plan_dot.rs | 12 - control_plane/src/physical/workload_cost.rs | 2 +- control_plane/src/query_plan.rs | 803 ++++---- .../src/query_plan/physical_values.rs | 445 ----- control_plane/src/query_plan/query_time.rs | 1774 +---------------- control_plane/tests/lifecycle_placement.rs | 2 +- control_plane/tests/offline_evidence.rs | 14 +- crates/asap_types/src/query_plan.rs | 101 +- .../asap_types/src/query_plan/query_time.rs | 92 +- .../query_engines/asap_query_engine/engine.rs | 37 +- .../asap_query_engine/exact_subqueries.rs | 93 +- .../asap_query_engine/logical_dag.rs | 1220 ++---------- .../logical_dag/native_values.rs | 127 +- .../asap_query_engine/post_asap_readout.rs | 44 +- .../asap_query_engine/test_plan.rs | 46 + .../tests/support/issue_701_702_process.rs | 87 +- .../control-plane/physical-compiler.md | 9 + 19 files changed, 835 insertions(+), 4363 deletions(-) delete mode 100644 control_plane/src/query_plan/physical_values.rs diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 575d8609f..016658d9e 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -1309,6 +1309,33 @@ fn preserve_invalid_exact_fallback_roots( Ok(()) } +/// Planner compiles query-time computation over stored readouts. A selected +/// root whose computation it cannot compile keeps no summary state; the +/// external exact engine evaluates the original query. +fn preserve_uncompiled_computation_roots( + queries: &mut [QueryCompilationInput], + canonical_roots: &[Rc], +) -> Result<(), CompileError> { + for (index, query) in queries.iter_mut().enumerate() { + let root = &query.selected_plan_root; + if super::maintained_population::supported_node(root) + || !crate::query_plan::is_query_computation(root) + || crate::query_plan::compile_query_computation(root).is_ok() + { + continue; + } + let parsed = original_root(query, index, canonical_roots)?; + query.selected_plan_root = + crate::planner_selection::keep_pre_asap(&parsed).map_err(|error| { + CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + } + })?; + } + Ok(()) +} + /// A MetricsQL query whose only selected states are Prometheus-specific /// counter readouts has no backend materialization to bind. Keep the original /// query as one native exact root. Mixed queries retain their other selected @@ -1445,6 +1472,7 @@ impl DeploymentPlanCompiler { &request.canonical_roots, request.allow_mixed_summary_and_exact_execution, )?; + preserve_uncompiled_computation_roots(&mut request.queries, &request.canonical_roots)?; if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite && !request.allow_mixed_summary_and_exact_execution @@ -1539,13 +1567,9 @@ impl DeploymentPlanCompiler { planner_types::post_asap::ExactKind::Max, _ ) - ) || crate::query_plan::query_time::selected_range_max_materialization( - &query.query_string, + ) || crate::query_plan::query_time::is_range_max_materialization( &state.node, - ) - .ok() - .flatten() - .is_some()) + )) // An Ephemeral state is rebuilt at query time, not maintained. && !placement.is_ephemeral(query_index, &state.node) }) @@ -2180,23 +2204,10 @@ impl DeploymentPlanCompiler { full_history: false, cumulative_readout: true, }; - // A whole-query native fallback need not be expressible in the local - // query-time algebra (for example an ERP-rejected entropy readout). - // Retain its native boundary without discarding other workload roots. - let native_root = request.allow_mixed_summary_and_exact_execution - && if let SummaryExpr::KeepPreAsap(expr) = &query.selected_plan_root.expr { - let original = original_root(query, query_index, &request.canonical_roots)?; - expr.as_ref() == &original - && crate::query_plan::query_time::compile_logical( - query.query_id.clone(), - canonical.clone(), - instant, - FallbackPolicy::ExactBackend, - ) - .is_err() - } else { - false - }; + // A query Planner keeps pre-ASAP has no stored input; Prometheus + // evaluates it whole. + let exact_root = request.allow_mixed_summary_and_exact_execution + && matches!(query.selected_plan_root.expr, SummaryExpr::KeepPreAsap(_)); let native_rate = if population_operators[query_index].is_none() && placement.raw_program(query_index).is_none() { @@ -2272,7 +2283,7 @@ impl DeploymentPlanCompiler { fallback: FallbackPolicy::ExactBackend, } } else { - crate::query_plan::compile_bound_composable_mapped( + crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &source, @@ -2380,25 +2391,10 @@ impl DeploymentPlanCompiler { instant, fallback: FallbackPolicy::ExactBackend, }) - } else if request.allow_mixed_summary_and_exact_execution && !native_root { - crate::query_plan::compile_bound_composable_mapped( - query.query_id.clone(), - canonical.clone(), - &query.selected_plan_root, - instant, - FallbackPolicy::ExactBackend, - binding, - |node, query_node| { - if let Some(post_asap_node) = executable_dags[query_index] - .as_ref() - .and_then(|compiled| compiled.node_ids.node_id(node)) - { - query_node_bindings.insert((query_index, post_asap_node), query_node); - } - }, - ) + } else if exact_root { + Ok(exact_query_entry(query, canonical.clone(), instant)) } else { - crate::query_plan::compile_bound_mapped( + match crate::query_plan::compile_bound_mapped( query.query_id.clone(), canonical.clone(), &query.selected_plan_root, @@ -2413,16 +2409,18 @@ impl DeploymentPlanCompiler { query_node_bindings.insert((query_index, post_asap_node), query_node); } }, - ) + ) { + // A selected state without a binding, or computation Planner + // cannot compile over readouts, leaves no local plan. + Err(crate::query_plan::QueryPlanError::UnsupportedNode(_)) + if request.allow_mixed_summary_and_exact_execution => + { + query_node_bindings.retain(|(index, _), _| *index != query_index); + Ok(exact_query_entry(query, canonical.clone(), instant)) + } + other => other, + } }?; - if request.allow_mixed_summary_and_exact_execution - && placement.raw_program(query_index).is_none() - { - // Any Planner-selected leaf without a physical summary binding - // is an exact subtree boundary. Deployed plans never retain a - // backend-local range index leaf. - crate::query_plan::query_time::finalize_query_time_nodes(&mut entry)?; - } if frontend == QueryFrontend::MetricsQl { entry.language = crate::query_plan::QueryLanguage::MetricsQl; } @@ -2433,14 +2431,6 @@ impl DeploymentPlanCompiler { .as_ref() .map(|candidate| &candidate.query), )?; - let linked = crate::query_plan::physical_values::compile(&mut entry)?; - for ((index, _), node) in &mut query_node_bindings { - if *index == query_index { - if let Some(root) = linked.get(node) { - *node = *root; - } - } - } let catalog_key = QueryPlan::catalog_key(entry.language, &canonical); if query_entries.insert(catalog_key, entry).is_some() { return Err(CompileError::Query { @@ -2717,6 +2707,31 @@ impl DeploymentPlanCompiler { /// Native query-time execution of a query that keeps no state: each raw input /// of the retained program is read by its range-selector `Scan`. +/// The whole query forwarded unchanged to the external exact engine. +fn exact_query_entry( + query: &QueryCompilationInput, + canonical: String, + instant: InstantExecution, +) -> QueryPlanEntry { + let root = crate::query_plan::QueryNodeId(0); + QueryPlanEntry { + physical_dag: None, + language: crate::query_plan::QueryLanguage::PromQl, + query_id: query.query_id.clone(), + canonical_query: canonical, + fixed_evaluation: None, + root, + nodes: BTreeMap::from([( + root, + crate::query_plan::QueryPlanNode::ExactFallback { + reason: "no stored input covers this query".into(), + }, + )]), + instant, + fallback: FallbackPolicy::ExactBackend, + } +} + fn raw_query_time_entry( query: &QueryCompilationInput, canonical: String, @@ -4919,30 +4934,6 @@ pub(crate) mod tests { assert!(error.to_string().contains("row-update executor"), "{error}"); } - // Compiler preserves the Planner's conditional-average execution guard. - #[test] - fn temporal_average_lowers_with_finite_division_guard() { - let mut environment = environment(10_000); - environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; - environment.target_collector_ids.clear(); - let request = request("average", "avg_over_time(a[1m])"); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert!(plan - .query_plan - .entries - .values() - .flat_map(|entry| entry.nodes.values()) - .flat_map( - |node| crate::query_plan::physical_values::operator_parameters( - node, - "VectorBinary" - ) - ) - .any(|parameters| parameters["operator"]["checked_finite_division"] == true)); - } - // The Planner's minimum state lowers without reconstructing direction from text. #[test] fn minimum_retains_its_typed_direction() { @@ -5012,7 +5003,7 @@ pub(crate) mod tests { fn complete_cost_selection_preserves_shared_sum_panes() { let mut snapshot = planning_snapshot(); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; - entry.query = Query("sum_over_time(a[1m]) / sum_over_time(a[10m])".into()); + entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))".into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); let plan = quoted_snapshot(snapshot, crate::physical::compiler::QueryFrontend::PromQl) .compile_promql() @@ -5088,7 +5079,6 @@ pub(crate) mod tests { #[test] fn issue_701_702_temporal_workloads_have_warm_candidates() { for text in [ - "avg_over_time(data[5m])", "min_over_time(data[5m])", "quantile_over_time(0.9,data[5m])", ] { @@ -5125,6 +5115,33 @@ pub(crate) mod tests { } } + // avg_over_time divides two per-series readouts. Planner does not yet + // match per-series rows in a Binary, so no candidate keeps local state and + // the exact engine evaluates the query. + #[test] + fn per_series_average_has_no_warm_candidate_until_planner_matches_series() { + let mut snapshot = planning_snapshot(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("avg_over_time(data[5m])".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + for candidate in + super::super::workload_cost::enumerate_exact_and_materialized_candidates(request) + .unwrap() + { + let Ok(plan) = DeploymentPlanCompiler.compile_promql(candidate, environment.clone()) + else { + continue; + }; + assert!(plan.precompute_plan.materializations.is_empty()); + assert!(plan + .query_plan + .entries + .values() + .all(|entry| entry.materialization_bindings().is_empty())); + } + } + // Quantile rank error does not certify relative error of a quotient. #[test] fn uncertified_quantile_ratios_retain_exact_execution() { @@ -5141,13 +5158,13 @@ pub(crate) mod tests { .unwrap(); assert!(plan.precompute_plan.materializations.is_empty(), "{text}"); assert!( - plan.query_plan - .entries - .values() - .all(|entry| entry.nodes.values().any(|node| matches!( - node, - crate::query_plan::QueryPlanNode::ExactFallback { .. } - ))), + plan.query_plan.entries.values().all(|entry| { + entry.nodes.len() == 1 + && matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + ) + }), "{text}" ); } @@ -5443,9 +5460,10 @@ pub(crate) mod tests { assert!(plan.precompute_plan.materializations.is_empty()); } - // Native exact values remain explicit; unsupported heaps publish no stored state. + // A ranking over an exact PromQL subtree has no stored input: Prometheus + // evaluates the whole query and the backend computes nothing. #[test] - fn hybrid_rate_topk_preserves_the_original_exact_subquery() { + fn hybrid_rate_topk_forwards_the_whole_exact_query() { let query = "topk(2, sum by (job) (rate(m[1m])))"; let evidence = TopKMembershipEvidence { selected_lower_bound: 101.0, @@ -5464,23 +5482,12 @@ pub(crate) mod tests { .compile_promql(request, environment) .unwrap(); let entry = plan.query_plan.lookup(query).unwrap(); - use crate::query_plan::{query_time::QueryTimeOperator, QueryPlanNode}; - let limits = crate::query_plan::physical_values::operator_parameters( + use crate::query_plan::QueryPlanNode; + assert_eq!(entry.nodes.len(), 1); + assert!(matches!( &entry.nodes[&entry.root], - "Limit", - ); - assert_eq!(limits.len(), 1); - assert_eq!(limits[0]["n"], 2); - assert_eq!( - entry - .nodes - .values() - .filter(|node| matches!(node, QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, .. - } if query == "sum by (job) (rate(m[1m]))")) - .count(), - 1 - ); + QueryPlanNode::ExactFallback { .. } + )); assert!(entry.materialization_bindings().is_empty()); assert!(plan.precompute_plan.materializations.is_empty()); let artifact = plan.to_publication_artifact().unwrap(); @@ -6167,19 +6174,26 @@ pub(crate) mod tests { } #[test] - fn metricsql_counter_gate_rejects_mixed_input_snapshots() { - let mut workload = request("mixed", "max_over_time(m[1m]) + rate(m[1m])"); + fn metricsql_counter_gate_forwards_mixed_operands_whole() { + // A counter branch that MetricsQL cannot read from stored state leaves + // no local operand; the exact engine evaluates the original query. + let query = "max_over_time(m[1m]) + rate(m[1m])"; + let mut workload = request("mixed", query); workload.allow_mixed_summary_and_exact_execution = true; let mut deployment = environment(10_000); deployment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; deployment.target_collector_ids.clear(); - let Err(error) = DeploymentPlanCompiler.compile_metricsql(workload, deployment) else { - panic!("mixed local/external snapshots must fail deployment binding"); - }; - assert!( - error.to_string().contains("common snapshot proof"), - "{error}" - ); + let plan = DeploymentPlanCompiler + .compile_metricsql(workload, deployment) + .unwrap(); + let entry = plan.query_plan.entries.values().next().unwrap(); + assert_eq!(entry.language, crate::query_plan::QueryLanguage::MetricsQl); + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + )); + assert_eq!(entry.canonical_query, query); + assert!(entry.materialization_bindings().is_empty()); } #[test] @@ -6787,18 +6801,13 @@ pub(crate) mod tests { ); assert!(!entry.materialization_bindings().is_empty()); } + // The grouped division is a Planner join over the two readouts. assert!(bundle .query_plan .entries .values() .flat_map(|entry| entry.nodes.values()) - .any( - |node| !crate::query_plan::physical_values::operator_parameters( - node, - "VectorBinary" - ) - .is_empty() - )); + .any(|node| !crate::query_plan::operator_parameters(node, "Join").is_empty())); } #[test] @@ -7233,7 +7242,7 @@ pub(crate) mod tests { for interval in [10_000, 60_000] { for (rhs, expected_states) in [("a", 1), ("b", 2), ("a{job=\"x\"}", 2)] { let mut snapshot = planning_snapshot(); - let query = format!("sum_over_time(a[1m]) / sum_over_time({rhs}[10m])"); + let query = format!("sum(sum_over_time(a[1m])) / sum(sum_over_time({rhs}[10m]))"); let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); @@ -7319,7 +7328,7 @@ pub(crate) mod tests { #[test] fn explicit_window_quotes_are_not_repriced_for_sharing() { let mut snapshot = planning_snapshot(); - let query = "sum_over_time(a[1m]) / sum_over_time(a[10m])"; + let query = "sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))"; let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; entry.query = Query(query.into()); entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); diff --git a/control_plane/src/physical/compiler/placement.rs b/control_plane/src/physical/compiler/placement.rs index 68034522c..abb1e1cb9 100644 --- a/control_plane/src/physical/compiler/placement.rs +++ b/control_plane/src/physical/compiler/placement.rs @@ -201,31 +201,11 @@ pub(super) fn place( .and_then(|root| raw_query_time_program(root).ok()) }) .collect(); - // A leaf the query-time lowering can externalize reads Prometheus directly. - let externalizable = |state: &SummaryNode, query: usize| { - raw_bindable - && (crate::query_plan::query_time::selected_counter_materialization( - &queries[query].query_string, - state, - ) - .ok() - .flatten() - .is_some() - || crate::query_plan::query_time::selected_range_max_materialization( - &queries[query].query_string, - state, - ) - .ok() - .flatten() - .is_some()) - }; let horizon = first.summary_lifecycle_inputs.horizon_seconds; let mut ephemeral = vec![false; states.len()]; let mut decisions = Vec::new(); for (state_index, (state, consumers)) in states.iter().enumerate() { - let bindable = consumers - .iter() - .all(|&query| raw_programs[query].is_some() || externalizable(state, query)); + let bindable = consumers.iter().all(|&query| raw_programs[query].is_some()); let lead = &queries[consumers[0]].summary_lifecycle_inputs; let interval = consumers .iter() @@ -283,8 +263,7 @@ pub(super) fn place( let mut changed = false; for (query, owned) in query_states.iter().enumerate() { let realizable = owned.iter().all(|state| { - index_of(state).is_some_and(|i| ephemeral[i]) - && (raw_programs[query].is_some() || externalizable(state, query)) + index_of(state).is_some_and(|i| ephemeral[i]) && raw_programs[query].is_some() }); if realizable { continue; diff --git a/control_plane/src/physical/plan_dot.rs b/control_plane/src/physical/plan_dot.rs index ba2033fba..1d97ed935 100644 --- a/control_plane/src/physical/plan_dot.rs +++ b/control_plane/src/physical/plan_dot.rs @@ -205,9 +205,6 @@ fn query_node_label(node: &QueryPlanNode) -> String { QueryPlanNode::Logical { operator, .. } => { format!("Logical\n{}", query_time_label(operator)) } - QueryPlanNode::Scalar { value } => format!("Scalar\n{value}"), - QueryPlanNode::Binary { operator, .. } => format!("Binary\n{operator:?}"), - QueryPlanNode::ReduceSum { .. } => "ReduceSum".into(), QueryPlanNode::ReadMaterialization { binding } => format!( "ReadMaterialization\n{}\nwindow={}ms\nlookback={:?}", binding.materialization.fingerprint(), @@ -228,15 +225,6 @@ fn query_time_label(operator: &QueryTimeOperator) -> &'static str { QueryTimeOperator::ExactSubquery { .. } => "ExactSubquery", QueryTimeOperator::CandidateExactSubquery { .. } => "CandidateExactSubquery", QueryTimeOperator::Scan { .. } => "Scan", - QueryTimeOperator::UnaryNegate => "UnaryNegate", - QueryTimeOperator::VectorToScalar => "VectorToScalar", - QueryTimeOperator::Aggregate { .. } => "Aggregate", - QueryTimeOperator::Limit { .. } => "Limit", - QueryTimeOperator::Binary { .. } => "Binary", - QueryTimeOperator::Temporal { .. } => "Temporal", - QueryTimeOperator::Sort { .. } => "Sort", - QueryTimeOperator::HistogramQuantile => "HistogramQuantile", - QueryTimeOperator::Subquery { .. } => "Subquery", } } diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index e5740894d..56d3b5d8a 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -1360,7 +1360,7 @@ mod tests { .store_per_byte_second = store; let q = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; q.query = planner_types::workload::Query( - "max_over_time(a[1m]) + max_over_time(b[1m])".into(), + "sum(max_over_time(a[1m])) + sum(max_over_time(b[1m]))".into(), ); q.requirements.accuracy = planner_types::workload::AccuracyRequirement::Explicit( crate::types::AccuracyTarget::Exact, diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 3cf562d80..f0d57e03d 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -1,9 +1,12 @@ //! Control-plane lowering from Planner IR to the shared installed query DAG. //! Serving consumes asap_types::query_plan; compilation stays in this component. +//! +//! The backend lowers only stored-state readouts itself. Query-time +//! computation over their decoded values is one Planner-compiled physical DAG +//! per maximal computation region. use asap_types::physical_plan_codec::PhysicalPlanCodec; mod clickhouse_exact; -pub mod physical_values; pub mod query_time; pub use asap_types::query_plan::*; @@ -37,7 +40,6 @@ where nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, - logical_source: None, preserve_relational: false, lowered: Some(&mut lowered), }; @@ -78,7 +80,6 @@ where nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, - logical_source: None, preserve_relational: true, lowered: Some(&mut lowered), }; @@ -96,123 +97,207 @@ where }) } -/// Compile a composable query while exposing the stable mapping from -/// Planner semantic nodes to installed query nodes. The control-plane -/// physical compiler uses this to persist backend placement without -/// relying on pointer values or reconstructing query shape later. -pub fn compile_bound_composable_mapped( - query_id: String, - canonical_query: String, - root: &Rc, - instant: InstantExecution, - fallback: FallbackPolicy, - mut bind: F, - mut lowered: G, -) -> Result -where - F: FnMut( - &Rc, - &SummaryFamilyType, - ) -> Result, - G: FnMut(&Rc, QueryNodeId), -{ - let mut compiler = DagCompiler { - next_id: 0, - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - bind: &mut bind, - logical_source: Some(canonical_query.clone()), - preserve_relational: false, - lowered: Some(&mut lowered), - }; - let root = compiler.lower(root)?; - let mut entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: compiler.nodes, - instant, - fallback, - }; - query_time::finalize_query_time_nodes(&mut entry)?; - Ok(entry) +/// Query-time computation over stored readouts, compiled once by Planner. +pub(crate) struct QueryComputation { + pub(crate) physical: asap_physical_operators::physical_planner::CompiledPhysicalDag, + /// Readout feeding each physical input contract, keyed by contract ID. + frontier: BTreeMap>, + computed: Vec>, + pruning: Option, } -fn compile_native_fragment( +/// Is this node decoded from stored state by the backend, rather than +/// computed over values? An exact aggregate over values is a query-time +/// reduction; a sketch over values is a derived stored output. +fn stored_readout(node: &SummaryNode) -> bool { + match &node.expr { + SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + .. + } => exact_accumulator_value_source(node) + .is_some_and(|source| !std::ptr::eq(source, node) && stored_readout(source)), + SummaryExpr::SummaryEstimate { .. } + | SummaryExpr::SummaryMerge { .. } + | SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } => true, + SummaryExpr::SummaryAgg { child, family, .. } => { + matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + || !matches!(family, SummaryFamilyType::ExactAggregate(..)) + } + _ => false, + } +} + +/// Does lowering `node` start a query-time computation region? +pub(crate) fn is_query_computation(node: &SummaryNode) -> bool { + !matches!( + node.expr, + SummaryExpr::KeepPreAsap(_) + | SummaryExpr::ValueOperation { + operation: planner_types::post_asap::ValueOperation::MaintainPopulation { .. } + | planner_types::post_asap::ValueOperation::ReadPopulation { .. }, + .. + } + ) && !stored_readout(node) +} + +/// Compile the maximal query-time region rooted at `root`. Its inputs are the +/// stored readouts below it; an exact PromQL selector inside the region has no +/// stored input, so the region is unsupported and the query runs exactly. +pub(crate) fn compile_query_computation( root: &Rc, - query_inputs: &[QueryNodeId], -) -> Result { - use asap_physical_operators::physical_planner::{compile, InputContract}; - use planner_types::post_asap::{compile_post_asap_dag, EdgeRole}; - let invalid = |e: String| QueryPlanError::Invalid(e); - let dag = compile_post_asap_dag(root).map_err(|e| invalid(e.to_string()))?; - let mut edges = dag - .edges - .iter() - .filter(|e| e.consumer == dag.root) - .collect::>(); - edges.sort_by_key(|e| match e.role { - EdgeRole::Left => 0, - EdgeRole::Input => 1, - EdgeRole::Right => 2, - }); - if edges.len() != query_inputs.len() { - return Err(invalid("physical frontier arity mismatch".into())); +) -> Result { + use asap_physical_operators::physical_planner::{compile, promql_fallback, InputContract}; + use planner_types::post_asap::{ + compile_post_asap_dag_with_node_ids, EdgeRole, PostAsapOperatorPayload as Payload, + }; + let unsupported = |message: String| QueryPlanError::UnsupportedNode(message); + let finalized = finalize_query_value(root); + let compilation = compile_post_asap_dag_with_node_ids(&finalized) + .map_err(|error| unsupported(error.to_string()))?; + let dag = &compilation.dag; + let mut pending = vec![dag.root]; + let mut visited = std::collections::BTreeSet::new(); + let mut contracts = BTreeMap::new(); + let mut frontier = BTreeMap::new(); + let mut computed = Vec::new(); + while let Some(id) = pending.pop() { + if !visited.insert(id) { + continue; + } + let node = dag + .nodes + .iter() + .find(|node| node.id == id) + .ok_or_else(|| unsupported("missing Planner node".into()))?; + let semantic = compilation.node_ids.summary_node(id); + let readout = id != dag.root && semantic.is_some_and(|semantic| stored_readout(semantic)); + if !readout { + if let Payload::Fallback { expression } = &node.payload { + if !promql_fallback::raw_series(expression) + .map_err(|error| unsupported(error.to_string()))? + .is_empty() + { + return Err(unsupported( + "query-time computation reads an exact PromQL selector".into(), + )); + } + } + if let Some(SummaryNode { + expr: SummaryExpr::BinaryOp { .. }, + guarantee, + .. + }) = semantic.map(Rc::as_ref) + { + // Planner certifies the combined value; without that, the + // backend must not publish the arithmetic result. + if guarantee.as_ref().is_none_or(|g| g.has_unknown()) { + return Err(unsupported( + "query-time arithmetic has no accuracy guarantee".into(), + )); + } + } + computed.extend(semantic.cloned()); + pending.extend( + dag.edges + .iter() + .filter(|edge| edge.consumer == id) + .map(|edge| edge.producer), + ); + continue; + } + let semantic = semantic.expect("readout has a semantic node"); + if matches!( + semantic.expr, + SummaryExpr::SummaryJoin { .. } + | SummaryExpr::SummarySubtract { .. } + | SummaryExpr::SummaryDelete { .. } + ) || node + .output_schema + .fields + .iter() + .any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))) + { + return Err(unsupported( + "query-time computation consumes state that has no decoded readout".into(), + )); + } + contracts.insert( + u64::from(id.0), + InputContract::bounded(std::sync::Arc::new(node.output_schema.clone())), + ); + frontier.insert(u64::from(id.0), Rc::clone(semantic)); } - let bindings = edges - .iter() - .zip(query_inputs) - .map(|(edge, &id)| (u64::from(edge.producer.0), id)) - .collect::>(); - let contracts = edges - .iter() - .map(|edge| { - ( - u64::from(edge.producer.0), - InputContract::bounded(std::sync::Arc::new(edge.intermediate_schema.clone())), - ) - }) - .collect(); - let physical = - compile(&dag, contracts, &[u64::from(dag.root.0)]).map_err(|e| invalid(e.to_string()))?; - let row_input = physical - .row_source(physical.roots()[0]) - .and_then(|source| physical.input_contracts().position(|(id, _)| id == source)); - let pruning = if let SummaryExpr::RelationalJoin { - left, - right, - pred, - pruning: Some(completeness), - .. - } = &root.expr - { - Some(asap_types::query_plan::PruningInputContract { - candidate_input: physical - .input_contracts() - .position(|(id, _)| bindings[&id] == query_inputs[1]) - .unwrap(), - keys: asap_physical_operators::physical_planner::equijoin_keys( - pred, - &left.schema, - &right.schema, - ) - .map_err(|e| invalid(e.to_string()))?, - completeness: completeness.clone(), - }) - } else { - None + let physical = compile(dag, contracts, &[u64::from(dag.root.0)]) + .map_err(|error| unsupported(error.to_string()))?; + let pruning = match &root.expr { + SummaryExpr::RelationalJoin { + left, + right, + pred, + pruning: Some(completeness), + .. + } => { + let candidates = dag + .edges + .iter() + .find(|edge| edge.consumer == dag.root && edge.role == EdgeRole::Right) + .map(|edge| u64::from(edge.producer.0)) + .ok_or_else(|| unsupported("pruned join has no candidate input".into()))?; + Some(PruningInputContract { + candidate_input: physical + .input_contracts() + .position(|(id, _)| id == candidates) + .ok_or_else(|| { + unsupported("pruning candidates must be a stored readout".into()) + })?, + keys: asap_physical_operators::physical_planner::equijoin_keys( + pred, + &left.schema, + &right.schema, + ) + .map_err(|error| unsupported(error.to_string()))?, + completeness: completeness.clone(), + }) + } + _ => None, }; - Ok(QueryPlanNode::PhysicalFragment { + Ok(QueryComputation { + physical, + frontier, + computed, pruning, - row_input, - inputs: physical - .input_contracts() - .map(|(id, _)| bindings[&id]) - .collect(), - dag: physical.encode().map_err(|e| invalid(e.to_string()))?, + }) +} + +/// A query-time exact aggregate yields accumulator state; its PromQL value is +/// that state finalized, as Planner does for stored exact states. +fn finalize_query_value(root: &Rc) -> Rc { + use planner_types::pre_asap::DataType; + if !matches!( + root.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(..), + .. + } + ) { + return Rc::clone(root); + } + let mut schema = root.schema.clone(); + for field in &mut schema.fields { + if matches!(field.dtype, SummaryFamilyType::ExactAggregate(..)) { + field.dtype = SummaryFamilyType::Plain(DataType::Float64); + } + } + Rc::new(SummaryNode { + expr: SummaryExpr::ValueOperation { + child: Rc::clone(root), + operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, + timing: planner_types::post_asap::ExecutionTiming::QueryTime, + }, + schema, + guarantee: root.guarantee.clone(), }) } @@ -221,7 +306,6 @@ struct DagCompiler<'a, F> { nodes: BTreeMap, seen: BTreeMap, bind: &'a mut F, - logical_source: Option, preserve_relational: bool, lowered: Option<&'a mut dyn FnMut(&Rc, QueryNodeId)>, } @@ -324,48 +408,35 @@ where Ok(id) } - fn graft( - &mut self, - id: QueryNodeId, - root: QueryNodeId, - nodes: BTreeMap, - ) -> Result { - let mut remap = BTreeMap::new(); - for local in nodes.keys() { - let global = if *local == root { - id - } else { - let next = QueryNodeId(self.next_id); - self.next_id += 1; - next - }; - remap.insert(*local, global); + /// One Planner physical fragment for the region, bound to backend readouts. + fn lower_computation(&mut self, root: &Rc) -> Result { + let computation = compile_query_computation(root)?; + let mut inputs = Vec::new(); + for (id, _) in computation.physical.input_contracts() { + inputs.push(self.lower(&computation.frontier[&id])?); } - for (local, mut physical) in nodes { - match &mut physical { - QueryPlanNode::Physical { inputs, .. } - | QueryPlanNode::PhysicalRelation { inputs, .. } - | QueryPlanNode::PhysicalFragment { inputs, .. } - | QueryPlanNode::Logical { inputs, .. } - | QueryPlanNode::SummaryMerge { inputs } - | QueryPlanNode::ExternalExact { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::Binary { inputs, .. } => { - for input in inputs { - *input = remap[input]; - } - } - QueryPlanNode::SummaryEstimate { input, .. } - | QueryPlanNode::ExactReadout { input, .. } - | QueryPlanNode::ReduceSum { input, .. } => *input = remap[input], - QueryPlanNode::Scalar { .. } - | QueryPlanNode::ReadMaterialization { .. } - | QueryPlanNode::ExactFallback { .. } => {} + let physical = &computation.physical; + let row_input = physical + .row_source(physical.roots()[0]) + .and_then(|source| physical.input_contracts().position(|(id, _)| id == source)); + let id = QueryNodeId(self.next_id); + self.next_id += 1; + self.nodes.insert( + id, + QueryPlanNode::PhysicalFragment { + inputs, + dag: physical + .encode() + .map_err(|e| QueryPlanError::Invalid(e.to_string()))?, + row_input, + pruning: computation.pruning, + }, + ); + for semantic in std::iter::once(root).chain(&computation.computed) { + self.seen.insert(Rc::as_ptr(semantic) as usize, id); + if let Some(lowered) = &mut self.lowered { + lowered(semantic, id); } - self.nodes.insert(remap[&local], physical); } Ok(id) } @@ -396,31 +467,12 @@ where } return Ok(child_id); } + if !self.preserve_relational && is_query_computation(node) { + return self.lower_computation(node); + } let id = QueryNodeId(self.next_id); self.next_id += 1; self.seen.insert(identity, id); - let query_time = match (&self.logical_source, &node.expr) { - (Some(original), SummaryExpr::KeepPreAsap(expr)) => { - Some(query_time::query_time_nodes(original, expr)?) - } - (Some(original), SummaryExpr::SummaryAgg { child, .. }) - if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && !matches!( - crate::physical::compiler::raw_materialization_input_contract(node), - Ok((_, Some(_), _)) - ) => - { - Some(query_time::selected_query_time_nodes(original, node)?) - } - _ => None, - }; - if let Some((root, nodes)) = query_time { - let id = self.graft(id, root, nodes)?; - if let Some(lowered) = &mut self.lowered { - lowered(node, id); - } - return Ok(id); - } let physical = match &node.expr { SummaryExpr::RelationalJoin { .. } if self.preserve_relational => { @@ -441,205 +493,12 @@ where { return self.lower_relation(node); } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Exact( - planner_types::post_asap::ExactOperation::Aggregate { - having: None, .. - }, - ), - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - let input = self.lower(child)?; - compile_native_fragment(node, &[input])? - } - SummaryExpr::ValueOperation { - child, - operation: - planner_types::post_asap::ValueOperation::Limit { .. } - | planner_types::post_asap::ValueOperation::Sort { .. } - | planner_types::post_asap::ValueOperation::Filter { .. }, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } => { - let input = self.lower(child)?; - compile_native_fragment(node, &[input])? - } SummaryExpr::ValueOperation { .. } => QueryPlanNode::ExactFallback { reason: "unsupported post-ASAP value operation".into(), }, - SummaryExpr::RelationalJoin { - right: candidates, - left: values, - kind: planner_types::pre_asap::JoinKind::Semi, - pruning, - .. - } => { - let candidate_input = self.lower(candidates)?; - let value_input = if let Some(original) = - self.logical_source.as_ref().filter(|_| pruning.is_some()) - { - let exact_expression = - query_time::selected_native_expression(original, values)?; - let value_id = QueryNodeId(self.next_id); - self.next_id += 1; - self.nodes.insert( - value_id, - QueryPlanNode::ExternalExact { - request: ExternalExactRequest { - language: QueryLanguage::PromQl, - expression: exact_expression.to_string(), - output: ExternalExactOutput::InstantVector, - parameters: BTreeMap::new(), - start_parameter: None, - end_parameter: None, - // The external source provides values; the Planner DAG owns matching. - input_contracts: vec![], - }, - inputs: vec![], - }, - ); - value_id - } else { - self.lower(values)? - }; - compile_native_fragment(node, &[value_input, candidate_input])? - } SummaryExpr::RelationalJoin { .. } => QueryPlanNode::ExactFallback { reason: "unsupported join in vector adapter".into(), }, - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic(_) - ) && operator.vector_match.is_none() - && !operator.checked_relative_division - && !operator.checked_finite_division - && node.guarantee.as_ref().is_some_and(|g| !g.has_unknown()) - && [lhs, rhs].iter().all(|operand| { - matches!( - operand.expr, - SummaryExpr::SummaryEstimate { - query: planner_types::post_asap::SketchQuery::Quantile { .. }, - .. - } - ) - }) => - { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(kind) = &operator.kind else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: kind.clone(), - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if self.logical_source.is_some() - || operator.checked_relative_division - || operator.checked_finite_division => - { - let operator = query_time::binary_operator(operator)?; - QueryPlanNode::Logical { - operator, - inputs: vec![self.lower(lhs)?, self.lower(rhs)?], - } - } - - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(kind, _), - child, - reduction, - .. - } if self.logical_source.is_some() - && !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => - { - if !matches!( - kind, - planner_types::post_asap::ExactKind::Sum - | planner_types::post_asap::ExactKind::Count - ) { - let operator = query_time::selected_aggregate_operator( - self.logical_source.as_deref().unwrap(), - node, - )?; - let input = self.lower(child)?; - self.nodes.insert( - id, - QueryPlanNode::Logical { - operator, - inputs: vec![input], - }, - ); - return Ok(id); - } - let operation = match kind { - planner_types::post_asap::ExactKind::Sum => query_time::Aggregation::Sum, - planner_types::post_asap::ExactKind::Count => query_time::Aggregation::Count, - _ => { - return Err(QueryPlanError::Invalid( - "unsupported aggregation over selected summary values".into(), - )) - } - }; - let keys = reduction.group_keys().ok_or_else(|| { - QueryPlanError::Invalid( - "per-entity summary reduction requires a temporal operator".into(), - ) - })?; - let labels = keys - .keys() - .iter() - .map(|&column| { - child - .schema - .fields - .get(column) - .map(|field| field.name.clone()) - .ok_or_else(|| { - QueryPlanError::Invalid("unresolved logical grouping column".into()) - }) - }) - .collect::, _>>()?; - QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Aggregate { - operation, - grouping: query_time::Grouping { - labels, - without: keys.is_without(), - }, - }, - inputs: vec![self.lower(child)?], - } - } - SummaryExpr::BinaryOp { - lhs, - rhs, - operator, - timing: planner_types::post_asap::ExecutionTiming::QueryTime, - } if exact_value_executable(node) => { - let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind - else { - unreachable!() - }; - QueryPlanNode::Binary { - inputs: [self.lower(lhs)?, self.lower(rhs)?], - operator: operator.clone(), - } - } - SummaryExpr::KeepPreAsap(expr) if scalar_literal(expr).is_some() => { - QueryPlanNode::Scalar { - value: scalar_literal(expr).unwrap(), - } - } SummaryExpr::KeepPreAsap(expr) if self.preserve_relational => { let mut expression = clickhouse_exact::render(expr).map_err(QueryPlanError::UnsupportedNode)?; @@ -678,20 +537,6 @@ where inputs: Vec::new(), } } - SummaryExpr::SummaryAgg { - family: - SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), - child, - reduction, - .. - } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && exact_value_executable(node) => - { - QueryPlanNode::ReduceSum { - input: self.lower(child)?, - grouping: physical_grouping(reduction, child)?, - } - } SummaryExpr::SummaryAgg { child, family, @@ -743,17 +588,10 @@ where .. } => match family { SummaryFamilyType::ExactAggregate(..) | SummaryFamilyType::Sketch(..) => { - let mut binding = match (self.bind)(node, family) { - Ok(binding) => binding, - Err(error) => { - if let Some(original) = &self.logical_source { - let (root, nodes) = - query_time::selected_query_time_nodes(original, node)?; - return self.graft(id, root, nodes); - } - return Err(error); - } - }; + // A selected state without a deployed binding leaves the + // query to the exact engine. + let mut binding = (self.bind)(node, family) + .map_err(|error| QueryPlanError::UnsupportedNode(error.to_string()))?; binding.output_grouping = physical_grouping(reduction, child)?; if let Some(readout) = exact_readout(family) { let existing = self.nodes.iter().find_map(|(id, node)| { @@ -814,6 +652,28 @@ where } } +/// Parameters of each `kind` operator inside a physical fragment, for tests. +#[cfg(test)] +pub(crate) fn operator_parameters(node: &QueryPlanNode, kind: &str) -> Vec { + let QueryPlanNode::PhysicalFragment { dag, .. } = node else { + return vec![]; + }; + asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag).unwrap(); + let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); + document["nodes"] + .as_object() + .unwrap() + .values() + .filter_map(|node| { + node.get("Operator")? + .get("operator")? + .get("kind")? + .get(kind) + .cloned() + }) + .collect() +} + fn exact_readout(family: &SummaryFamilyType) -> Option { use planner_types::post_asap::ExactKind; match family { @@ -1175,9 +1035,11 @@ mod catalog_binding_tests { mod tests { use super::*; + // Planner's guarded average divides two per-series readouts; until + // Planner matches per-series rows, lowering refuses it instead of + // computing it in the backend. #[test] - fn guarded_division_retains_checks_in_both_query_compilers() { - // Both compilers retain the finite/relative guard supplied by Planner. + fn per_series_guarded_division_is_not_lowered_locally() { let query = "avg_over_time(m[5m])"; let canonical = crate::query_parser::parse_query_expr_canonical( query, @@ -1189,77 +1051,88 @@ mod tests { panic!("expected the Planner's average rewrite"); }; assert!(operator.checked_finite_division); - for relative in [false, true] { - let mut guarded = root.as_ref().clone(); - let SummaryExpr::BinaryOp { operator, .. } = &mut guarded.expr else { - unreachable!(); - }; - operator.checked_finite_division = !relative; - operator.checked_relative_division = relative; - let guarded = Rc::new(guarded); - for composable in [false, true] { - let instant = InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }; - let bind = |_: &Rc, _: &SummaryFamilyType| { - Ok(MaterializationBinding { - full_window_slide_ms: None, - materialization: PolicyFingerprint(7).into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - PolicyFingerprint(7).into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - item_labels: Vec::new(), - }) - }; - let entry = if composable { - compile_bound_composable_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) - } else { - compile_bound_mapped( - "guarded".into(), - query.into(), - &guarded, - instant, - FallbackPolicy::ExactBackend, - bind, - |_, _| {}, - ) + let error = compile_bound_mapped( + "guarded".into(), + query.into(), + &root, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::ExactBackend, + |_: &Rc, _: &SummaryFamilyType| { + Ok(MaterializationBinding { + full_window_slide_ms: None, + materialization: PolicyFingerprint(7).into(), + stored_output_reference: asap_types::sds::StoredOutputReference::for_output( + PolicyFingerprint(7).into(), + ), + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + item_labels: Vec::new(), + }) + }, + |_, _| {}, + ) + .unwrap_err(); + assert!( + matches!(error, QueryPlanError::UnsupportedNode(_)), + "{error}" + ); + } + + // Every o11y corpus query compiles as backend readouts under Planner + // fragments, or forwards whole; no backend value operator remains. + #[test] + fn o11y_corpus_computes_only_through_planner_fragments() { + use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../tests/fixtures/o11y_queries.json")).unwrap(); + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let template = fixture["query_workload"]["repeating_queries"][0].clone(); + let queries = corpus["queries"] + .as_array() + .unwrap() + .iter() + .map(|row| row["query"].as_str().unwrap()) + .collect::>(); + fixture["query_workload"]["repeating_queries"] = queries + .iter() + .map(|query| { + let mut entry = template.clone(); + entry["query"] = (*query).into(); + entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); + entry + }) + .collect::>() + .into(); + let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); + let plan = DeploymentPlanCompiler + .compile_promql(request, environment) + .unwrap(); + assert_eq!(plan.query_plan.entries.len(), queries.len()); + let mut fragments = 0; + for entry in plan.query_plan.entries.values() { + for node in entry.nodes.values() { + match node { + QueryPlanNode::PhysicalFragment { .. } => fragments += 1, + QueryPlanNode::ReadMaterialization { .. } + | QueryPlanNode::ExactReadout { .. } + | QueryPlanNode::SummaryEstimate { .. } + | QueryPlanNode::SummaryMerge { .. } => {} + QueryPlanNode::ExactFallback { .. } => assert_eq!(entry.nodes.len(), 1), + other => panic!("{}: unexpected {other:?}", entry.canonical_query), } - .unwrap(); - let QueryPlanNode::Logical { - operator: query_time::QueryTimeOperator::Binary { operation, .. }, - .. - } = &entry.nodes[&entry.root] - else { - panic!( - "expected guarded division (composable={composable}): {:?}", - entry.nodes - ); - }; - assert_eq!( - *operation, - if relative { - query_time::BinaryOperation::CheckedDiv - } else { - query_time::BinaryOperation::FiniteDiv - } - ); - assert!(!entry.materialization_bindings().is_empty()); } } + assert!(fragments > 0); } #[test] diff --git a/control_plane/src/query_plan/physical_values.rs b/control_plane/src/query_plan/physical_values.rs deleted file mode 100644 index ec14b64cd..000000000 --- a/control_plane/src/query_plan/physical_values.rs +++ /dev/null @@ -1,445 +0,0 @@ -//! Retain Planner-compiled scalar/vector fragments before publishing an installed plan. -use super::*; -use asap_physical_operators::physical_planner::{promql_values as physical, CompiledPhysicalDag}; -use asap_types::physical_plan_codec::PhysicalPlanCodec; -use planner_types::{ - post_asap::BinaryOperator, - pre_asap::{AggIntent, ArithmeticOpKind, BinaryOpKind, ColumnRef, CompareOpKind, GroupKeys}, -}; -use query_time::{Aggregation, BinaryOperation, Grouping, QueryTimeOperator as Operation}; - -fn invalid(error: impl std::fmt::Display) -> QueryPlanError { - QueryPlanError::Invalid(error.to_string()) -} -fn grouping(value: &Grouping) -> GroupKeys { - let keys = value.labels.iter().cloned().map(ColumnRef::Named).collect(); - if value.without { - GroupKeys::without(keys) - } else { - GroupKeys::by(keys) - } -} -fn binary(operation: BinaryOperation) -> BinaryOperator { - use ArithmeticOpKind as A; - use BinaryOperation as O; - use CompareOpKind as C; - let kind = match operation { - O::Add => BinaryOpKind::Arithmetic(A::Add), - O::Sub => BinaryOpKind::Arithmetic(A::Sub), - O::Mul => BinaryOpKind::Arithmetic(A::Mul), - O::Div | O::CheckedDiv | O::FiniteDiv => BinaryOpKind::Arithmetic(A::Div), - O::Mod => BinaryOpKind::Arithmetic(A::Mod), - O::Pow => BinaryOpKind::Arithmetic(A::Pow), - O::Equal => BinaryOpKind::Compare(C::Eq), - O::NotEqual => BinaryOpKind::Compare(C::Ne), - O::Less => BinaryOpKind::Compare(C::Lt), - O::LessEqual => BinaryOpKind::Compare(C::Le), - O::Greater => BinaryOpKind::Compare(C::Gt), - O::GreaterEqual => BinaryOpKind::Compare(C::Ge), - }; - BinaryOperator { - kind, - vector_match: None, - checked_relative_division: operation == O::CheckedDiv, - checked_finite_division: operation == O::FiniteDiv, - } -} - -pub fn compile( - entry: &mut QueryPlanEntry, -) -> Result, QueryPlanError> { - if entry.language == QueryLanguage::ClickHouseSql || entry.physical_dag.is_some() { - return Ok(BTreeMap::new()); - } - let mut scalars = BTreeMap::new(); - for id in entry.topological_order()? { - let node = &entry.nodes[&id]; - let inputs = node.inputs().to_vec(); - let scalar_input = |position: usize| { - inputs - .get(position) - .and_then(|id| scalars.get(id)) - .copied() - .unwrap_or(false) - }; - let mut scalar = false; - let compiled = match node { - QueryPlanNode::Scalar { .. } => { - scalar = true; - let QueryPlanNode::Scalar { value } = node else { - unreachable!() - }; - Some(physical::compile_scalar(*value).map_err(invalid)?) - } - QueryPlanNode::Logical { - operator: - Operation::ExactSubquery { query } | Operation::CandidateExactSubquery { query, .. }, - .. - } => { - scalar = promql_parser::parser::parse(query) - .map_err(invalid)? - .value_type() - == promql_parser::parser::value::ValueType::Scalar; - None - } - QueryPlanNode::Logical { operator, .. } => match operator { - Operation::Binary { - operation, - return_bool, - } => { - scalar = scalar_input(0) && scalar_input(1); - Some( - physical::compile_binary( - &binary(*operation), - *return_bool, - scalar_input(0), - scalar_input(1), - ) - .map_err(invalid)?, - ) - } - Operation::UnaryNegate => { - scalar = scalar_input(0); - Some(physical::compile_negate(scalar).map_err(invalid)?) - } - Operation::VectorToScalar => { - scalar = true; - Some(physical::compile_vector_to_scalar().map_err(invalid)?) - } - Operation::Aggregate { - operation, - grouping: groups, - } => { - let intent = match operation { - Aggregation::Sum => AggIntent::Sum { col: None }, - Aggregation::Avg => AggIntent::Avg { col: None }, - Aggregation::Min => AggIntent::Min { col: None }, - Aggregation::Max => AggIntent::Max { col: None }, - Aggregation::Count => AggIntent::Count { - accuracy: planner_types::types::AccuracyTarget::Exact, - }, - }; - Some(physical::compile_aggregate(&intent, &grouping(groups)).map_err(invalid)?) - } - Operation::Sort { - descending, - grouping: groups, - } => Some(physical::compile_sort(*descending, &grouping(groups)).map_err(invalid)?), - Operation::Limit { - n, - offset, - grouping: groups, - } => { - Some(physical::compile_limit(*n, *offset, &grouping(groups)).map_err(invalid)?) - } - Operation::Temporal { operation } => { - use query_time::TemporalOperation as T; - let intent = match operation { - T::Rate => AggIntent::Rate, - T::Increase => AggIntent::Increase, - T::Sum => AggIntent::Sum { col: None }, - T::Avg => AggIntent::Avg { col: None }, - T::Min => AggIntent::Min { col: None }, - T::Max => AggIntent::Max { col: None }, - T::Count => AggIntent::Count { - accuracy: planner_types::types::AccuracyTarget::Exact, - }, - }; - let preserve = entry.language == QueryLanguage::MetricsQl - && matches!(operation, T::Min | T::Max | T::Avg); - Some(physical::compile_temporal(&intent, preserve).map_err(invalid)?) - } - Operation::HistogramQuantile => { - Some(physical::compile_histogram_quantile().map_err(invalid)?) - } - _ => None, - }, - QueryPlanNode::ReduceSum { - grouping: groups, .. - } => Some(match groups { - PhysicalGrouping::Reduce(labels) => physical::compile_aggregate( - &AggIntent::Sum { col: None }, - &GroupKeys::by(labels.iter().cloned().map(ColumnRef::Named).collect()), - ) - .map_err(invalid)?, - PhysicalGrouping::PerEntity => CompiledPhysicalDag::from_operators( - BTreeMap::from([( - 0, - asap_physical_operators::physical_planner::InputContract::bounded( - physical::vector_schema(), - ), - )]), - BTreeMap::from([( - 1, - ( - vec![0], - asap_physical_operators::operators::Operator::project( - physical::vector_schema(), - vec![ - ( - "labels".into(), - asap_physical_operators::expressions::Expression::Column(0), - ), - ( - "value".into(), - asap_physical_operators::expressions::Expression::Column(1), - ), - ], - ) - .map_err(invalid)?, - ), - )]), - vec![1], - ) - .map_err(invalid)?, - }), - QueryPlanNode::Binary { operator, .. } => { - scalar = scalar_input(0) && scalar_input(1); - Some( - physical::compile_binary( - &BinaryOperator { - kind: BinaryOpKind::Arithmetic(operator.clone()), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - false, - scalar_input(0), - scalar_input(1), - ) - .map_err(invalid)?, - ) - } - QueryPlanNode::PhysicalFragment { dag, .. } => { - let compiled = CompiledPhysicalDag::decode(dag).map_err(invalid)?; - scalar = compiled - .output_contract(compiled.roots()[0]) - .map_err(invalid)? - .schema - == physical::scalar_schema(); - None - } - _ => None, - }; - scalars.insert(id, scalar); - if let Some(compiled) = compiled { - entry.nodes.insert( - id, - QueryPlanNode::PhysicalFragment { - inputs, - dag: compiled.encode().map_err(invalid)?, - row_input: None, - pruning: None, - }, - ); - } - } - combine(entry) -} - -fn combine( - entry: &mut QueryPlanEntry, -) -> Result, QueryPlanError> { - use asap_physical_operators::physical_planner::InputContract; - use std::collections::BTreeSet; - let supported = |schema: &asap_physical_operators::values::Schema| { - schema == &physical::scalar_schema() - || schema == &physical::vector_schema() - || schema == &physical::matrix_schema() - }; - let mut programs = BTreeMap::new(); - for (&id, node) in &entry.nodes { - if let QueryPlanNode::PhysicalFragment { - dag, - row_input: None, - pruning: None, - .. - } = node - { - let graph = CompiledPhysicalDag::decode(dag).map_err(invalid)?; - if graph.roots().len() == 1 - && graph - .input_contracts() - .all(|(_, input)| supported(&input.schema)) - && supported( - &graph - .output_contract(graph.roots()[0]) - .map_err(invalid)? - .schema, - ) - { - programs.insert(id, graph); - } - } - } - let mut roots = BTreeSet::new(); - if programs.contains_key(&entry.root) { - roots.insert(entry.root); - } - for (id, node) in &entry.nodes { - if !programs.contains_key(id) { - roots.extend( - node.inputs() - .iter() - .filter(|id| programs.contains_key(id)) - .copied(), - ); - } - } - let reachable = |root: QueryNodeId, boundaries: &BTreeSet| { - let mut pending = vec![root]; - let mut seen = BTreeSet::new(); - while let Some(id) = pending.pop() { - if !programs.contains_key(&id) - || (id != root && boundaries.contains(&id)) - || !seen.insert(id) - { - continue; - } - pending.extend(entry.nodes[&id].inputs()); - } - seen - }; - // A producer consumed across an I/O boundary remains one separately scheduled - // physical output, rather than being duplicated into both downstream graphs. - let mut owners = BTreeMap::::new(); - for &root in &roots { - for id in reachable(root, &BTreeSet::new()) { - *owners.entry(id).or_default() += 1; - } - } - roots.extend( - owners - .into_iter() - .filter_map(|(id, count)| (count > 1).then_some(id)), - ); - let mut replacements = BTreeMap::new(); - let mut remap = BTreeMap::new(); - for &root in &roots { - let members = reachable(root, &roots); - let mut sources = BTreeMap::::new(); - let mut fragments = BTreeMap::new(); - for &id in &members { - let graph = &programs[&id]; - let inputs = entry.nodes[&id].inputs(); - for ((_, contract), input) in graph.input_contracts().zip(inputs) { - if !members.contains(input) - && sources - .insert(input.0, contract.clone()) - .is_some_and(|previous| previous.schema != contract.schema) - { - return Err(invalid("shared physical input has inconsistent schemas")); - } - } - fragments.insert( - id.0, - (inputs.iter().map(|id| id.0).collect(), graph.clone()), - ); - if id != root { - remap.insert(id, root); - } - } - let graph = - CompiledPhysicalDag::compose(sources, fragments, vec![root.0]).map_err(invalid)?; - replacements.insert( - root, - QueryPlanNode::PhysicalFragment { - inputs: graph - .input_contracts() - .map(|(id, _)| QueryNodeId(id)) - .collect(), - dag: graph.encode().map_err(invalid)?, - row_input: None, - pruning: None, - }, - ); - } - for id in remap.keys() { - entry.nodes.remove(id); - } - entry.nodes.extend(replacements); - Ok(remap) -} - -#[cfg(test)] -pub(crate) fn operator_parameters(node: &QueryPlanNode, kind: &str) -> Vec { - let QueryPlanNode::PhysicalFragment { dag, .. } = node else { - return vec![]; - }; - CompiledPhysicalDag::decode(dag).unwrap(); - let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); - document["nodes"] - .as_object() - .unwrap() - .values() - .filter_map(|node| { - node.get("Operator")? - .get("operator")? - .get("kind")? - .get(kind) - .cloned() - }) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - - // A sum over finalized summaries must be priced and installed as computation, - // never reconstructed by the query worker. - #[test] - fn finalized_summary_rollup_is_retained_as_a_physical_graph() { - for grouping in [ - PhysicalGrouping::PerEntity, - PhysicalGrouping::Reduce(vec!["service".into()]), - ] { - let mut entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "rollup".into(), - canonical_query: "sum by (service) (foo)".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([ - ( - QueryNodeId(0), - QueryPlanNode::Logical { - operator: Operation::ExactSubquery { - query: "foo".into(), - }, - inputs: vec![], - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::ReduceSum { - input: QueryNodeId(0), - grouping, - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 1000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::Reject, - }; - compile(&mut entry).unwrap(); - let QueryPlanNode::PhysicalFragment { dag, inputs, .. } = &entry.nodes[&entry.root] - else { - panic!("rollup was left for Backend execution"); - }; - let compiled = CompiledPhysicalDag::decode(dag).unwrap(); - assert_eq!(inputs, &[QueryNodeId(0)]); - assert_eq!( - compiled - .output_contract(compiled.roots()[0]) - .unwrap() - .schema, - physical::vector_schema() - ); - } - } -} diff --git a/control_plane/src/query_plan/query_time.rs b/control_plane/src/query_plan/query_time.rs index 1d755d5d1..39434e519 100644 --- a/control_plane/src/query_plan/query_time.rs +++ b/control_plane/src/query_plan/query_time.rs @@ -1,1039 +1,14 @@ -//! The query-time half of a plan: everything a summary did not replace. -//! -//! Typed operations compiled once by the control plane, never parsed at -//! serving time. This is the counterpart to the precompute half, which runs -//! ahead of the query and ends at stored outputs. -use super::{ - FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanEntry, QueryPlanError, QueryPlanNode, -}; -#[cfg(test)] -use asap_types::physical_plan_codec::PhysicalPlanCodec; -use promql_parser::{ - label::MatchOp, - parser::{self, Expr, LabelModifier, Offset, VectorSelector}, -}; -use std::collections::BTreeMap; - +//! Installed query-time leaves: raw selectors, exact subtrees and +//! current-series readouts. Computation over their values is Planner-compiled. pub use asap_types::query_plan::query_time::*; -/// Stable identity of a Planner-authorized materializable DAG leaf. This is a -/// workload-selection key, not another physical materialization definition. -#[derive(Debug, Clone, serde::Serialize, PartialEq)] -struct MaterializationCandidateIdentity { - metric: String, - matchers: Vec, - range_ms: u64, - offset_ms: i64, - operation: TemporalOperation, -} - -fn invalid(message: impl Into) -> QueryPlanError { - QueryPlanError::Invalid(message.into()) -} -fn millis(duration: std::time::Duration) -> Result { - u64::try_from(duration.as_millis()).map_err(|_| invalid("logical duration overflow")) -} -fn offset(value: &Option) -> Result { - match value { - None => Ok(0), - Some(Offset::Pos(d)) => i64::try_from(millis(*d)?).map_err(|_| invalid("offset overflow")), - Some(Offset::Neg(d)) => i64::try_from(millis(*d)?) - .map(|v| -v) - .map_err(|_| invalid("offset overflow")), - } -} - -struct Lower { - nodes: BTreeMap, - seen: BTreeMap, -} -impl Lower { - fn add(&mut self, node: QueryPlanNode) -> Result { - let key = serde_json::to_string(&node).map_err(|e| invalid(e.to_string()))?; - if let Some(id) = self.seen.get(&key) { - return Ok(*id); - } - let id = QueryNodeId(self.nodes.len() as u64); - self.nodes.insert(id, node); - self.seen.insert(key, id); - Ok(id) - } - fn operation( - &mut self, - operator: QueryTimeOperator, - inputs: Vec, - ) -> Result { - operator.validate(inputs.len())?; - self.add(QueryPlanNode::Logical { operator, inputs }) - } - fn scan( - &mut self, - s: &VectorSelector, - range_ms: Option, - ) -> Result { - if s.at.is_some() || !s.matchers.or_matchers.is_empty() { - return Err(invalid("logical @/OR selector is not supported")); - } - let matchers = s - .matchers - .matchers - .iter() - .map(|m| LabelMatcher { - name: m.name.clone(), - value: m.value.clone(), - operation: match m.op { - MatchOp::Equal => LabelMatch::Equal, - MatchOp::NotEqual => LabelMatch::NotEqual, - MatchOp::Re(_) => LabelMatch::Regex, - MatchOp::NotRe(_) => LabelMatch::NotRegex, - }, - }) - .collect(); - self.operation( - QueryTimeOperator::Scan { - metric: s.name.clone(), - matchers, - range_ms, - offset_ms: offset(&s.offset)?, - }, - vec![], - ) - } - fn lower(&mut self, expr: &Expr) -> Result { - match expr { - Expr::NumberLiteral(n) if n.val.is_finite() => { - self.add(QueryPlanNode::Scalar { value: n.val }) - } - Expr::Paren(p) => self.lower(&p.expr), - Expr::Unary(u) => { - let input = self.lower(&u.expr)?; - self.operation(QueryTimeOperator::UnaryNegate, vec![input]) - } - Expr::VectorSelector(s) => self.scan(s, None), - Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), - Expr::Subquery(s) => { - if s.at.is_some() { - return Err(invalid("logical subquery @ is unsupported")); - } - let input = self.lower(&s.expr)?; - self.operation( - QueryTimeOperator::Subquery { - range_ms: millis(s.range)?, - // Prometheus uses its configured default evaluation - // interval when `[range:]` omits the resolution. The - // backend-local deployment uses the Prometheus default - // of one minute; unsupported subquery operands are - // externalized as one exact subtree before execution. - step_ms: millis( - s.step.unwrap_or_else(|| std::time::Duration::from_secs(60)), - )?, - offset_ms: offset(&s.offset)?, - }, - vec![input], - ) - } - Expr::Aggregate(a) => { - let grouping = match &a.modifier { - None => Grouping { - labels: vec![], - without: false, - }, - Some(LabelModifier::Include(labels)) => Grouping { - labels: labels.labels.clone(), - without: false, - }, - Some(LabelModifier::Exclude(labels)) => Grouping { - labels: labels.labels.clone(), - without: true, - }, - }; - if a.op.to_string() == "topk" { - let Some(Expr::NumberLiteral(parameter)) = a.param.as_deref() else { - return Err(invalid("topk requires a literal scalar parameter")); - }; - if !parameter.val.is_finite() { - return Err(invalid("topk requires a finite scalar parameter")); - } - // Prometheus converts the scalar parameter to int64 before - // selection. Values below one produce an empty vector. - let k = parameter.val as i64; - // Keep the selection node local even when its operand has - // unsupported syntax (for example a subquery with an - // implicit resolution). Prometheus evaluates that maximal - // instant-vector child; the backend still performs topk. - let nodes_before = self.nodes.clone(); - let seen_before = self.seen.clone(); - let input = match self.lower(&a.expr) { - Ok(input) => input, - Err(_) => { - self.nodes = nodes_before; - self.seen = seen_before; - self.operation( - QueryTimeOperator::ExactSubquery { - query: a.expr.to_string(), - }, - vec![], - )? - } - }; - let sorted = self.operation( - QueryTimeOperator::Sort { - descending: true, - grouping: grouping.clone(), - }, - vec![input], - )?; - return self.operation( - QueryTimeOperator::Limit { - n: u64::try_from(k).unwrap_or(0), - offset: 0, - grouping, - }, - vec![sorted], - ); - } - if a.param.is_some() { - return Err(invalid("unsupported parameterized aggregate")); - } - let operation = match a.op.to_string().as_str() { - "sum" => Aggregation::Sum, - "max" => Aggregation::Max, - "min" => Aggregation::Min, - "avg" => Aggregation::Avg, - "count" => Aggregation::Count, - other => return Err(invalid(format!("unsupported logical aggregate {other}"))), - }; - let input = self.lower(&a.expr)?; - self.operation( - QueryTimeOperator::Aggregate { - operation, - grouping, - }, - vec![input], - ) - } - Expr::Call(c) => { - let operator = match c.func.name { - "scalar" => QueryTimeOperator::VectorToScalar, - "histogram_quantile" => QueryTimeOperator::HistogramQuantile, - "sort" => QueryTimeOperator::Sort { - descending: false, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - "sort_desc" => QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - name => QueryTimeOperator::Temporal { - operation: match name { - "rate" => TemporalOperation::Rate, - "increase" => TemporalOperation::Increase, - "avg_over_time" => TemporalOperation::Avg, - "max_over_time" => TemporalOperation::Max, - "min_over_time" => TemporalOperation::Min, - "sum_over_time" => TemporalOperation::Sum, - "count_over_time" => TemporalOperation::Count, - _ => { - return Err(invalid(format!("unsupported logical function {name}"))) - } - }, - }, - }; - let inputs = c - .args - .args - .iter() - .map(|e| self.lower(e)) - .collect::, _>>()?; - self.operation(operator, inputs) - } - Expr::Binary(b) => { - if b.modifier.as_ref().is_some_and(|m| { - m.matching.is_some() - || !matches!(m.card, parser::VectorMatchCardinality::OneToOne) - }) { - return Err(invalid("logical explicit vector matching unsupported")); - } - let operation = match b.op.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "==" => BinaryOperation::Equal, - "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => return Err(invalid(format!("unsupported logical binary {other}"))), - }; - let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; - self.operation( - QueryTimeOperator::Binary { - operation, - return_bool: b.return_bool(), - }, - inputs, - ) - } - _ => Err(invalid("unsupported logical expression")), - } - } -} - -/// Lower a Planner-authorized native fragment into typed backend operations. -/// Callers retain a separate external-native alternative for cost comparison. -pub fn compile_logical( - query_id: String, - canonical_query: String, - instant: InstantExecution, - fallback: FallbackPolicy, -) -> Result { - let expr = parser::parse(&canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expr)?; - let entry = QueryPlanEntry { - physical_dag: None, - language: super::QueryLanguage::PromQl, - query_id, - canonical_query, - fixed_evaluation: None, - root, - nodes: lower.nodes, - instant, - fallback, - }; - entry.validate(&Default::default())?; - Ok(entry) -} - -fn horizons(expr: &planner_types::pre_asap::QueryExpr, out: &mut Vec) { - use planner_types::pre_asap::QueryExpr; - if let QueryExpr::TimeRange { range, .. } = expr { - if let Ok(ms) = u64::try_from(range.as_millis()) { - out.push(ms); - } - } - match expr { - QueryExpr::PromqlScalarBridge(child) - | QueryExpr::PromqlVectorFromScalar(child) - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } - | QueryExpr::PromqlSeriesSample { child, .. } - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::Filter { child, .. } - | QueryExpr::Project { child, .. } - | QueryExpr::Aggregate { child, .. } - | QueryExpr::Dedup { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::TimeRange { child, .. } - | QueryExpr::TimeShift { child, .. } => horizons(child, out), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - horizons(lhs, out); - horizons(rhs, out); - } - QueryExpr::Join { left, right, .. } | QueryExpr::SetOp { left, right, .. } => { - horizons(left, out); - horizons(right, out); - } - _ => {} - } -} - -/// Match fragments by semantic IR equality, not display text or source names. -/// This ensures a subtree parsed for physical lowering is the subtree Planner kept. -/// Does this re-parsed subtree denote the same computation as the Planner -/// fragment? -/// -/// Not `==`, and deliberately so. Planner documents a PromQL leaf's `schema` as -/// usage-derived: "the `(ts, value)` floor + the labels the query references", -/// and marks it `closed: false` precisely because it does not enumerate the -/// row. A fragment resolved inside the whole query therefore carries every -/// label the *query* mentions, while the same fragment re-parsed on its own -/// carries only the labels *it* mentions. -/// -/// So `sum by (label_0) (rate(data[1m]))` yields a fragment whose leaf scan has -/// columns `[ts, value, label_0]`, while re-parsing the subtree `rate(data[1m])` -/// yields `[ts, value]`. Identical source, predicates, range, measures and -/// reduction; one extra column that the isolated parse had no way to know -/// about. Requiring equality there asks an isolated parse to reproduce -/// whole-query context, and every other part of the comparison is what actually -/// discriminates: a different matcher, range or metric still fails. -/// -/// Open leaf schemas are therefore compared by containment. Containment is a -/// *prefix*, not an arbitrary subset, because `ColumnId`s are positional: the -/// floor comes first and context only appends, so a prefix keeps every column -/// id in `predicates` and grouping keys meaning the same column on both sides. -/// Closed (catalog-backed SQL) schemas do enumerate the row, so they keep exact -/// equality. -fn fragment_matches( - candidate: &planner_types::pre_asap::QueryExpr, - fragment: &planner_types::pre_asap::QueryExpr, -) -> bool { - match ( - serde_json::to_value(candidate), - serde_json::to_value(fragment), - ) { - (Ok(candidate), Ok(fragment)) => same_modulo_open_leaf_schema(&candidate, &fragment), - // Fall back to the strict comparison rather than accepting anything we - // could not inspect. - _ => candidate == fragment, - } -} - -fn same_modulo_open_leaf_schema( - candidate: &serde_json::Value, - fragment: &serde_json::Value, -) -> bool { - use serde_json::Value; - match (candidate, fragment) { - (Value::Object(candidate), Value::Object(fragment)) => { - if is_open_schema(candidate) && is_open_schema(fragment) { - return open_schema_is_widened(candidate, fragment); - } - candidate.len() == fragment.len() - && candidate.iter().all(|(key, value)| { - fragment - .get(key) - .is_some_and(|other| same_modulo_open_leaf_schema(value, other)) - }) - } - (Value::Array(candidate), Value::Array(fragment)) => { - candidate.len() == fragment.len() - && candidate - .iter() - .zip(fragment) - .all(|(a, b)| same_modulo_open_leaf_schema(a, b)) - } - _ => candidate == fragment, - } -} - -fn is_open_schema(value: &serde_json::Map) -> bool { - value.get("closed") == Some(&serde_json::Value::Bool(false)) && value.contains_key("columns") -} - -/// The isolated parse's columns must be a prefix of the whole-query ones: -/// context appends the labels it references, it never removes or reorders the -/// floor. Every other schema field still has to agree exactly. -fn open_schema_is_widened( - candidate: &serde_json::Map, - fragment: &serde_json::Map, -) -> bool { - let (Some(narrow), Some(wide)) = ( - candidate.get("columns").and_then(|v| v.as_array()), - fragment.get("columns").and_then(|v| v.as_array()), - ) else { - return false; - }; - candidate - .iter() - .filter(|(key, _)| key.as_str() != "columns") - .all(|(key, value)| fragment.get(key) == Some(value)) - && narrow.len() <= wide.len() - && narrow.iter().zip(wide).all(|(a, b)| a == b) -} - -pub(super) fn query_time_nodes( - original: &str, - fragment: &planner_types::pre_asap::QueryExpr, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - fn visit<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { - out.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, out), - Expr::Unary(e) => visit(&e.expr, out), - Expr::Subquery(e) => visit(&e.expr, out), - Expr::Aggregate(e) => visit(&e.expr, out), - Expr::Binary(e) => { - visit(&e.lhs, out); - visit(&e.rhs, out); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, out); - } - } - _ => {} - } - } - let original = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&original, &mut expressions); - // Reconstruct equality witnesses with the selected IR's source horizon, - // not the compatibility parser's default. Explicit matrix ranges remain - // query-owned and equality still checks the complete tree. - let mut intervals = vec![1_000]; - horizons(fragment, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - // Accuracy annotations select a candidate, but exact execution still - // implements that candidate's computation. Reconstruct the same typed IR - // before comparing it; do not erase operators or source predicates. - let accuracy = match fragment { - planner_types::pre_asap::QueryExpr::Aggregate { measures, .. } => measures - .iter() - .find_map(|intent| { - use planner_types::pre_asap::AggIntent; - match intent { - AggIntent::Quantile { accuracy, .. } - | AggIntent::Cardinality { accuracy, .. } - | AggIntent::Count { accuracy } - | AggIntent::TopK { accuracy, .. } - | AggIntent::FrequencyL2 { accuracy, .. } - | AggIntent::FrequencyEntropy { accuracy, .. } => Some(accuracy.clone()), - _ => None, - } - }) - .unwrap_or(planner_types::types::AccuracyTarget::Exact), - _ => planner_types::types::AccuracyTarget::Exact, - }; - for expression in expressions { - for interval in &intervals { - if let Ok(candidate) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - accuracy.clone(), - *interval, - ) { - if fragment_matches(&candidate, fragment) { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(expression)?; - return Ok((root, lower.nodes)); - } - } - } - } - Err(invalid( - "Planner logical fragment does not match any original query subtree", - )) -} - -pub(super) fn binary_operator( - operator: &planner_types::post_asap::BinaryOperator, -) -> Result { - if operator.checked_relative_division || operator.checked_finite_division { - if (operator.checked_relative_division && operator.checked_finite_division) - || operator.vector_match.is_some() - || !matches!( - operator.kind, - planner_types::pre_asap::BinaryOpKind::Arithmetic( - planner_types::pre_asap::ArithmeticOpKind::Div - ) - ) - { - return Err(invalid("invalid Planner checked division contract")); - } - return Ok(QueryTimeOperator::Binary { - operation: if operator.checked_finite_division { - BinaryOperation::FiniteDiv - } else { - BinaryOperation::CheckedDiv - }, - return_bool: false, - }); - } - if operator.vector_match.is_some() { - return Err(invalid("explicit logical vector matching unsupported")); - } - let operation = match operator.kind.to_string().as_str() { - "+" => BinaryOperation::Add, - "-" => BinaryOperation::Sub, - "*" => BinaryOperation::Mul, - "/" => BinaryOperation::Div, - "%" => BinaryOperation::Mod, - "^" => BinaryOperation::Pow, - "=" | "==" => BinaryOperation::Equal, - "<>" | "!=" => BinaryOperation::NotEqual, - "<" => BinaryOperation::Less, - "<=" => BinaryOperation::LessEqual, - ">" => BinaryOperation::Greater, - ">=" => BinaryOperation::GreaterEqual, - other => { - return Err(invalid(format!( - "unsupported Planner binary operator {other}" - ))) - } - }; - Ok(QueryTimeOperator::Binary { - operation, - return_bool: false, - }) -} - -/// Prove a physical-native substitute represents exactly the selected summary leaf. -/// A second Planner invocation is an equality witness, not a replacement selection. -pub(crate) fn selected_query_time_nodes( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { - let expression = selected_native_expression(original, selected)?; - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let root = lower.lower(&expression)?; - Ok((root, lower.nodes)) -} - -/// Resolve the selected exact subtree to a verified native expression before -/// binding an external input. Never substitute the top-level query's child. -pub(super) fn selected_native_expression( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { - return Err(invalid( - "native subtree substitution requires an exact selected value", - )); - } - let selected = match &selected.expr { - planner_types::post_asap::SummaryExpr::ValueOperation { - child, - operation: planner_types::post_asap::ValueOperation::FinalizeExactAccumulator, - .. - } => child.as_ref(), - _ => selected, - }; - fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { - output.push(expr); - match expr { - Expr::Paren(e) => visit(&e.expr, output), - Expr::Unary(e) => visit(&e.expr, output), - Expr::Subquery(e) => visit(&e.expr, output), - Expr::Aggregate(e) => visit(&e.expr, output), - Expr::Binary(e) => { - visit(&e.lhs, output); - visit(&e.rhs, output); - } - Expr::Call(e) => { - for input in &e.args.args { - visit(input, output); - } - } - _ => {} - } - } - let parsed = parser::parse(original).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - visit(&parsed, &mut expressions); - fn selected_horizons(node: &planner_types::post_asap::SummaryNode, out: &mut Vec) { - use planner_types::post_asap::SummaryExpr; - match &node.expr { - SummaryExpr::KeepPreAsap(expr) => horizons(expr, out), - SummaryExpr::ValueOperation { child, .. } | SummaryExpr::SummaryAgg { child, .. } => { - selected_horizons(child, out) - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - selected_horizons(summary_input, out) - } - SummaryExpr::BinaryOp { - lhs: left, - rhs: right, - .. - } - | SummaryExpr::RelationalJoin { left, right, .. } - | SummaryExpr::SummaryJoin { - outer: left, - inner: right, - .. - } - | SummaryExpr::SummarySubtract { left, right } => { - selected_horizons(left, out); - selected_horizons(right, out); - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - selected_horizons(child, out); - } - } - } - } - let mut intervals = vec![1_000]; - selected_horizons(selected, &mut intervals); - intervals.sort_unstable(); - intervals.dedup(); - let mut matched = None; - for expression in expressions { - for interval in &intervals { - let Ok(canonical) = crate::query_parser::parse_query_expr_with_interval( - &expression.to_string(), - planner_types::types::AccuracyTarget::Exact, - *interval, - ) else { - continue; - }; - // Match provenance against all exact candidates. Do not make a - // second selection or assume the first enumerated candidate won. - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - if candidates.iter().any(|candidate| matches!(&candidate.replacement, Replacement::Summary(node) if node.as_ref() == selected)) { - let candidate = expression.clone(); - if matched - .as_ref() - .is_some_and(|previous| previous != &candidate) - { - return Err(invalid( - "ambiguous original subtrees share a Planner summary representation", - )); - } - matched = Some(candidate); - } - } - } - matched.ok_or_else(|| { - invalid("selected summary leaf has no semantically identical original subtree witness") - }) -} - -/// Read the original aggregate operation only after proving its selected-node identity. -/// Min and max share a Planner accumulator family, so the family name alone is insufficient. -pub(super) fn selected_aggregate_operator( - original: &str, - selected: &planner_types::post_asap::SummaryNode, -) -> Result { - let (root, nodes) = selected_query_time_nodes(original, selected)?; - match nodes.get(&root) { - Some(QueryPlanNode::Logical { - operator: operator @ QueryTimeOperator::Aggregate { .. }, - .. - }) => Ok(operator.clone()), - _ => Err(invalid( - "selected value aggregation has no verified original aggregate operator", - )), - } -} - -#[cfg(test)] -mod hybrid_tests { - use super::*; - use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; - #[test] - fn external_binding_rejects_an_unrelated_selected_exact_subtree() { - let exact = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(other_metric[5m])", - planner_types::types::AccuracyTarget::Exact, - 1_000, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&exact).unwrap(); - assert!(selected_native_expression("topk(2, sum_over_time(m[5m]))", &selected).is_err()); - assert_eq!( - selected_native_expression("sum_over_time(other_metric[5m])", &selected) - .unwrap() - .to_string(), - "sum_over_time(other_metric[5m])" - ); - } - - #[test] - fn selected_summary_and_filtered_residual_share_installed_binary() { - // Both filtered and unfiltered leaves bind independently. - let query = "sum_over_time(m[5m]) + sum_over_time(m{job=\"api\"}[5m])"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let entry = - crate::query_plan::compile_bound_composable_mapped( - "hybrid".into(), - query.into(), - &selected, - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |node, _| { - let (_, _, spatial_filter) = - crate::physical::compiler::raw_materialization_input_contract(node) - .map_err(QueryPlanError::Invalid)?; - Ok(MaterializationBinding { - full_window_slide_ms: None, - item_labels: Vec::new(), - materialization: asap_types::PolicyFingerprint( - if spatial_filter.is_empty() { 7 } else { 8 }, - ) - .into(), - stored_output_reference: asap_types::sds::StoredOutputReference::for_output( - asap_types::PolicyFingerprint(if spatial_filter.is_empty() { - 7 - } else { - 8 - }) - .into(), - ), - output_grouping: PhysicalGrouping::PerEntity, - window_ms: 300_000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - }) - }, - |_, _| {}, - ) - .unwrap(); - assert_eq!(entry.materialization_bindings().len(), 2); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - assert!(!entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { .. }, - .. - } - )); - entry - .validate( - &[ - asap_types::PolicyFingerprint(7), - asap_types::PolicyFingerprint(8), - ] - .into_iter() - .collect(), - ) - .unwrap(); - } - - #[test] - fn different_filter_cannot_witness_selected_residual() { - // Equality includes filter predicates, not just family, source, or window. - let canonical = crate::query_parser::parse_query_expr_canonical( - "sum_over_time(m{job=\"api\"}[5m])", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - assert!( - selected_query_time_nodes("sum_over_time(m{job=\"worker\"}[5m])", &selected).is_err() - ); - } -} - -#[cfg(test)] -mod planner_workload_tests { - use super::*; - use crate::physical::compiler::{BackendLocalPlanningInput, DeploymentPlanCompiler}; - - fn compile_one(query: &str) -> crate::physical::compiler::CompiledPhysicalPlan { - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let mut entry = fixture["query_workload"]["repeating_queries"][0].clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - fixture["query_workload"]["repeating_queries"] = vec![entry].into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot - .into_physical_compilation_request() - .unwrap_or_else(|error| panic!("{query}: {error}")); - DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap_or_else(|error| panic!("{query}: {error}")) - } - - fn assert_local_limit(node: &QueryPlanNode) { - match node { - QueryPlanNode::PhysicalFragment { dag, .. } => { - let plan = - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .unwrap(); - assert!( - plan.operator_name(plan.roots()[0]) == Some("Limit") - || super::super::physical_values::operator_parameters(node, "Limit").len() - == 1 - ); - } - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } => {} - _ => panic!("expected local Limit, got {node:?}"), - } - } - - #[test] - fn evaluation_topk_queries_retain_a_local_selection_root() { - for query in [ - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - "topk(1, sum by (job) (rate(backend_process_cpu_seconds_total[6h])))", - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })), - "{query}" - ); - } - } - - #[test] - fn planner_value_topk_preserves_direct_summary_children() { - for query in [ - "topk(2, rate(backend_process_cpu_seconds_total[1h]))", - "topk(2, max_over_time(backend_retry_backlog_depth[6h]))", - ] { - let plan = compile_one(query); - let entry = plan.query_plan.entries.values().next().unwrap(); - assert_local_limit(&entry.nodes[&entry.root]); - assert!( - !entry.materialization_bindings().is_empty(), - "{query} must retain its SummaryStore child: {:?}", - entry.nodes - ); - } - } - - #[test] - fn whole_o11y_planner_candidate_lowers_every_original_query() { - // AST support alone is insufficient: the actual selected Planner forest must bind too. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - let mut fixture: serde_json::Value = serde_json::from_str(include_str!( - "../../../docs/examples/asapquery-planning-snapshot.json" - )) - .unwrap(); - let template = fixture["query_workload"]["repeating_queries"][0].clone(); - let mut seen = std::collections::BTreeSet::new(); - let mut entries = Vec::new(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - if !seen.insert(query.to_string()) { - continue; - } - let mut entry = template.clone(); - entry["query"] = query.into(); - entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); - entries.push(entry); - } - fixture["query_workload"]["repeating_queries"] = entries.into(); - let snapshot: BackendLocalPlanningInput = serde_json::from_value(fixture).unwrap(); - let (request, environment) = snapshot.into_physical_compilation_request().unwrap(); - assert!(request.allow_mixed_summary_and_exact_execution); - let plan = DeploymentPlanCompiler - .compile_promql(request, environment) - .unwrap(); - assert_eq!(plan.query_plan.entries.len(), 24); - assert!(plan.query_plan.entries.values().all(|entry| !entry - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })))); - } - - #[test] - fn max_of_selected_values_uses_original_max_operator() { - // MinMax storage type does not authorize choosing min or replacing the selected operand graph. - let query = "max(sum_over_time(m[5m]) / 2)"; - let canonical = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - use asap_aware_mapping::{ - Replacement, ReplacementStrategy, SketchAlgorithmStrategy, TargetSubDAG, - }; - let root = std::rc::Rc::new(canonical); - let candidates = SketchAlgorithmStrategy::new(&asap_aware_mapping::DefaultCostModel) - .replacements(&TargetSubDAG::new(&root)); - let [candidate] = candidates.as_slice() else { - panic!("expected one exact aggregate candidate") - }; - let Replacement::Summary(selected) = &candidate.replacement else { - panic!("expected exact summary fixture") - }; - let operator = selected_aggregate_operator(query, selected).unwrap(); - assert!(matches!( - operator, - QueryTimeOperator::Aggregate { - operation: Aggregation::Max, - .. - } - )); - } - - #[test] - fn ambiguous_extremum_witness_is_rejected() { - // Different readouts over the same MinMax state cannot be resolved by taking the first AST match. - let canonical = crate::query_parser::parse_query_expr_canonical( - "min(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&canonical).unwrap(); - let maximum = crate::query_parser::parse_query_expr_canonical( - "max(m)", - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let maximum = crate::planner_selection::plan_test_query(&maximum).unwrap(); - let result = selected_query_time_nodes("min(m) + max(m)", &selected); - if selected == maximum { - assert!(result.is_err()); - } else { - let (root, nodes) = result.unwrap(); - assert!(matches!( - nodes[&root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Aggregate { - operation: Aggregation::Min, - .. - }, - .. - } - )); - } - } -} +use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType, SummaryNode}; +use planner_types::pre_asap::{QueryExpr, Reduction}; -/// Preserve the exact original operator direction because MinMax family alone -/// does not distinguish min from max. The full Planner-node witness is required. -pub(crate) fn selected_range_max_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; +/// A per-series `max_over_time` over one plain range selector, the only +/// maximum state the backend maintains. Planner distinguishes Max from Min +/// in the family, so the selected node alone identifies it. +pub(crate) fn is_range_max_materialization(node: &SummaryNode) -> bool { let node = match &node.expr { SummaryExpr::ValueOperation { child, @@ -1042,733 +17,52 @@ pub(crate) fn selected_range_max_materialization( } => child.as_ref(), _ => node, }; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Max, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Temporal { - operation: TemporalOperation::Max, - }, - inputs, - }) = nodes.get(&root) - else { - return Ok(None); - }; - if inputs.len() != 1 || nodes.len() != 2 { - return Ok(None); - } - let Some(QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms: 0, - }, - .. - }) = nodes.get(&inputs[0]) - else { - return Ok(None); - }; - Ok(Some(materialization_candidate_key( - MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: 0, - operation: TemporalOperation::Max, - }, - )?)) -} - -#[cfg(test)] -mod range_max_materialization_tests { - use super::*; - #[test] - fn real_gauge_queries_have_planner_authorized_exact_materializations() { - for (query, metric, range_ms) in [ - ( - r#"max_over_time(service_cache_refresh_lag_seconds{job="user-service"}[12h])"#, - "service_cache_refresh_lag_seconds", - 43_200_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job=~".+"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ( - r#"max_over_time(service_retry_queue_depth{job="order-service"}[6h])"#, - "service_retry_queue_depth", - 21_600_000, - ), - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - let key = selected_range_max_materialization(query, &selected) - .unwrap() - .unwrap(); - assert!(key.contains(metric)); - assert!(key.contains(&range_ms.to_string())); - } - } - #[test] - fn min_and_shifted_or_nested_windows_do_not_become_max_materializations() { - for query in [ - "min_over_time(m[1m])", - "max_over_time(m[1m] offset 1m)", - "max_over_time((m + m)[1m:1s])", - ] { - let original = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, - ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&original).unwrap(); - assert!( - selected_range_max_materialization(query, &selected) - .unwrap() - .is_none(), - "{query}" - ); - } - } -} - -/// Stable contract identity used by priced physical candidates, independent of node IDs. -fn materialization_candidate_key( - candidate: MaterializationCandidateIdentity, -) -> Result { - let mut value = serde_json::to_value(candidate).map_err(|e| invalid(e.to_string()))?; - if let Some(object) = value.as_object_mut() { - // Retention is a consumer lifetime requirement, not the materialization read's - // semantics. Equivalent matcher conjunctions must share policy keys. - object.remove("retention_ms"); - if let Some(matchers) = object.get_mut("matchers").and_then(|v| v.as_array_mut()) { - matchers.sort_by_cached_key(|m| m.to_string()); - } - } - serde_json::to_string(&value).map_err(|e| invalid(e.to_string())) -} - -fn counter_contract( - root: QueryNodeId, - nodes: &BTreeMap, -) -> Option { - let QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { operation }, - inputs, - } = nodes.get(&root)? - else { - return None; - }; - if !matches!( - operation, - TemporalOperation::Rate | TemporalOperation::Increase - ) || inputs.len() != 1 - { - return None; - } - let QueryPlanNode::Logical { - operator: - QueryTimeOperator::Scan { - metric: Some(metric), - matchers, - range_ms: Some(range_ms), - offset_ms, - }, + let SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(ExactKind::Max, _), + reduction: Reduction::PerEntity, + child, .. - } = nodes.get(&inputs[0])? + } = &node.expr else { - return None; - }; - Some(MaterializationCandidateIdentity { - metric: metric.clone(), - matchers: matchers.clone(), - range_ms: *range_ms, - offset_ms: *offset_ms, - operation: *operation, - }) -} - -pub(crate) fn selected_counter_materialization( - original: &str, - node: &planner_types::post_asap::SummaryNode, -) -> Result, QueryPlanError> { - use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType}; - if !matches!( - &node.expr, - SummaryExpr::SummaryAgg { - family: SummaryFamilyType::ExactAggregate(ExactKind::Rate | ExactKind::Increase, _), - reduction: planner_types::pre_asap::Reduction::PerEntity, - .. - } - ) { - return Ok(None); - } - let (root, nodes) = selected_query_time_nodes(original, node)?; - counter_contract(root, &nodes) - .map(materialization_candidate_key) - .transpose() -} - -fn prune(entry: &mut QueryPlanEntry) { - let mut seen = std::collections::BTreeSet::new(); - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if seen.insert(id) { - if let Some(node) = entry.nodes.get(&id) { - pending.extend(node.inputs()); - } - } - } - entry.nodes.retain(|id, _| seen.contains(id)); -} - -/// Finish the installed DAG by externalizing every fragment raw subtree. -pub fn finalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - externalize_query_time_nodes(entry)?; - assign_retention(entry) -} - -pub fn eligible_materialization_keys( - original: &str, - selected: &std::rc::Rc, -) -> Result, QueryPlanError> { - use planner_types::post_asap::SummaryExpr; - fn visit( - original: &str, - node: &std::rc::Rc, - keys: &mut std::collections::BTreeSet, - ) -> Result<(), QueryPlanError> { - if let Some(key) = selected_counter_materialization(original, node)? - .or(selected_range_max_materialization(original, node)?) - { - keys.insert(key); - } - match &node.expr { - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - visit(original, lhs, keys)?; - visit(original, rhs, keys)?; - } - SummaryExpr::RelationalJoin { left, right, .. } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - - SummaryExpr::ValueOperation { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryAgg { child, .. } => visit(original, child, keys)?, - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - visit(original, summary_input, keys)? - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - visit(original, child, keys)?; - } - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - visit(original, outer, keys)?; - visit(original, inner, keys)?; - } - SummaryExpr::SummarySubtract { left, right } => { - visit(original, left, keys)?; - visit(original, right, keys)?; - } - SummaryExpr::KeepPreAsap(_) => {} - } - Ok(()) - } - let mut keys = std::collections::BTreeSet::new(); - visit(original, selected, &mut keys)?; - Ok(keys) -} - -fn expression_shape( - id: QueryNodeId, - nodes: &BTreeMap, -) -> Result { - let node = nodes - .get(&id) - .ok_or_else(|| invalid("missing expression node"))?; - if let QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { query }, - .. - } = node - { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - let expr = parser::parse(query).map_err(|e| invalid(e.to_string()))?; - let root = lower.lower(&expr)?; - return expression_shape(root, &lower.nodes); - } - let value = match node { - QueryPlanNode::Logical { operator, .. } => { - serde_json::to_string(operator).map_err(|e| invalid(e.to_string()))? - } - QueryPlanNode::Scalar { value } => format!("scalar:{:x}", value.to_bits()), - _ => return Err(invalid("summary node has no raw expression shape")), + return false; }; - let children = node - .inputs() - .iter() - .map(|child| expression_shape(*child, nodes)) - .collect::, _>>()?; - Ok(format!("{value}({})", children.join(";"))) -} - -/// Collapse only maximal exact fragment subtrees whose full typed expression is -/// witnessed in the original query. Matrix boundaries remain inside Prometheus. -pub fn externalize_query_time_nodes(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - fn gather(expr: &Expr, out: &mut Vec) { - if !matches!(expr, Expr::MatrixSelector(_) | Expr::Subquery(_)) { - out.push(expr.clone()); - } - match expr { - Expr::Paren(e) => gather(&e.expr, out), - Expr::Unary(e) => gather(&e.expr, out), - Expr::Subquery(e) => gather(&e.expr, out), - Expr::Aggregate(e) => gather(&e.expr, out), - Expr::Binary(e) => { - gather(&e.lhs, out); - gather(&e.rhs, out); - } - Expr::Call(e) => { - for arg in &e.args.args { - gather(arg, out); - } - } - _ => {} - } - } - let expr = parser::parse(&entry.canonical_query).map_err(|e| invalid(e.to_string()))?; - let mut expressions = Vec::new(); - gather(&expr, &mut expressions); - let mut witnesses = BTreeMap::new(); - for expression in expressions { - let mut lower = Lower { - nodes: BTreeMap::new(), - seen: BTreeMap::new(), - }; - if let Ok(root) = lower.lower(&expression) { - witnesses.insert( - expression_shape(root, &lower.nodes)?, - expression.to_string(), - ); - } - } - fn flags(id: QueryNodeId, nodes: &BTreeMap) -> (bool, bool) { - let Some(node) = nodes.get(&id) else { - return (true, false); - }; - let mut indexed = !matches!( - node, - QueryPlanNode::Logical { .. } | QueryPlanNode::Scalar { .. } - ) || matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { .. }, - .. - } - ); - let mut exact = false; - if let QueryPlanNode::Logical { operator, .. } = node { - exact = matches!( - operator, - QueryTimeOperator::Scan { .. } - | QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } - ); - } - for child in node.inputs() { - let (a, b) = flags(*child, nodes); - indexed |= a; - exact |= b; - } - (indexed, exact) - } - let mut pending = vec![entry.root]; - while let Some(id) = pending.pop() { - if matches!( - entry.nodes.get(&id), - Some(QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. }, - .. - }) - ) { - continue; - } - let (indexed, exact) = flags(id, &entry.nodes); - if !indexed && exact { - if let Ok(shape) = expression_shape(id, &entry.nodes) { - if let Some(query) = witnesses.get(&shape) { - entry.nodes.insert( - id, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { - query: query.clone(), - }, - inputs: vec![], - }, - ); - continue; - } - } - } - pending.extend(entry.nodes[&id].inputs()); - } - prune(entry); - if entry.nodes.values().any(|node| { - matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ) - }) { - return Err(invalid( - "local Scan is not deployable; exact subtree requires a complete Prometheus boundary", - )); - } - Ok(()) -} - -fn assign_retention(entry: &mut QueryPlanEntry) -> Result<(), QueryPlanError> { - let mut pending = vec![(entry.root, 0u64)]; - let mut depths = BTreeMap::new(); - while let Some((id, depth)) = pending.pop() { - if depths.get(&id).is_some_and(|prior| *prior >= depth) { - continue; - } - depths.insert(id, depth); - let node = entry - .nodes - .get_mut(&id) - .ok_or_else(|| invalid("missing index ancestor"))?; - let mut child_depth = depth; - if let QueryPlanNode::Logical { operator, .. } = node { - if let QueryTimeOperator::Subquery { - range_ms, - offset_ms, - .. - } = operator - { - child_depth = depth - .checked_add(*range_ms) - .and_then(|v| v.checked_add((*offset_ms).max(0) as u64)) - .ok_or_else(|| invalid("retention overflow"))?; - } - } - pending.extend(node.inputs().iter().map(|child| (*child, child_depth))); - } - Ok(()) -} - -#[cfg(test)] -mod remote_boundary_regressions { - use super::*; - - #[test] - fn stored_output_identity_is_independent_of_matcher_order() { - let first = LabelMatcher { - name: "job".into(), - value: "orders".into(), - operation: LabelMatch::Equal, - }; - let second = LabelMatcher { - name: "status".into(), - value: "5..".into(), - operation: LabelMatch::Regex, - }; - let a = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![first.clone(), second.clone()], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - let b = MaterializationCandidateIdentity { - metric: "requests".into(), - matchers: vec![second, first], - range_ms: 300_000, - offset_ms: 0, - operation: TemporalOperation::Rate, - }; - assert_eq!( - materialization_candidate_key(a).unwrap(), - materialization_candidate_key(b).unwrap() - ); - } - - #[test] - fn real_error_ratio_exposes_two_independent_materialization_candidates() { - let query = "sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m]))"; - let parsed = crate::query_parser::parse_query_expr_canonical( - query, - planner_types::types::AccuracyTarget::Exact, + matches!( + &child.expr, + SummaryExpr::KeepPreAsap(expr) if matches!( + expr.as_ref(), + QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. }) ) - .unwrap(); - let selected = crate::planner_selection::plan_test_query(&parsed).unwrap(); - assert_eq!( - eligible_materialization_keys(query, &selected) - .unwrap() - .len(), - 2 - ); - } + ) } #[cfg(test)] mod tests { use super::*; - // A grouped query's fragment carries the grouping label in its leaf scan - // schema, because Planner resolves that schema against the whole query. - // Re-parsing the subtree alone cannot know the label, so requiring equal - // column sets rejected a fragment that is the subtree. - #[test] - fn grouped_query_residual_matches_its_own_subtree() { - for (query, subtree) in [ - ("sum by (label_0) (rate(data[1m]))", "rate(data[1m])"), - ( - "sum by (label_0) (sum_over_time(data[1m]))", - "sum_over_time(data[1m])", - ), - ] { - let fragment = crate::query_parser::parse_query_expr_with_interval( - subtree, - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!( - query_time_nodes(query, &fragment).is_ok(), - "{query}: fragment {subtree} must resolve against its own query" - ); - } - } - - // Widening is only accepted for the context-derived leaf columns. Anything - // that actually identifies the computation still has to match exactly. - #[test] - fn widened_leaf_schema_does_not_excuse_a_different_computation() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "rate(data[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - // Different metric. - assert!(query_time_nodes("sum by (label_0) (rate(other[1m]))", &fragment).is_err()); - // Different range. - assert!(query_time_nodes("sum by (label_0) (rate(data[2m]))", &fragment).is_err()); - // Different function. - assert!(query_time_nodes("sum by (label_0) (increase(data[1m]))", &fragment).is_err()); - // Different matcher. - let filtered = crate::query_parser::parse_query_expr_with_interval( - "rate(data{job=\"api\"}[1m])", - planner_types::types::AccuracyTarget::Exact, - 60_000, - ) - .unwrap(); - assert!(query_time_nodes( - "sum by (label_0) (rate(data{job=\"worker\"}[1m]))", - &filtered - ) - .is_err()); - } - // A workload horizon changes the equality witness, never its filter or explicit range. - #[test] - fn workload_horizon_residual_keeps_semantic_equality() { - let fragment = crate::query_parser::parse_query_expr_with_interval( - "sum(m{job=\"api\"})", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum(m{job=\"api\"})", &fragment).is_ok()); - assert!(query_time_nodes("sum(m{job=\"worker\"})", &fragment).is_err()); - let range = crate::query_parser::parse_query_expr_with_interval( - "sum_over_time(m[1m])", - planner_types::types::AccuracyTarget::Exact, - 5_000, - ) - .unwrap(); - assert!(query_time_nodes("sum_over_time(m[2m])", &range).is_err()); - } - - fn instant() -> InstantExecution { - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - } - } - - #[test] - fn complete_o11y_corpus_lowers_to_serialized_operations() { - // Every original workload occurrence must compile to an executable typed graph. - let corpus: serde_json::Value = - serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); - for row in corpus["queries"].as_array().unwrap() { - let query = row["query"].as_str().unwrap(); - let entry = crate::query_plan::query_time::compile_logical( - row["id"].as_str().unwrap().into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - let encoded = serde_json::to_string(&entry).unwrap(); - let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); - restored.validate(&Default::default()).unwrap(); - assert!(!restored - .nodes - .values() - .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); - } - } - #[test] - fn query_time_mapping_preserves_filters_and_rejects_different_sources() { - // Physical lowering must prove correspondence with the Planner-kept semantic subtree. - let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; - let fragment = crate::query_parser::parse_query_expr_canonical( + fn selected(query: &str) -> std::rc::Rc { + let original = crate::query_parser::parse_query_expr_canonical( query, planner_types::types::AccuracyTarget::Exact, ) .unwrap(); - let (_, nodes) = query_time_nodes(query, &fragment).unwrap(); - assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: QueryTimeOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); - assert!(query_time_nodes("sum(rate(other_total[5m]))", &fragment).is_err()); - } - #[test] - fn repeated_subexpressions_share_node_identity() { - // Serialized edges must retain CSE rather than duplicating raw work. - let entry = crate::query_plan::query_time::compile_logical( - "q".into(), - "sum(up) / sum(up)".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { - panic!("binary expected") - }; - assert_eq!(inputs[0], inputs[1]); - } - #[test] - fn malformed_operator_arity_is_rejected_at_installation() { - // A serialized graph cannot bypass the operation's input contract. - assert!(QueryTimeOperator::HistogramQuantile.validate(1).is_err()); - assert!(QueryTimeOperator::Subquery { - range_ms: 60_000, - step_ms: 0, - offset_ms: 0 - } - .validate(1) - .is_err()); + crate::planner_selection::plan_test_query(&original).unwrap() } + // Plain gauge maxima are maintained; minima, offsets and nested windows are not. #[test] - fn real_topk_queries_lower_to_value_selection() { - for (query, k) in [ - ( - "topk(2, sum by (job) (rate(backend_process_cpu_seconds_total[1h])))", - 2, - ), - ( - "topk(2, sum by (job) (backend_process_resident_memory_bytes))", - 2, - ), - ("topk(2, max_over_time(backend_retry_backlog_depth[6h]))", 2), - ( - "topk(1, sum by (job) (increase(backend_http_5xx_total[6h])) / sum by (job) (increase(backend_http_requests_total[6h])))", - 1, - ), - ( - "topk(3, avg_over_time((sum by (job) (backend_process_resident_memory_bytes))[6h:]))", - 3, - ), + fn range_max_materialization_is_identified_from_the_selected_node() { + for query in [ + r#"max_over_time(service_cache_refresh_lag_seconds{job="user-service"}[12h])"#, + r#"max_over_time(service_retry_queue_depth{job=~".+"}[6h])"#, ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap_or_else(|error| panic!("{query}: {error}")); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: actual, .. }, - .. - } if actual == k - )); + assert!(is_range_max_materialization(&selected(query)), "{query}"); } - } - - #[test] - fn topk_keeps_unsupported_child_as_exact_leaf() { - let entry = crate::query_plan::query_time::compile_logical( - "topk-subquery".into(), - "topk(3, label_replace(memory_bytes, \"dst\", \"$1\", \"src\", \"(.*)\"))".into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { n: 3, .. }, - .. - } - )); - assert!(entry.nodes.values().any(|node| matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ))); - } - - #[test] - fn topk_preserves_by_and_without_partitioning() { - for (query, labels, without) in [ - ("topk by (cluster) (2, m)", vec!["cluster"], false), - ("topk without (pod) (2, m)", vec!["pod"], true), + for query in [ + "min_over_time(m[1m])", + "max_over_time(m[1m] offset 1m)", + "max_over_time((m + m)[1m:1s])", ] { - let entry = crate::query_plan::query_time::compile_logical( - "topk-group".into(), - query.into(), - instant(), - FallbackPolicy::Reject, - ) - .unwrap(); - assert!(matches!( - &entry.nodes[&entry.root], - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { grouping, .. }, - .. - } if grouping.labels == labels && grouping.without == without - )); + assert!(!is_range_max_materialization(&selected(query)), "{query}"); } } } diff --git a/control_plane/tests/lifecycle_placement.rs b/control_plane/tests/lifecycle_placement.rs index e89b2751f..9f3c48ef8 100644 --- a/control_plane/tests/lifecycle_placement.rs +++ b/control_plane/tests/lifecycle_placement.rs @@ -116,7 +116,7 @@ fn decisions(queries: &[&str]) -> Vec { #[test] fn shared_state_is_priced_once_with_all_reads() { let [alone] = decisions(&["sum_over_time(m[1m])"]).try_into().unwrap(); - let [shared] = decisions(&["sum_over_time(m[1m])", "sum_over_time(m[1m]) * 2"]) + let [shared] = decisions(&["sum_over_time(m[1m])", "sort(sum_over_time(m[1m]))"]) .try_into() .unwrap(); assert_eq!(shared["query_ids"].as_array().unwrap().len(), 2); diff --git a/control_plane/tests/offline_evidence.rs b/control_plane/tests/offline_evidence.rs index f24dddeec..40cd612d8 100644 --- a/control_plane/tests/offline_evidence.rs +++ b/control_plane/tests/offline_evidence.rs @@ -362,11 +362,11 @@ fn incompatible_evidence_preserves_deployment_behavior() { } } -/// Binary operations over these approximate sketch values retain explicit -/// fallback even though the warm tier now supports exact additive binaries. +/// Binary operations over these approximate sketch values have no accuracy +/// guarantee, so the backend refuses to compute them and the query runs exactly. #[test] fn binary_summary_has_explicit_warm_tier_fallback() { - use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; + use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanError}; use planner_types::{post_asap::BinaryOperator, pre_asap::BinaryOpKind}; let child = bound(&model()); let root = std::rc::Rc::new(SummaryNode { @@ -384,7 +384,7 @@ fn binary_summary_has_explicit_warm_tier_fallback() { schema: child.schema.clone(), guarantee: None, }); - let plan = control_plane::query_plan::compile_bound_mapped( + let error = control_plane::query_plan::compile_bound_mapped( "test".into(), "left / right".into(), &root, @@ -397,9 +397,9 @@ fn binary_summary_has_explicit_warm_tier_fallback() { |_, _| panic!("unsupported binary plan must not bind a materialization"), |_, _| {}, ) - .unwrap(); + .unwrap_err(); assert!( - matches!(&plan.nodes[&plan.root], QueryPlanNode::ExactFallback { reason } if !reason.is_empty()) + matches!(error, QueryPlanError::UnsupportedNode(_)), + "{error}" ); - assert!(plan.materialization_bindings().is_empty()); } diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 2ded15630..614ef9e4b 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -164,17 +164,6 @@ impl QueryPlan { } } for node in entry.nodes.values() { - if matches!( - node, - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } - ) { - return Err(QueryPlanError::Invalid( - "installed value computation requires a retained Planner physical graph" - .into(), - )); - } let QueryPlanNode::ExactReadout { input, readout } = node else { continue; }; @@ -651,9 +640,6 @@ impl QueryPlanEntry { if let QueryPlanNode::Logical { operator, inputs } = node { operator.validate(inputs.len())?; } - if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { - return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); - } if let QueryPlanNode::ExternalExact { request, inputs } = node { if request.expression.trim().is_empty() { return Err(QueryPlanError::Invalid( @@ -854,17 +840,6 @@ pub enum QueryPlanNode { operator: query_time::QueryTimeOperator, inputs: Vec, }, - Scalar { - value: f64, - }, - Binary { - inputs: [QueryNodeId; 2], - operator: planner_types::pre_asap::ArithmeticOpKind, - }, - ReduceSum { - input: QueryNodeId, - grouping: PhysicalGrouping, - }, ReadMaterialization { binding: MaterializationBinding, }, @@ -893,13 +868,10 @@ pub enum QueryPlanNode { impl QueryPlanNode { pub fn inputs(&self) -> &[QueryNodeId] { match self { - Self::Scalar { .. } | Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => { - &[] + Self::ReadMaterialization { .. } | Self::ExactFallback { .. } => &[], + Self::SummaryEstimate { input, .. } | Self::ExactReadout { input, .. } => { + std::slice::from_ref(input) } - Self::Binary { inputs, .. } => inputs, - Self::ReduceSum { input, .. } - | Self::SummaryEstimate { input, .. } - | Self::ExactReadout { input, .. } => std::slice::from_ref(input), Self::Physical { inputs, .. } | Self::PhysicalRelation { inputs, .. } | Self::PhysicalFragment { inputs, .. } @@ -1014,52 +986,29 @@ mod contract_tests { #[cfg(test)] mod retired_plan_tests { - // Recovery cannot reactivate the removed request-time scalar compiler. + // Recovery cannot reactivate the removed backend value operators. #[test] - fn catalog_rejects_uncompiled_value_computation() { - use super::*; - let catalog = - crate::summary_catalog::SummaryCatalog::from_materializations(1, 1, &[]).unwrap(); - let entry = QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "scalar".into(), - canonical_query: "1".into(), - fixed_evaluation: None, - root: QueryNodeId(0), - nodes: BTreeMap::from([(QueryNodeId(0), QueryPlanNode::Scalar { value: 1. })]), - instant: InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::Reject, - }; - let mut plan = QueryPlan { - plan_id: 1, - plan_version: 1, - clickhouse_context: None, - selected_dags: BTreeMap::new(), - entries: BTreeMap::from([("1".into(), entry)]), - }; - assert!(plan - .validate_against_catalog(&catalog) - .unwrap_err() - .to_string() - .contains("retained Planner physical graph")); - plan.entries.get_mut("1").unwrap().nodes.insert( - QueryNodeId(0), - QueryPlanNode::PhysicalFragment { - inputs: vec![], - dag: asap_physical_operators::physical_planner::promql_values::compile_scalar(1.) - .unwrap() - .encode() - .unwrap(), - row_input: None, - pruning: None, - }, - ); - plan.validate_against_catalog(&catalog).unwrap(); + fn retired_value_operators_are_not_accepted() { + for kind in ["scalar", "binary", "reduce_sum"] { + let error = + serde_json::from_value::(serde_json::json!({"op":kind})) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant"), "{error}"); + } + for kind in [ + "binary", + "aggregate", + "temporal", + "subquery", + "sort", + "limit", + ] { + let error = serde_json::from_value::( + serde_json::json!({"kind":kind}), + ) + .unwrap_err(); + assert!(error.to_string().contains("unknown variant"), "{error}"); + } } // Row-preserving operators cannot opt out of the original vector identity. diff --git a/crates/asap_types/src/query_plan/query_time.rs b/crates/asap_types/src/query_plan/query_time.rs index a5b3fa9ef..a67942f11 100644 --- a/crates/asap_types/src/query_plan/query_time.rs +++ b/crates/asap_types/src/query_plan/query_time.rs @@ -1,4 +1,5 @@ -//! Typed installed query-time operators; no Planner selection or AST lowering. +//! Installed query-time leaves: raw selectors, exact subtrees and +//! current-series readouts. Computation over them is Planner-compiled. use super::QueryPlanError; use promql_parser::parser::{self, Expr}; use serde::{Deserialize, Serialize}; @@ -15,50 +16,16 @@ pub enum QueryTimeOperator { readout: super::current_series::SeriesReadout, }, /// A maximal exact scalar/vector subtree evaluated by Prometheus. - ExactSubquery { - query: String, - }, + ExactSubquery { query: String }, /// Prometheus exact subtree whose selectors are restricted at runtime by /// the candidate vector produced by its single input. - CandidateExactSubquery { - query: String, - item_label: String, - }, + CandidateExactSubquery { query: String, item_label: String }, Scan { metric: Option, matchers: Vec, range_ms: Option, offset_ms: i64, }, - UnaryNegate, - VectorToScalar, - Aggregate { - operation: Aggregation, - grouping: Grouping, - }, - /// Select an ordered slice independently within each group. - Limit { - n: u64, - offset: u64, - grouping: Grouping, - }, - Binary { - operation: BinaryOperation, - return_bool: bool, - }, - Temporal { - operation: TemporalOperation, - }, - Sort { - descending: bool, - grouping: Grouping, - }, - HistogramQuantile, - Subquery { - range_ms: u64, - step_ms: u64, - offset_ms: i64, - }, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -82,47 +49,6 @@ pub enum LabelMatch { Regex, NotRegex, } -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum Aggregation { - Sum, - Max, - Min, - Avg, - Count, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum BinaryOperation { - Add, - Sub, - Mul, - Div, - /// Division with the Planner relative-value certificate domain checks. - CheckedDiv, - /// Division for conditional exact rewrites: finite inputs and finite output. - FiniteDiv, - Mod, - Pow, - Equal, - NotEqual, - Less, - LessEqual, - Greater, - GreaterEqual, -} -#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "snake_case")] -pub enum TemporalOperation { - Rate, - Increase, - Avg, - Max, - Min, - Sum, - Count, -} - impl QueryTimeOperator { pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { if let Self::CurrentSeries { @@ -148,8 +74,6 @@ impl QueryTimeOperator { let expected = match self { Self::Scan { .. } | Self::ExactSubquery { .. } | Self::CurrentSeries { .. } => 0, Self::CandidateExactSubquery { .. } => 1, - Self::Binary { .. } | Self::HistogramQuantile => 2, - _ => 1, }; if inputs != expected { return Err(invalid("logical operator input arity mismatch")); @@ -171,14 +95,6 @@ impl QueryTimeOperator { )); } } - if let Self::Subquery { - range_ms, step_ms, .. - } = self - { - if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { - return Err(invalid("invalid or excessive subquery grid")); - } - } Ok(()) } } diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 66906aba4..824081016 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -388,14 +388,13 @@ impl ASAPQueryEngine { )?; // Native installed DAGs require no remote request preparation. Avoid // constructing evaluation-grid maps for this common deployment path. - // Subqueries and raw scans still take the checked preparation path. + // Raw scans still take the checked preparation path. if entry.nodes.values().all(|node| match node { asap_types::query_plan::QueryPlanNode::ExternalExact { .. } => false, asap_types::query_plan::QueryPlanNode::Logical { operator, .. } => !matches!( operator, asap_types::query_plan::query_time::QueryTimeOperator::ExactSubquery { .. } | asap_types::query_plan::query_time::QueryTimeOperator::CandidateExactSubquery { .. } - | asap_types::query_plan::query_time::QueryTimeOperator::Subquery { .. } | asap_types::query_plan::query_time::QueryTimeOperator::Scan { .. } ), _ => true, @@ -2885,6 +2884,7 @@ mod range_stitch_tests { #[tokio::test] async fn active_metricsql_entry_reaches_the_shared_dag_executor() { + use asap_types::physical_plan_codec::PhysicalPlanCodec; use asap_types::query_plan::{ FallbackPolicy, InstantExecution, QueryLanguage, QueryNodeId, QueryPlanEntry, QueryPlanNode, @@ -2906,18 +2906,21 @@ mod range_stitch_tests { query_id: "vm-scalar".into(), canonical_query: identity.clone(), fixed_evaluation: None, - root: QueryNodeId(2), - nodes: std::collections::BTreeMap::from([ - (QueryNodeId(0), QueryPlanNode::Scalar { value: 1.0 }), - (QueryNodeId(1), QueryPlanNode::Scalar { value: 2.0 }), - ( - QueryNodeId(2), - QueryPlanNode::Binary { - inputs: [QueryNodeId(0), QueryNodeId(1)], - operator: planner_types::pre_asap::ArithmeticOpKind::Add, - }, - ), - ]), + root: QueryNodeId(0), + nodes: std::collections::BTreeMap::from([( + QueryNodeId(0), + QueryPlanNode::PhysicalFragment { + inputs: vec![], + dag: asap_physical_operators::physical_planner::promql_values::compile_scalar( + 3.0, + ) + .unwrap() + .encode() + .unwrap(), + row_input: None, + pruning: None, + }, + )]), instant: InstantExecution { lookback_ms: 1, full_history: false, @@ -2926,12 +2929,6 @@ mod range_stitch_tests { fallback: FallbackPolicy::ExactBackend, }, ); - let key = - asap_types::query_plan::QueryPlan::catalog_key(QueryLanguage::MetricsQl, &identity); - control_plane::query_plan::physical_values::compile( - plan.query_plan.entries.get_mut(&key).unwrap(), - ) - .unwrap(); let mut active = crate::drivers::query::servers::http::validate_and_build_runtime_plan( crate::drivers::query::servers::http::PhysicalPlanInstallRequest { summary_catalog: plan.summary_catalog, diff --git a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs index 9ab038fc7..0cc1facba 100644 --- a/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs +++ b/data_plane/src/query_engines/asap_query_engine/exact_subqueries.rs @@ -16,7 +16,7 @@ fn miss(message: impl Into) -> EngineError { EngineError::capability_miss("exact_subquery", message.into()) } -/// Traverse only the installed graph, including epoch-aligned nested subquery grids. +/// Traverse only the installed graph. #[derive(Debug, Clone)] enum ExactLeaf { Legacy(QueryTimeOperator), @@ -56,35 +56,6 @@ fn leaves( | QueryTimeOperator::CandidateExactSubquery { .. } => { result.insert((id, at), ExactLeaf::Legacy(operator.clone())); } - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - } => { - let step = i64::try_from(*step_ms).map_err(|_| miss("step overflow"))?; - let range = i64::try_from(*range_ms).map_err(|_| miss("range overflow"))?; - if step <= 0 || range / step > 100_000 { - return Err(miss("invalid subquery grid")); - } - let end = at - .checked_sub(*offset_ms) - .ok_or_else(|| miss("offset overflow"))?; - let start = end - .checked_sub(range) - .ok_or_else(|| miss("range overflow"))?; - let mut t = start - .div_euclid(step) - .checked_add(1) - .and_then(|n| n.checked_mul(step)) - .ok_or_else(|| miss("grid overflow"))?; - let input = *inputs - .first() - .ok_or_else(|| miss("missing subquery input"))?; - while t <= end { - pending.push((input, t)); - t = t.checked_add(step).ok_or_else(|| miss("grid overflow"))?; - } - } _ => pending.extend(inputs.iter().map(|input| (*input, at))), }, // A join is a typed composition node rather than a Logical @@ -734,21 +705,7 @@ mod tests { QueryPlanNode::PhysicalFragment { inputs: [QueryNodeId(0), QueryNodeId(1)].to_vec(), dag: compiled.encode().unwrap(), row_input: Some(0), pruning: (Some(CandidateCompleteness::BestEffort { guarantee: None })).map(|completeness| asap_types::query_plan::PruningInputContract { candidate_input: 1, keys: vec![(0,0)], completeness }) } } }); - entry.nodes.insert( - QueryNodeId(3), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 2, - grouping: asap_types::query_plan::query_time::Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(2)], - }, - ); - entry.root = QueryNodeId(3); + entry.root = QueryNodeId(2); let dependencies = external_dependencies(&entry, &[1_000]).unwrap(); assert_eq!(dependencies, vec![(QueryNodeId(0), QueryNodeId(1), 1_000)]); let prepared = prepare_external( @@ -852,10 +809,31 @@ mod tests { )); server.abort(); } + /// Planner's label-map division, the computation over a prepared exact leaf. + fn planner_division() -> Vec { + use planner_types::{ + post_asap::BinaryOperator, + pre_asap::{ArithmeticOpKind, BinaryOpKind}, + }; + asap_physical_operators::physical_planner::promql_values::compile_binary( + &BinaryOperator { + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Div), + vector_match: None, + checked_relative_division: false, + checked_finite_division: false, + }, + false, + false, + false, + ) + .unwrap() + .encode() + .unwrap() + } + #[tokio::test] async fn exact_leaf_calls_prometheus_and_combines_with_prepared_summary() { // A successful exact branch remains an intermediate, not a whole-root fallback. - use asap_types::query_plan::query_time::BinaryOperation; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -874,12 +852,11 @@ mod tests { let mut entry = entry(BTreeMap::from([ ( QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Div, - return_bool: false, - }, + QueryPlanNode::PhysicalFragment { inputs: vec![QueryNodeId(1), QueryNodeId(2)], + dag: planner_division(), + row_input: None, + pruning: None, }, ), ( @@ -894,7 +871,6 @@ mod tests { }, ), ])); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); let leaves = prepare( &entry, &[1000], @@ -970,9 +946,7 @@ mod tests { }; use crate::storage_engines::types::Measurement; use asap_summary_state::summary_kernels::IncreaseAccumulator; - use asap_types::query_plan::{ - query_time::BinaryOperation, ExactReadout, MaterializationBinding, PhysicalGrouping, - }; + use asap_types::query_plan::{ExactReadout, MaterializationBinding, PhysicalGrouping}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -1012,12 +986,11 @@ mod tests { let entry = entry(BTreeMap::from([ ( QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Div, - return_bool: false, - }, + QueryPlanNode::PhysicalFragment { inputs: vec![QueryNodeId(1), QueryNodeId(2)], + dag: planner_division(), + row_input: None, + pruning: None, }, ), ( diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs index 1aa86c2aa..a5e57c3dd 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -9,10 +9,6 @@ use crate::query_engines::{ use crate::storage_engines::types::KeyByLabelValues; use asap_physical_operators::dag as physical; use asap_types::query_plan::query_time::QueryTimeOperator; -#[cfg(test)] -use asap_types::query_plan::query_time::{ - Aggregation, BinaryOperation, Grouping, TemporalOperation, -}; use asap_types::query_plan::{CandidateCompleteness, QueryNodeId, QueryPlanEntry, QueryPlanNode}; use futures::{FutureExt, StreamExt}; use std::cell::RefCell; @@ -65,11 +61,6 @@ pub struct ExecutionStats { fn miss(detail: impl Into) -> EngineError { EngineError::capability_miss("installed_logical_dag", detail) } -#[cfg(test)] -fn no_name(mut labels: Labels) -> Labels { - labels.remove("__name__"); - labels -} fn vector(value: Value) -> Result { let Value::Vector(values) = value else { return Err(miss("instant vector required")); @@ -285,7 +276,6 @@ impl Result> ValueRuntim at: i64, node: &QueryPlanNode, inputs: &[&Value], - dependencies: &[(QueryNodeId, i64)], context: &physical::RunContext, ) -> Result { if let Some(leaf) = self.leaves.get(&(id, at)) { @@ -300,14 +290,6 @@ impl Result> ValueRuntim return Ok(value); } let value = match node.clone() { - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } => { - return Err(physical::Error::Invalid( - "installed computation requires a retained Planner physical graph".into(), - ) - .into()); - } QueryPlanNode::Logical { operator: QueryTimeOperator::CurrentSeries { .. }, .. @@ -318,18 +300,10 @@ impl Result> ValueRuntim u64::try_from(at).map_err(|_| miss("negative current-series timestamp"))?, )?)? } - QueryPlanNode::Logical { operator, .. } => { - if matches!( - operator, - QueryTimeOperator::Scan { .. } - | QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } - ) { - return Err(miss( - "installed Prometheus leaf was not prepared; backend raw execution is forbidden", - )); - } - self.logical(operator, inputs, dependencies, at, context)? + QueryPlanNode::Logical { .. } => { + return Err(miss( + "installed Prometheus leaf was not prepared; backend raw execution is forbidden", + )); } QueryPlanNode::PhysicalFragment { dag, @@ -369,88 +343,8 @@ impl Result> ValueRuntim }; Ok(value) } - fn logical( - &mut self, - operator: QueryTimeOperator, - inputs: &[&Value], - dependencies: &[(QueryNodeId, i64)], - at: i64, - context: &physical::RunContext, - ) -> Result { - match operator { - QueryTimeOperator::ExactSubquery { .. } - | QueryTimeOperator::CandidateExactSubquery { .. } => { - Err(miss("Prometheus exact leaf was not prepared")) - } - QueryTimeOperator::CurrentSeries { .. } => Err(miss( - "current-series leaf must use its installed node identity", - )), - QueryTimeOperator::Scan { .. } => { - Err(miss("local raw Scan is forbidden in deployed plans")) - } - QueryTimeOperator::UnaryNegate - | QueryTimeOperator::VectorToScalar - | QueryTimeOperator::Aggregate { .. } - | QueryTimeOperator::Limit { .. } - | QueryTimeOperator::Binary { .. } - | QueryTimeOperator::Temporal { .. } - | QueryTimeOperator::Sort { .. } - | QueryTimeOperator::HistogramQuantile => Err(physical::Error::Invalid( - "installed computation must contain Planner physical operators".into(), - ) - .into()), - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - } => { - let (start, end, _) = subquery_grid(at, range_ms, step_ms, offset_ms)?; - let mut values: BTreeMap> = BTreeMap::new(); - if inputs.len() != dependencies.len() { - return Err(miss("subquery grid input mismatch")); - } - for (value, (_, time)) in inputs.iter().zip(dependencies) { - for (labels, value) in vector((**value).clone())? { - values.entry(labels).or_default().push((*time, value)); - } - } - Ok(Value::Matrix(values.into_iter().collect(), start, end)) - } - } - } } -fn subquery_grid( - at: i64, - range_ms: u64, - step_ms: u64, - offset_ms: i64, -) -> Result<(i64, i64, Vec), EngineError> { - let end = at - .checked_sub(offset_ms) - .ok_or_else(|| miss("offset overflow"))?; - let range = i64::try_from(range_ms).map_err(|_| miss("range overflow"))?; - let step = i64::try_from(step_ms).map_err(|_| miss("step overflow"))?; - if step <= 0 || range / step > 100_000 { - return Err(miss("invalid or excessive subquery steps")); - } - let start = end - .checked_sub(range) - .ok_or_else(|| miss("range overflow"))?; - let mut time = start - .div_euclid(step) - .checked_add(1) - .and_then(|n| n.checked_mul(step)) - .ok_or_else(|| miss("subquery grid overflow"))?; - let mut times = Vec::new(); - while time <= end { - times.push(time); - time = time - .checked_add(step) - .ok_or_else(|| miss("subquery time overflow"))?; - } - Ok((start, end, times)) -} fn expanded_inputs(node: &QueryPlanNode, at: i64) -> Result, EngineError> { match node { QueryPlanNode::Logical { @@ -462,26 +356,11 @@ fn expanded_inputs(node: &QueryPlanNode, at: i64) -> Result Err(miss( "installed leaf was not prepared; local raw execution is forbidden", )), - QueryPlanNode::Logical { - operator: - QueryTimeOperator::Subquery { - range_ms, - step_ms, - offset_ms, - }, - inputs, - } => { - let [input] = inputs.as_slice() else { - return Err(miss("subquery requires one input")); - }; - let (_, _, times) = subquery_grid(at, *range_ms, *step_ms, *offset_ms)?; - Ok(times.into_iter().map(|time| (*input, time)).collect()) - } QueryPlanNode::Logical { operator: QueryTimeOperator::CurrentSeries { .. }, .. } => Ok(vec![]), - QueryPlanNode::PhysicalFragment { inputs, .. } | QueryPlanNode::Logical { inputs, .. } => { + QueryPlanNode::PhysicalFragment { inputs, .. } => { Ok(inputs.iter().map(|&id| (id, at)).collect()) } @@ -535,14 +414,7 @@ impl Result> let values = values.iter().map(|v| v.value()).collect::>(); self.runtime .borrow_mut() - .execute_node( - self.id, - self.time, - self.node, - &values, - &self.dependencies, - &context, - ) + .execute_node(self.id, self.time, self.node, &values, &context) .map_err(|error| { *self.error.borrow_mut() = Some(error); physical::Error::Operator(format!( @@ -640,63 +512,6 @@ fn native_labels(labels: &Labels) -> physical::values::Value { .into(), ) } -#[cfg(test)] -fn native_vector_batch( - values: Vector, - grouping: &Grouping, -) -> Result { - use physical::values::{Batch, Value as Cell}; - use planner_types::{ - post_asap::{SummaryFamilyType, SummaryField, SummarySchema}, - pre_asap::DataType, - }; - let label_type = DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Utf8), - value_nullable: false, - }; - let schema = std::sync::Arc::new(SummarySchema { - fields: vec![ - ("labels", label_type.clone()), - ("group", label_type), - ("value", DataType::Float64), - ] - .into_iter() - .map(|(name, dtype)| SummaryField { - name: name.into(), - dtype: SummaryFamilyType::Plain(dtype), - nullable: false, - }) - .collect(), - time_index: None, - }); - let rows = values - .into_iter() - .map(|(labels, value)| { - vec![ - native_labels(&labels), - native_labels(&grouping_key(&labels, grouping)), - Cell::Float64(value), - ] - }) - .collect(); - Batch::try_new(schema, rows).map_err(EngineError::from) -} -#[cfg(test)] -fn native_batch_rows( - batch: physical::values::Batch, - ops: Vec, - context: &physical::RunContext, -) -> Result>, EngineError> { - physical::batch_execution::evaluate_batch(batch, ops, context.clone()) - .map(|batches| { - batches - .into_iter() - .flat_map(|batch| batch.rows().to_vec()) - .collect() - }) - .map_err(EngineError::from) -} fn native_vector_output( rows: Vec>, label_column: usize, @@ -728,296 +543,6 @@ fn native_vector_output( }) .collect() } -#[cfg(test)] -fn aggregate( - operation: Aggregation, - grouping: &Grouping, - values: Vector, - context: &physical::RunContext, -) -> Result { - use physical::operators::{Operator, Reduction}; - let batch = native_vector_batch(values, grouping)?; - let reduction = match operation { - Aggregation::Sum => Reduction::Sum(2), - Aggregation::Avg => Reduction::Avg(2), - Aggregation::Count => Reduction::Count, - Aggregation::Max => Reduction::Max(2), - Aggregation::Min => Reduction::Min(2), - }; - let operator = Operator::aggregate( - batch.schema().clone(), - vec![1], - vec![("value".into(), reduction)], - ) - .map_err(EngineError::from)?; - native_vector_output(native_batch_rows(batch, vec![operator], context)?, 0, 1) -} - -#[cfg(test)] -fn negate(value: Value, context: &physical::RunContext) -> Result { - use physical::operators::{Expression, Operator}; - let scalar = matches!(value, Value::Scalar(_)); - let values = match value { - Value::Scalar(v) => vec![(Labels::new(), v)], - Value::Vector(v) => v, - _ => return Err(miss("cannot negate range vector")), - }; - let batch = native_vector_batch( - values, - &Grouping { - labels: vec![], - without: false, - }, - )?; - let operator = Operator::project( - batch.schema().clone(), - vec![ - ("labels".into(), Expression::Column(0)), - ( - "value".into(), - Expression::Negate(Box::new(Expression::Column(2))), - ), - ], - ) - .map_err(EngineError::from)?; - let result = native_vector_output(native_batch_rows(batch, vec![operator], context)?, 0, 1)?; - Ok(if scalar { - Value::Scalar(result[0].1) - } else { - Value::Vector(result) - }) -} -#[cfg(test)] -fn grouping_key(labels: &Labels, grouping: &Grouping) -> Labels { - labels - .iter() - .filter(|(key, _)| { - if grouping.without { - key.as_str() != "__name__" && !grouping.labels.contains(key) - } else { - grouping.labels.contains(key) - } - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() -} - -/// Select by the child sample value while retaining every selected series' -/// labels. NaN ranks below every numeric value, matching Prometheus' TOPK heap. -/// Stable sorting also leaves equal-valued series in the child's order. -#[cfg(test)] -fn topk_selection( - k: u64, - grouping: &Grouping, - values: Vector, - context: &physical::RunContext, -) -> Result { - use physical::operators::{Operator, SortKey}; - let batch = native_vector_batch(values, grouping)?; - let sort = Operator::sort( - batch.schema().clone(), - vec![SortKey { - column: 2, - descending: true, - nulls_first: false, - }], - vec![1], - ) - .map_err(EngineError::from)?; - let limit = Operator::limit(sort.schema(), k, 0, vec![1]).map_err(EngineError::from)?; - let mut output = - native_vector_output(native_batch_rows(batch, vec![sort, limit], context)?, 0, 2)?; - // The HTTP adapter preserves canonical label-group presentation; native Sort - // already determined score order within each group. - output.sort_by_key(|(labels, _)| grouping_key(labels, grouping)); - Ok(output) -} - -#[cfg(test)] -fn binary( - operation: BinaryOperation, - boolean: bool, - left: Value, - right: Value, -) -> Result { - binary_in_context(operation, boolean, left, right, &test_native_context()) -} -#[cfg(test)] -fn binary_in_context( - operation: BinaryOperation, - boolean: bool, - left: Value, - right: Value, - context: &physical::RunContext, -) -> Result { - use physical::{ - operators::{Expression, Operator}, - values::{Batch, Value as Cell}, - }; - use planner_types::{ - post_asap::{BinaryOperator, SummaryFamilyType, SummaryField, SummarySchema}, - pre_asap::{ArithmeticOpKind as A, BinaryOpKind, CompareOpKind as C, DataType}, - }; - let kind = match operation { - BinaryOperation::Add => BinaryOpKind::Arithmetic(A::Add), - BinaryOperation::Sub => BinaryOpKind::Arithmetic(A::Sub), - BinaryOperation::Mul => BinaryOpKind::Arithmetic(A::Mul), - BinaryOperation::Div | BinaryOperation::CheckedDiv | BinaryOperation::FiniteDiv => { - BinaryOpKind::Arithmetic(A::Div) - } - BinaryOperation::Mod => BinaryOpKind::Arithmetic(A::Mod), - BinaryOperation::Pow => BinaryOpKind::Arithmetic(A::Pow), - BinaryOperation::Equal => BinaryOpKind::Compare(C::Eq), - BinaryOperation::NotEqual => BinaryOpKind::Compare(C::Ne), - BinaryOperation::Less => BinaryOpKind::Compare(C::Lt), - BinaryOperation::LessEqual => BinaryOpKind::Compare(C::Le), - BinaryOperation::Greater => BinaryOpKind::Compare(C::Gt), - BinaryOperation::GreaterEqual => BinaryOpKind::Compare(C::Ge), - }; - let arithmetic = matches!(kind, BinaryOpKind::Arithmetic(_)); - let scalar_output = matches!((&left, &right), (Value::Scalar(_), Value::Scalar(_))); - if scalar_output && !arithmetic && !boolean { - return Err(miss("scalar comparison requires bool")); - } - if boolean - && matches!( - operation, - BinaryOperation::CheckedDiv | BinaryOperation::FiniteDiv - ) - { - return Err(miss("checked division cannot return bool")); - } - // Matching and metric-name presentation are protocol bindings; all numeric - // computation and checked arithmetic execute in the shared operator. - let mut pairs = Vec::new(); - let scalar_left = matches!(left, Value::Scalar(_)); - match (left, right) { - (Value::Scalar(a), Value::Scalar(b)) => pairs.push((Labels::new(), a, b)), - (Value::Vector(values), Value::Scalar(b)) => { - for (labels, a) in vector(Value::Vector(values))? { - pairs.push((labels, a, b)); - } - } - (Value::Scalar(a), Value::Vector(values)) => { - for (labels, b) in vector(Value::Vector(values))? { - pairs.push((labels, a, b)); - } - } - (Value::Vector(left), Value::Vector(right)) => { - let mut rhs = BTreeMap::new(); - for (labels, value) in right { - if rhs.insert(no_name(labels), value).is_some() { - return Err(miss("duplicate vector matching labels")); - } - } - let mut seen = BTreeSet::new(); - for (labels, value) in left { - let key = no_name(labels.clone()); - if !seen.insert(key.clone()) { - return Err(miss("duplicate vector matching labels")); - } - if let Some(right) = rhs.get(&key) { - pairs.push((labels, value, *right)); - } - } - } - _ => return Err(miss("binary matrix unsupported")), - } - let schema = std::sync::Arc::new(SummarySchema { - fields: ["left", "right"] - .into_iter() - .map(|name| SummaryField { - name: name.into(), - dtype: SummaryFamilyType::Plain(DataType::Float64), - nullable: false, - }) - .collect(), - time_index: None, - }); - let batch = Batch::try_new( - schema.clone(), - pairs - .iter() - .map(|(_, a, b)| vec![Cell::Float64(*a), Cell::Float64(*b)]) - .collect(), - ) - .map_err(EngineError::from)?; - let operator = Operator::project( - schema, - vec![( - "value".into(), - Expression::Binary { - operator: BinaryOperator { - kind, - vector_match: None, - checked_relative_division: operation == BinaryOperation::CheckedDiv, - checked_finite_division: operation == BinaryOperation::FiniteDiv, - }, - left: Box::new(Expression::Column(0)), - right: Box::new(Expression::Column(1)), - }, - )], - ) - .map_err(EngineError::from)?; - let rows = native_batch_rows(batch, vec![operator], context)?; - let mut output = Vec::new(); - for ((labels, a, b), row) in pairs.into_iter().zip(rows) { - let value = match row.first() { - Some(Cell::Float64(value)) => *value, - Some(Cell::Bool(value)) if boolean => { - if *value { - 1. - } else { - 0. - } - } - Some(Cell::Bool(true)) => { - if scalar_left { - b - } else { - a - } - } - Some(Cell::Bool(false)) => continue, - _ => return Err(miss("native binary result schema mismatch")), - }; - output.push(( - if arithmetic || boolean { - no_name(labels) - } else { - labels - }, - value, - )); - } - if scalar_output { - return Ok(Value::Scalar( - output - .first() - .ok_or_else(|| miss("missing scalar result"))? - .1, - )); - } - Ok(Value::Vector(vector(Value::Vector(output))?)) -} - -#[cfg(test)] -fn test_state_binding() -> QueryPlanNode { - let output = asap_types::sds::StoredOutputId(99); - QueryPlanNode::ReadMaterialization { - binding: asap_types::query_plan::MaterializationBinding { - stored_output_reference: asap_types::sds::StoredOutputReference::for_output(output), - materialization: output, - output_grouping: asap_types::query_plan::PhysicalGrouping::PerEntity, - item_labels: vec![], - window_ms: 1000, - pane_origin_ms: Some(0), - readout_lookback_ms: Some(300_000), - full_window_slide_ms: None, - }, - } -} - #[cfg(test)] fn test_native_context() -> physical::RunContext { physical::RunContext::new( @@ -1031,7 +556,7 @@ fn test_native_context() -> physical::RunContext { } #[cfg(test)] -mod topk_tests { +mod join_tests { use super::*; use asap_types::query_plan::{FallbackPolicy, InstantExecution}; @@ -1042,412 +567,6 @@ mod topk_tests { .collect() } - // An overflowing sum cannot implement average, but zero/subnormal averages remain valid. - #[test] - fn finite_division_guards_temporal_average_without_rejecting_zero() { - let mut sum = asap_summary_state::summary_kernels::sum::SumAccumulator::new(); - sum.update(1e308); - sum.update(1e308); - assert!(binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(sum.sum), - Value::Scalar(2.0) - ) - .is_err()); - for (a, b, expected) in [ - (0.0, 2.0, 0.0), - (10.0, 2.0, 5.0), - (f64::MIN_POSITIVE, 2.0, f64::MIN_POSITIVE / 2.0), - ] { - let Value::Scalar(value) = binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(a), - Value::Scalar(b), - ) - .unwrap() else { - panic!("scalar") - }; - assert_eq!(value, expected); - } - assert!(binary( - BinaryOperation::FiniteDiv, - false, - Value::Scalar(1.0), - Value::Scalar(0.0) - ) - .is_err()); - } - - // A conditional accuracy certificate must fall back rather than return an unbounded ratio. - #[test] - fn checked_relative_division_enforces_its_execution_domain() { - for (a, b) in [ - (1., 0.), - (0., 0.), - (1., f64::INFINITY), - (f64::NAN, 2.), - (f64::MAX, f64::MIN_POSITIVE), - (f64::MIN_POSITIVE, f64::MAX), - ] { - assert!(binary( - BinaryOperation::CheckedDiv, - false, - Value::Scalar(a), - Value::Scalar(b) - ) - .is_err()); - } - let Value::Scalar(value) = binary( - BinaryOperation::CheckedDiv, - false, - Value::Scalar(5.), - Value::Scalar(10.), - ) - .unwrap() else { - panic!("scalar"); - }; - assert_eq!(value, 0.5); - } - - #[test] - fn topk_selects_by_sample_value_and_preserves_series_labels() { - let values = vec![ - ( - labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "a")]), - 4.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "api"), ("pod", "b")]), - 9.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "c")]), - 7.0, - ), - ( - labels(&[("__name__", "cpu"), ("job", "db"), ("pod", "d")]), - 2.0, - ), - ]; - let selected = topk_selection( - 1, - &Grouping { - labels: vec!["job".into()], - without: false, - }, - values, - &test_native_context(), - ) - .unwrap(); - assert_eq!(selected.len(), 2); - assert_eq!(selected[0].0["pod"], "b"); - assert_eq!(selected[0].1, 9.0); - assert_eq!(selected[1].0["pod"], "c"); - assert_eq!(selected[1].1, 7.0); - assert!(selected - .iter() - .all(|(labels, _)| labels.contains_key("__name__"))); - } - - #[test] - fn topk_ranks_nan_below_numbers_and_keeps_exact_child_values() { - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - vec![ - (labels(&[("series", "nan")]), f64::NAN), - (labels(&[("series", "low")]), -1.0), - (labels(&[("series", "high")]), 3.0), - ], - &test_native_context(), - ) - .unwrap(); - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - selected, - &test_native_context(), - ) - .unwrap(); - assert_eq!( - selected - .iter() - .map(|row| row.0["series"].as_str()) - .collect::>(), - vec!["high", "low"] - ); - assert_eq!( - selected.iter().map(|row| row.1).collect::>(), - vec![3.0, -1.0] - ); - } - - #[test] - fn installed_topk_combines_with_prometheus_exact_child() { - let mut entry = control_plane::query_plan::query_time::compile_logical( - "hybrid-topk".into(), - "topk(2, m)".into(), - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::ExactBackend, - ) - .unwrap(); - control_plane::query_plan::query_time::finalize_query_time_nodes(&mut entry).unwrap(); - let leaf = entry - .nodes - .iter() - .find_map(|(id, node)| { - matches!( - node, - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { .. }, - .. - } - ) - .then_some(*id) - }) - .unwrap(); - let at = 1_000_u64; - let leaves = [( - (leaf, at as i64), - PreparedLeaf { - value: Value::Vector(vec![ - (labels(&[("pod", "a")]), 1.0), - (labels(&[("pod", "b")]), 8.0), - (labels(&[("pod", "c")]), 5.0), - ]), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - )] - .into_iter() - .collect(); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, stats) = execute_installed(&entry, &leaves, at, |_, _| { - panic!("summary callback must not run for an exact-child topk") - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("instant vector expected") - }; - assert_eq!( - result - .values - .iter() - .map(|point| point.value) - .collect::>(), - vec![8.0, 5.0] - ); - assert_eq!(stats.remote_branch_evaluations, 1); - assert_eq!(stats.remote_rpcs, 1); - assert_eq!(stats.raw_scan_evaluations, 0); - } - - // A temporal operator over an external subquery follows the same language - // policy as a summary readout; changing execution placement cannot drop names. - #[test] - fn metricsql_temporal_subdag_preserves_names_only_for_value_rollups() { - use control_plane::query_plan::QueryLanguage; - for language in [QueryLanguage::PromQl, QueryLanguage::MetricsQl] { - for operation in [ - TemporalOperation::Max, - TemporalOperation::Min, - TemporalOperation::Avg, - TemporalOperation::Sum, - TemporalOperation::Count, - TemporalOperation::Rate, - ] { - let entry = QueryPlanEntry { - physical_dag: None, - language, - query_id: "labels".into(), - canonical_query: "test".into(), - fixed_evaluation: None, - root: QueryNodeId(1), - nodes: BTreeMap::from([ - ( - QueryNodeId(0), - QueryPlanNode::Logical { - operator: QueryTimeOperator::ExactSubquery { - query: "m[1s]".into(), - }, - inputs: vec![], - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { operation }, - inputs: vec![QueryNodeId(0)], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 1000, - full_history: false, - cumulative_readout: true, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let leaves = BTreeMap::from([( - (QueryNodeId(0), 1000), - PreparedLeaf { - value: Value::Matrix( - vec![( - labels(&[("__name__", "m"), ("job", "api")]), - vec![(100, 1.), (900, 3.)], - )], - 0, - 1000, - ), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - )]); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, _) = execute_installed(&entry, &leaves, 1000, |_, _| { - panic!("external child supplied") - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("vector required") - }; - let expected = language == QueryLanguage::MetricsQl - && matches!( - operation, - TemporalOperation::Max | TemporalOperation::Min | TemporalOperation::Avg - ); - assert_eq!( - result.values[0] - .label_keys_override - .as_ref() - .unwrap() - .iter() - .any(|name| name == "__name__"), - expected, - "{language:?} {operation:?}" - ); - } - } - } - - #[test] - fn installed_topk_ranks_exact_rate_summary_values() { - let summary = QueryNodeId(0); - let root = QueryNodeId(1); - let entry = QueryPlanEntry { - physical_dag: None, - language: asap_types::query_plan::QueryLanguage::PromQl, - query_id: "summary-rate-topk".into(), - canonical_query: "topk(2, rate(requests_total[5m]))".into(), - fixed_evaluation: None, - root, - nodes: BTreeMap::from([ - (QueryNodeId(99), test_state_binding()), - ( - summary, - QueryPlanNode::ExactReadout { - input: QueryNodeId(99), - readout: asap_types::query_plan::ExactReadout::Rate, - }, - ), - ( - root, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 2, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(98)], - }, - ), - ( - QueryNodeId(98), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![summary], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - }; - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { - assert_eq!(id, summary); - assert_eq!(at, 300_000); - Ok(QueryResult::Vector( - crate::query_engines::query_result::InstantVector { - values: vec![ - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["a".into()]), - 0.4, - ) - .with_label_keys_override(vec!["pod".into()]), - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["b".into()]), - 1.2, - ) - .with_label_keys_override(vec!["pod".into()]), - InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["c".into()]), - 0.8, - ) - .with_label_keys_override(vec!["pod".into()]), - ], - timestamp: at, - warnings: vec![], - accuracy: None, - window_used: Some((0, at)), - }, - )) - }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("instant vector expected") - }; - assert_eq!( - result - .values - .iter() - .map(|point| point.value) - .collect::>(), - vec![1.2, 0.8] - ); - assert_eq!(stats.summary_readout_evaluations, 1); - assert_eq!(stats.remote_branch_evaluations, 0); - } - fn topk_membership_guarantee() -> planner_types::post_asap::ResultGuarantee { use planner_types::post_asap::{BoundExpr, ErrorMetric, ProbabilityExpr, ResultGuarantee}; ResultGuarantee { @@ -1507,53 +626,12 @@ mod topk_tests { } } - #[test] - fn candidate_sidecar_intersects_then_reranks_exact_values() { - let candidates = vec![ - (labels(&[("pod", "b")]), 99.0), - (labels(&[("pod", "c")]), 50.0), - ]; - let exact = vec![ - (labels(&[("pod", "a")]), 10.0), - (labels(&[("pod", "b")]), 8.0), - (labels(&[("pod", "c")]), 9.0), - ]; - let (selected, warning) = semi_join( - candidates, - exact, - &[("pod".into(), "pod".into())], - Some(&CandidateCompleteness::Certified { - guarantee: topk_membership_guarantee(), - }), - &test_native_context(), - ) - .unwrap(); - let selected = topk_selection( - 2, - &Grouping { - labels: vec![], - without: false, - }, - selected, - &test_native_context(), - ) - .unwrap(); - assert_eq!( - selected - .iter() - .map(|row| row.0["pod"].as_str()) - .collect::>(), - vec!["c", "b"] - ); - assert!(warning.is_none()); - } - + // The pruning join reads both the candidate and the exact readout. #[test] fn installed_candidate_sidecar_reads_both_summary_inputs() { let candidate_id = QueryNodeId(0); let value_id = QueryNodeId(1); - let filter = QueryNodeId(2); - let root = QueryNodeId(3); + let root = QueryNodeId(2); let entry = QueryPlanEntry { physical_dag: None, language: asap_types::query_plan::QueryLanguage::PromQl, @@ -1574,7 +652,7 @@ mod topk_tests { reason: "prepared exact counter readout".into(), }, ), - (filter, { + (root, { let schema = planner_types::post_asap::SummarySchema { fields: vec![planner_types::post_asap::SummaryField { name: "pod".into(), @@ -1647,33 +725,6 @@ mod topk_tests { } } }), - ( - root, - QueryPlanNode::Logical { - operator: QueryTimeOperator::Limit { - offset: 0, - n: 1, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![QueryNodeId(98)], - }, - ), - ( - QueryNodeId(98), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Sort { - descending: true, - grouping: Grouping { - labels: vec![], - without: false, - }, - }, - inputs: vec![filter], - }, - ), ]), instant: InstantExecution { lookback_ms: 300_000, @@ -1710,8 +761,6 @@ mod topk_tests { }, ), ]); - let mut entry = entry.clone(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); let (result, stats) = execute_installed(&entry, &leaves, at as u64, |_, _| { panic!("both inputs are prepared") }) @@ -1719,9 +768,15 @@ mod topk_tests { let QueryResult::Vector(result) = result else { panic!("vector expected") }; - assert_eq!(result.values.len(), 1); - assert_eq!(result.values[0].value, 3.0, "exact value is authoritative"); - assert_eq!(result.values[0].labels.labels, vec!["c"]); + // The candidates keep only b and c; their exact values are authoritative. + assert_eq!( + result + .values + .iter() + .map(|point| (point.labels.labels.clone(), point.value)) + .collect::>(), + vec![(vec!["b".to_string()], 1.0), (vec!["c".to_string()], 3.0)] + ); assert_eq!(stats.summary_readout_evaluations, 2); assert!(result.warnings.is_empty()); } @@ -1757,117 +812,127 @@ mod topk_tests { } #[cfg(test)] -mod shared_runtime_tests { +mod planner_computation_tests { use super::*; - use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryLanguage}; + use crate::query_engines::asap_query_engine::test_plan::planner_computed_entry; - fn entry() -> QueryPlanEntry { - QueryPlanEntry { - physical_dag: None, - language: QueryLanguage::PromQl, - query_id: "shared-grid".into(), - canonical_query: "shared-grid".into(), - fixed_evaluation: None, - root: QueryNodeId(3), - // The callback binds a readout boundary backed by the declared stored source. - nodes: BTreeMap::from([ - (QueryNodeId(99), test_state_binding()), - ( - QueryNodeId(0), - QueryPlanNode::ExactReadout { - input: QueryNodeId(99), - readout: asap_types::query_plan::ExactReadout::Sum, - }, - ), - ( - QueryNodeId(1), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Subquery { - range_ms: 2000, - step_ms: 1000, - offset_ms: 0, - }, - inputs: vec![QueryNodeId(0)], - }, - ), - ( - QueryNodeId(2), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Temporal { - operation: TemporalOperation::Sum, - }, - inputs: vec![QueryNodeId(1)], - }, - ), - ( - QueryNodeId(3), - QueryPlanNode::Logical { - operator: QueryTimeOperator::Binary { - operation: BinaryOperation::Add, - return_bool: false, - }, - inputs: vec![QueryNodeId(2), QueryNodeId(2)], - }, - ), - ]), - instant: InstantExecution { - lookback_ms: 2000, - full_history: false, - cumulative_readout: false, - }, - fallback: FallbackPolicy::ExactBackend, - } + fn readout(values: &[(&str, f64)], at: u64) -> QueryResult { + QueryResult::vector( + values + .iter() + .map(|(job, value)| { + InstantVectorElement::new( + KeyByLabelValues::new_with_labels(vec![(*job).into()]), + *value, + ) + .with_label_keys_override(vec!["job".into()]) + }) + .collect(), + at, + ) } - // A shared time-grid node runs once per query; distinct times and runs stay isolated. + fn readouts(entry: &QueryPlanEntry) -> Vec { + entry + .nodes + .iter() + .filter(|(_, node)| matches!(node, QueryPlanNode::ExactReadout { .. })) + .map(|(id, _)| *id) + .collect() + } + + // A ratio of grouped readouts, formerly a backend Binary operator, runs as + // one Planner physical fragment over the two readouts. #[test] - fn shared_subquery_scopes_do_not_duplicate_or_leak_values() { - let mut entry = entry(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let QueryPlanNode::PhysicalFragment { dag, .. } = &entry.nodes[&entry.root] else { - panic!("compiled root required") - }; - let graph: serde_json::Value = serde_json::from_slice(dag).unwrap(); - let shared = graph["nodes"] - .as_object() - .unwrap() + fn grouped_ratio_executes_as_one_planner_fragment() { + let entry = planner_computed_entry( + "sum by (job) (rate(errors_total[5m])) / sum by (job) (rate(requests_total[5m]))", + ); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::PhysicalFragment { .. } + )); + assert!(!entry + .nodes .values() - .find_map(|node| { - let operator = node.get("Operator")?; - operator["operator"]["kind"] - .get("VectorBinary") - .map(|_| operator["inputs"].as_array().unwrap()) - }) - .unwrap(); - assert_eq!(shared.len(), 2); - assert_eq!(shared[0], shared[1]); - let mut calls = Vec::new(); - for (at, expected) in [(3000, 10.), (4000, 14.)] { - let (result, stats) = execute_installed(&entry, &BTreeMap::new(), at, |id, time| { - assert_eq!(id, QueryNodeId(0)); - calls.push(time); - Ok(QueryResult::vector( - vec![InstantVectorElement::new( - KeyByLabelValues::new_with_labels(vec!["a".into()]), - time as f64 / 1000., - ) - .with_label_keys_override(vec!["pod".into()])], - time, - )) + .any(|node| matches!(node, QueryPlanNode::Logical { .. }))); + let [errors, requests] = readouts(&entry).try_into().unwrap(); + let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { + Ok(if id == errors { + readout(&[("api", 1.0), ("db", 3.0)], at) + } else { + assert_eq!(id, requests); + readout(&[("api", 4.0), ("db", 6.0), ("web", 1.0)], at) }) - .unwrap(); - let QueryResult::Vector(result) = result else { - panic!("vector required"); - }; - assert_eq!(result.values[0].value, expected); - assert_eq!(stats.summary_readout_evaluations, 2); - } - assert_eq!(calls, vec![2000, 3000, 3000, 4000]); + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + let mut values = result + .values + .iter() + .map(|point| (point.labels.labels.clone(), point.value)) + .collect::>(); + values.sort_by(|a, b| a.0.cmp(&b.0)); + assert_eq!( + values, + vec![ + (vec!["api".to_string()], 0.25), + (vec!["db".to_string()], 0.5) + ] + ); + assert_eq!(stats.summary_readout_evaluations, 2); + } + + // topk over an exact readout, formerly backend Sort and Limit operators, + // ranks in a Planner fragment and keeps the readout's series labels. + #[test] + fn topk_over_readout_executes_as_a_planner_fragment() { + let entry = planner_computed_entry("topk(2, rate(requests_total[5m]))"); + let QueryPlanNode::PhysicalFragment { + row_input: Some(0), .. + } = entry.nodes[&entry.root] + else { + panic!( + "expected a row-preserving Planner fragment: {:?}", + entry.nodes + ) + }; + let [summary] = readouts(&entry).try_into().unwrap(); + let (result, stats) = execute_installed(&entry, &BTreeMap::new(), 300_000, |id, at| { + assert_eq!(id, summary); + Ok(readout(&[("a", 0.4), ("b", 1.2), ("c", 0.8)], at)) + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector expected") + }; + assert_eq!( + result + .values + .iter() + .map(|point| (point.labels.labels[0].as_str(), point.value)) + .collect::>(), + vec![("b", 1.2), ("c", 0.8)] + ); + assert_eq!(stats.summary_readout_evaluations, 1); + } + + // Source failures keep their routing classification across the shared runtime. + #[test] + fn source_error_classification_survives_execution() { + let entry = planner_computed_entry("sum(rate(requests_total[5m])) * 2"); + let error = execute_installed(&entry, &BTreeMap::new(), 300_000, |_, _| { + Err(EngineError::capability_miss("source", "failed")) + }) + .unwrap_err(); + assert!(matches!(error,EngineError::CapabilityMiss{engine_id,..} if engine_id=="source")); } - // Query adapters use native computation and its parent execution budget. + // Planner scalar programs run under the parent request budget and cancellation. #[test] - fn native_scalar_and_aggregation_share_parent_resource_control() { + fn native_scalar_shares_parent_resource_control() { let context = test_native_context(); let graph = asap_physical_operators::physical_planner::promql_values::compile_scalar(7.) .unwrap() @@ -1877,32 +942,7 @@ mod shared_runtime_tests { native_values::complete_values(&graph, &[], context.clone()).unwrap(), Some(Value::Scalar(7.)) )); - let output = aggregate( - Aggregation::Sum, - &Grouping { - labels: vec![], - without: false, - }, - vec![(Labels::new(), 2.), (Labels::new(), 5.)], - &context, - ) - .unwrap(); - assert_eq!(output, vec![(Labels::new(), 7.)]); - assert!(context.peak_bytes() > 0); context.cancel(); assert!(native_values::complete_values(&graph, &[], context.clone()).is_err()); - assert!(negate(Value::Scalar(1.), &context).is_err()); - } - - // Source failures keep their routing classification across the shared runtime. - #[test] - fn source_error_classification_survives_execution() { - let mut entry = entry(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - let error = execute_installed(&entry, &BTreeMap::new(), 3000, |_, _| { - Err(EngineError::capability_miss("source", "failed")) - }) - .unwrap_err(); - assert!(matches!(error,EngineError::CapabilityMiss{engine_id,..} if engine_id=="source")); } } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index c69ae80bd..85b8519a6 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -330,102 +330,9 @@ mod tests { Error, }; - // An installed computation binds complete label maps; ranking and division are native. - #[test] - fn compiled_vector_composition_preserves_grouping_and_runs_independently() { - use super::super::{ - execute_installed, PreparedLeaf, PreparedLeaves, Value as ProtocolValue, - }; - use asap_types::query_plan::{ - query_time::QueryTimeOperator, FallbackPolicy, InstantExecution, QueryPlanNode, - }; - let query = "topk by (job) (1, sum without(instance) (left_metric) / sum without(instance) (right_metric))"; - let mut entry = control_plane::query_plan::query_time::compile_logical( - "compiled-values".into(), - query.into(), - InstantExecution { - lookback_ms: 300_000, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - ) - .unwrap(); - control_plane::query_plan::physical_values::compile(&mut entry).unwrap(); - entry.validate(&Default::default()).unwrap(); - assert!(entry.nodes.values().all(|node| matches!( - node, - QueryPlanNode::PhysicalFragment { .. } - | QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { .. }, - .. - } - ))); - for scale in [1., 2.] { - let mut leaves = PreparedLeaves::new(); - for (id, node) in &entry.nodes { - if let QueryPlanNode::Logical { - operator: QueryTimeOperator::Scan { metric, .. }, - .. - } = node - { - let left = metric.as_deref() == Some("left_metric"); - let values = [ - ("api", "a", if left { 4. * scale } else { 2. }), - ("api", "b", if left { 2. * scale } else { 1. }), - ("worker", "c", if left { 8. * scale } else { 2. }), - ]; - leaves.insert( - (*id, 1000), - PreparedLeaf { - value: ProtocolValue::Vector( - values - .into_iter() - .map(|(job, instance, value)| { - ( - Labels::from([ - ("__name__".into(), metric.clone().unwrap()), - ("job".into(), job.into()), - ("instance".into(), instance.into()), - ]), - value, - ) - }) - .collect(), - ), - remote: true, - remote_evaluations: 1, - remote_rpcs: 1, - }, - ); - } - } - let (result, _) = execute_installed(&entry, &leaves, 1000, |_, _| { - panic!("all inputs were bound") - }) - .unwrap(); - let crate::query_engines::query_result::QueryResult::Vector(result) = result else { - panic!("expected vector") - }; - let actual = result - .values - .into_iter() - .map(|point| (point.labels.labels[0].clone(), point.value)) - .collect::>(); - assert_eq!( - actual, - std::collections::BTreeMap::from([ - ("api".into(), 2. * scale), - ("worker".into(), 4. * scale) - ]) - ); - } - } - // An available label is bound by Backend; Planner evaluates its predicate. #[test] fn planner_filter_compiles_and_executes_bound_labels() { - use asap_types::query_plan::{FallbackPolicy, InstantExecution, QueryPlanNode}; use planner_types::{ post_asap::{ execution_data_state::lift_plain, ExecutionTiming, SummaryExpr, SummaryNode, @@ -470,24 +377,24 @@ mod tests { timing: ExecutionTiming::QueryTime, }, }); - let entry = control_plane::query_plan::compile_bound_mapped( - "filter".into(), - query.into(), - &root, - InstantExecution { - lookback_ms: 0, - full_history: false, - cumulative_readout: false, - }, - FallbackPolicy::Reject, - |_, _| panic!("filter requires no materialization"), - |_, _| {}, + // The pre-ASAP selector is the fragment's bound input, as an exact + // engine result would be. + let dag = planner_types::post_asap::compile_post_asap_dag(&root).unwrap(); + let source = dag.nodes.iter().find(|node| node.id != dag.root).unwrap(); + let program = asap_physical_operators::physical_planner::compile( + &dag, + [( + u64::from(source.id.0), + InputContract::bounded(std::sync::Arc::new(source.output_schema.clone())), + )] + .into(), + &[u64::from(dag.root.0)], ) .unwrap(); - let QueryPlanNode::PhysicalFragment { dag, row_input, .. } = &entry.nodes[&entry.root] - else { - panic!("filter was rejected: {:?}", entry.nodes) - }; + let row_input = program + .row_source(program.roots()[0]) + .and_then(|id| program.input_contracts().position(|(input, _)| input == id)); + let dag = program.encode().unwrap(); let values = vec![ ( [ @@ -501,7 +408,7 @@ mod tests { ]; let expected = values[0].clone(); assert_eq!( - physical(dag, vec![values], *row_input, 42, context(1 << 20)).unwrap(), + physical(&dag, vec![values], row_input, 42, context(1 << 20)).unwrap(), vec![expected] ); } diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 16ef57150..d4fb9f414 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -156,13 +156,6 @@ impl PhysicalQueryRuntime<'_> { context: &dag::RunContext, ) -> Result { match node { - QueryPlanNode::Scalar { .. } - | QueryPlanNode::Binary { .. } - | QueryPlanNode::ReduceSum { .. } => { - Err(PhysicalNodeError::Physical(dag::Error::Invalid( - "installed value computation requires a retained Planner physical graph".into(), - ))) - } QueryPlanNode::PhysicalFragment { dag, row_input: None, @@ -963,40 +956,6 @@ mod tests { } } - // Old computation nodes must fail explicitly rather than construct operators during a request. - #[test] - fn uncompiled_value_nodes_are_rejected() { - let store = SketchStore::new(); - let runtime = PhysicalQueryRuntime { - counter_parameters: Default::default(), - language: asap_types::query_plan::QueryLanguage::PromQl, - catalog: None, - context: QueryExecutionContext { - index: &store, - t0_ms: 0, - t1_ms: 2000, - is_cumulative: true, - allowed_materializations: None, - }, - }; - for node in [ - QueryPlanNode::Scalar { value: 1. }, - QueryPlanNode::Binary { - inputs: [QueryNodeId(0), QueryNodeId(0)], - operator: ArithmeticOpKind::Add, - }, - QueryPlanNode::ReduceSum { - input: QueryNodeId(0), - grouping: PhysicalGrouping::Reduce(vec![]), - }, - ] { - assert!( - matches!(runtime.execute_node(QueryNodeId(1), &node, &[], &test_value_context()), - Err(PhysicalNodeError::Physical(dag::Error::Invalid(message))) if message.contains("retained Planner physical graph")) - ); - } - } - // Real DAG resource failures survive both summary-readout and engine error adapters. #[test] fn summary_dag_resource_failures_never_become_capability_misses() { @@ -1949,7 +1908,8 @@ mod tests { )) .unwrap(); let entry = &mut snapshot["query_workload"]["repeating_queries"][0]; - entry["query"] = serde_json::json!("sum_over_time(a[1m]) / sum_over_time(a[10m])"); + entry["query"] = + serde_json::json!("sum(sum_over_time(a[1m])) / sum(sum_over_time(a[10m]))"); entry["requirements"]["accuracy"]["explicit"] = serde_json::json!("Exact"); entry["demand"]["fixed_interval_at"]["interval"] = serde_json::json!(60_000); let snapshot: BackendLocalPlanningInput = serde_json::from_value(snapshot).unwrap(); diff --git a/data_plane/src/query_engines/asap_query_engine/test_plan.rs b/data_plane/src/query_engines/asap_query_engine/test_plan.rs index 7da038b36..617c7794a 100644 --- a/data_plane/src/query_engines/asap_query_engine/test_plan.rs +++ b/data_plane/src/query_engines/asap_query_engine/test_plan.rs @@ -192,3 +192,49 @@ pub(super) fn bound_reference( } catalog.output_reference(output).unwrap() } + +/// An exact PromQL query lowered by the control plane: backend readouts over +/// fixture bindings, with Planner-compiled computation above them. +pub(super) fn planner_computed_entry(query: &str) -> QueryPlanEntry { + let canonical = canonical_promql(query).unwrap(); + let expr = control_plane::query_parser::parse_query_expr_canonical( + &canonical, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = control_plane::planner_selection::select_query( + &expr, + &control_plane::physical::post_asap::cost_model::ControlPlaneCostModel::new( + planner_types::types::AccuracyTarget::Exact, + ), + ) + .unwrap(); + let mut next = 0; + control_plane::query_plan::compile_bound_mapped( + canonical.clone(), + canonical, + &selected, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::ExactBackend, + |_, _| { + next += 1; + let output = asap_types::sds::StoredOutputId(next); + Ok(MaterializationBinding { + stored_output_reference: asap_types::sds::StoredOutputReference::for_output(output), + materialization: output, + output_grouping: PhysicalGrouping::PerEntity, + item_labels: vec![], + window_ms: 300_000, + pane_origin_ms: Some(0), + readout_lookback_ms: Some(300_000), + full_window_slide_ms: None, + }) + }, + |_, _| {}, + ) + .unwrap() +} diff --git a/data_plane/tests/support/issue_701_702_process.rs b/data_plane/tests/support/issue_701_702_process.rs index e617e575b..4989e6bc8 100644 --- a/data_plane/tests/support/issue_701_702_process.rs +++ b/data_plane/tests/support/issue_701_702_process.rs @@ -1,6 +1,5 @@ //! Issue workloads execute their selected Planner DAG on the production HTTP path. use super::*; -use asap_types::physical_plan_codec::PhysicalPlanCodec; use control_plane::physical::{ compiler::{ BackendLocalPlanningInput, DeploymentPlanCompiler, BACKEND_REVISION, PLANNER_REVISION, @@ -55,7 +54,7 @@ fn queries() -> Vec<(String, u64, u64)> { )); queries.push((format!("quantile by(job)({q}, issue701_data)"), 1, 1)); } - for operation in ["sum", "count", "avg", "min", "max"] { + for operation in ["sum", "count", "min", "max"] { queries.push((format!("{operation}_over_time(issue701_data[5m])"), 300, 30)); } for operation in ["sum", "count", "avg"] { @@ -77,7 +76,7 @@ fn queries() -> Vec<(String, u64, u64)> { queries } -// A single mixed workload covers moving windows, current series, minimum/average, +// A single mixed workload covers moving windows, current series and extrema, // without uncertified ratios. Optional native URL adds a real Prometheus differential oracle. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn issue_workloads_execute_warm_at_successive_evaluations() { @@ -312,7 +311,6 @@ async fn issue_702_individual_queries_execute_without_fallback() { "quantile by (job) (0.9, issue701_data)", "sum by (job) (issue701_data)", "sum(issue701_data)", - "avg_over_time(issue701_data[5m])", "count by (job) (issue701_data)", "count(issue701_data)", "avg by (job) (issue701_data)", @@ -369,9 +367,10 @@ fn issue_701_702_uncertified_ratios_require_exact_fallback() { } } -// Finite input can overflow sum; the installed average must fall back while zero stays warm. +// A per-series average is forwarded exactly while its sum and count stay warm; +// Planner does not yet match per-series rows in a division. #[tokio::test] -async fn temporal_average_overflow_falls_back_after_state_is_warm() { +async fn temporal_average_forwards_exactly_while_sum_and_count_stay_warm() { let native = std::env::var("ASAP_CURRENT_SERIES_PROMETHEUS_URL").ok(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let mock_url = format!("http://{}", listener.local_addr().unwrap()); @@ -399,26 +398,18 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .into(); let snapshot = quote_snapshot_for_test(serde_json::from_value(fixture).unwrap()); let plan = snapshot.clone().compile_promql().unwrap(); + let average = plan + .query_plan + .entries + .values() + .find(|entry| entry.canonical_query.starts_with("avg_over_time")) + .unwrap(); assert!( - plan.query_plan - .entries - .values() - .flat_map(|entry| entry.nodes.values()) - .any(|node| { - let control_plane::query_plan::QueryPlanNode::PhysicalFragment { dag, .. } = node - else { - return false; - }; - asap_physical_operators::physical_planner::CompiledPhysicalDag::decode(dag) - .unwrap(); - let document: serde_json::Value = serde_json::from_slice(dag).unwrap(); - document["nodes"].as_object().unwrap().values().any(|node| { - node["Operator"]["operator"]["kind"]["VectorBinary"]["operator"] - ["checked_finite_division"] - == true - }) - }), - "average must retain its native finite-division contract" + matches!( + average.nodes.get(&average.root), + Some(control_plane::query_plan::QueryPlanNode::ExactFallback { .. }) + ), + "a per-series average has no Planner-compiled local plan: {average:?}" ); let output = tempfile::tempdir().unwrap(); let path = output.path().join("snapshot.json"); @@ -471,20 +462,20 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .await; } let query = "avg_over_time(average_overflow[5s])"; - if value == 0.0 { - let result = wait_for_issue_warm_instant( - &client, - &backend, - query, - at, - &output.path().join("query_engine.log"), - ) - .await; - assert_eq!(first_value(&result, "value"), Some(0.0)); - } else { - let params = [("query", query.to_string()), ("time", at.to_string())]; - let actual: Value = client - .get(format!("{backend}/api/v1/query")) + let params = [("query", query.to_string()), ("time", at.to_string())]; + let actual: Value = client + .get(format!("{backend}/api/v1/query")) + .query(¶ms) + .send() + .await + .unwrap() + .json() + .await + .unwrap(); + assert!(!is_warm(&actual), "average must be forwarded: {actual}"); + if let Some(url) = &native { + let expected: Value = client + .get(format!("{url}/api/v1/query")) .query(¶ms) .send() .await @@ -492,23 +483,9 @@ async fn temporal_average_overflow_falls_back_after_state_is_warm() { .json() .await .unwrap(); - assert!( - !is_warm(&actual), - "overflowed average must fall back: {actual}" - ); + assert_eq!(actual["data"], expected["data"]); + } else { assert_eq!(first_value(&actual, "value"), Some(1e308), "{actual}"); - if let Some(url) = &native { - let expected: Value = client - .get(format!("{url}/api/v1/query")) - .query(¶ms) - .send() - .await - .unwrap() - .json() - .await - .unwrap(); - assert_eq!(actual["data"], expected["data"]); - } } } mock.abort(); diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index dc086a377..0404c1395 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -224,6 +224,15 @@ family or operator on that path is unsupported, the query entry is emitted as `ExactFallback`; the compiler does not provision otherwise-unused warm state for a query that cannot be executed end to end. +The backend lowers only stored-state readouts itself: `ReadMaterialization`, +`ExactReadout`, `SummaryEstimate` and `SummaryMerge`. Computation over their +decoded values is one `PhysicalFragment` per query-time region, compiled by +`asap_physical_operators::physical_planner::compile` with the readouts as input +contracts. Before binding, `preserve_uncompiled_computation_roots` checks that +compilation; a root Planner cannot compile, or one whose region reads a raw +PromQL selector, keeps no state and forwards the whole query as +`ExactFallback`. + Graph traversal is separate from node definitions and store semantics. Activation validates roots, edges, bindings, reachability, and cycles. The shared physical DAG runtime creates one producer per reachable node and From 2136735c211f598189c8590b87ab73dbb12de3ef Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 06:56:55 +0000 Subject: [PATCH 2/4] fix: drop __name__ from query-time physical results per PromQL rules Planner keeps __name__ in the series identity it computes, so a raw-series program for quantile_over_time(0.5, m[5m]) answered with __name__="m", which Prometheus drops. The compiler records on the Physical node whether the query result keeps the metric name (only a selector, through ordering, selection, subqueries and last_over_time does), and the adapter removes it when decoding series identities. Co-Authored-By: Claude Opus 5.5 --- control_plane/src/physical/compiler.rs | 12 +- control_plane/src/query_plan.rs | 21 +++ crates/asap_types/src/query_plan.rs | 4 + crates/asap_types/src/query_plan/native.rs | 12 ++ .../logical_dag/native_values.rs | 126 ++++++++++-------- .../asap_query_engine/raw_source.rs | 25 ++++ .../control-plane/physical-compiler.md | 4 +- 7 files changed, 146 insertions(+), 58 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 016658d9e..1ca317c9f 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2244,7 +2244,12 @@ impl DeploymentPlanCompiler { None }; let mut entry = if let Some(raw) = placement.raw_program(query_index) { - Ok(raw_query_time_entry(query, canonical.clone(), raw)?) + Ok(raw_query_time_entry( + query, + canonical.clone(), + &original_root(query, query_index, &request.canonical_roots)?, + raw, + )?) } else if let Some((source, program)) = native_rate { let native_state_binding = if let SummaryExpr::SummaryAgg { family: @@ -2320,6 +2325,9 @@ impl DeploymentPlanCompiler { max_bytes: request .retained_summary_memory_budget_bytes .unwrap_or(64 * 1024 * 1024), + drop_metric_name: !crate::query_plan::result_keeps_metric_name( + &original_root(query, query_index, &request.canonical_roots)?, + ), }, ); entry.root = root; @@ -2735,6 +2743,7 @@ fn exact_query_entry( fn raw_query_time_entry( query: &QueryCompilationInput, canonical: String, + root_expr: &QueryExpr, raw: &placement::RawQueryTimeProgram, ) -> Result { let mut nodes = BTreeMap::new(); @@ -2759,6 +2768,7 @@ fn raw_query_time_entry( inputs, source_nodes, max_bytes: 64 * 1024 * 1024, + drop_metric_name: !crate::query_plan::result_keeps_metric_name(root_expr), }, ); let entry = QueryPlanEntry { diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index f0d57e03d..929da269f 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -652,6 +652,27 @@ where } } +/// Does PromQL keep `__name__` on this query's result series? Only a series +/// selector keeps it, through ordering, selection, subqueries and +/// `last_over_time`; functions, aggregations and arithmetic drop it. +pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bool { + use planner_types::pre_asap::{AggIntent, QueryExpr}; + match expr { + QueryExpr::Scan { .. } => true, + QueryExpr::TimeRange { child, .. } + | QueryExpr::TimeShift { child, .. } + | QueryExpr::Sort { child, .. } + | QueryExpr::Limit { child, .. } + | QueryExpr::PromqlSubquery { child, .. } => result_keeps_metric_name(child), + QueryExpr::Aggregate { + child, measures, .. + } if matches!(measures.as_slice(), [AggIntent::LastOverTime]) => { + result_keeps_metric_name(child) + } + _ => false, + } +} + /// Parameters of each `kind` operator inside a physical fragment, for tests. #[cfg(test)] pub(crate) fn operator_parameters(node: &QueryPlanNode, kind: &str) -> Vec { diff --git a/crates/asap_types/src/query_plan.rs b/crates/asap_types/src/query_plan.rs index 614ef9e4b..8aa371acb 100644 --- a/crates/asap_types/src/query_plan.rs +++ b/crates/asap_types/src/query_plan.rs @@ -822,6 +822,10 @@ pub enum QueryPlanNode { inputs: Vec, source_nodes: Vec, max_bytes: u64, + /// PromQL removes `__name__` from the series identity of this result; + /// Planner keeps it in the identity it computes. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + drop_metric_name: bool, }, /// Complete Planner-compiled relation computation; inputs follow its typed slots. PhysicalRelation { diff --git a/crates/asap_types/src/query_plan/native.rs b/crates/asap_types/src/query_plan/native.rs index 4f9feb627..42b10e3da 100644 --- a/crates/asap_types/src/query_plan/native.rs +++ b/crates/asap_types/src/query_plan/native.rs @@ -92,11 +92,23 @@ impl QueryPlanEntry { inputs, source_nodes, max_bytes, + .. }) => Some((inputs, source_nodes, *max_bytes)), _ => None, } } + /// Whether the root physical result drops `__name__` from series identities. + pub fn drops_metric_name(&self) -> bool { + matches!( + self.nodes.get(&self.root), + Some(QueryPlanNode::Physical { + drop_metric_name: true, + .. + }) + ) + } + /// Validate bound counter vectors or stored aggregate batches against the /// retained physical program; recovery never lowers logical operators. pub fn recover_vector_physical_dag(&self) -> Result { diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 85b8519a6..3a0585aa3 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -495,7 +495,7 @@ mod tests { #[test] fn selected_candidate_input_budget_is_a_terminal_error() { let plan = CompiledPhysicalDag::decode(&sorted()).unwrap(); - let error = execute_batches(&plan, 1, 1, 42, |_, contract| { + let error = execute_batches(&plan, 1, 1, 42, false, |_, contract| { Ok(BoundInput::Rows(Batch::try_new( contract.schema.clone(), vec![vec![Value::Float64(1.)]], @@ -675,6 +675,7 @@ where max_bytes, bindings.len(), at, + entry.drops_metric_name(), |input_id, contract| { let schema = &contract.schema; let values = super::vector(super::from_result(callback(bindings[&input_id], at)?)?)?; @@ -717,59 +718,67 @@ pub(in crate::query_engines::asap_query_engine) fn execute_stored( let program = entry .recover_vector_physical_dag() .map_err(|e| miss(e.to_string()))?; - execute_batches(&program, max_bytes, inputs.len(), at, |id, contract| { - let schema = &contract.schema; - let index = sources - .iter() - .position(|source| *source == id) - .ok_or_else(|| miss("native source is unbound"))?; - let node = entry.nodes.get(&inputs[index]); - if let Some(asap_types::query_plan::QueryPlanNode::Logical { operator, .. }) = node { - let (client, endpoint) = raw_endpoint - .ok_or_else(|| miss("query-time raw input has no Prometheus endpoint"))?; - let at = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; - return Ok(BoundInput::Lazy(super::super::raw_source::bind( - contract, operator, at, client, endpoint, - )?)); - } - let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = node - else { - return Err(miss("native stored source has no deployed summary binding")); - }; - let store = store.ok_or_else(|| { - EngineError::capability_miss("native_stored", "summary store unavailable") - })?; - let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; - let start = at - .checked_sub(binding.window_ms) - .ok_or_else(|| miss("native window underflow"))?; - let address = asap_types::sds::StoredSummaryKey { - plan_id, - plan_version, - stored_output_id: binding.stored_output_reference.stored_output_id, - population: std::collections::BTreeMap::new(), - window: asap_types::sds::HalfOpenTimeRange { - start_ms: start as i64, - end_ms: end, - }, - }; - store - .read_bound_native_summary( - &address, - &binding.stored_output_reference, - schema.clone(), - max_bytes as usize, - ) - .map(BoundInput::Rows) - .map_err(|error| match error { - crate::storage_engines::sketch_db::index::NativeReadError::Unavailable(message) => { - miss(message) - } - crate::storage_engines::sketch_db::index::NativeReadError::Physical(error) => { - EngineError::from(error) - } - }) - }) + let drop_metric_name = entry.drops_metric_name(); + execute_batches( + &program, + max_bytes, + inputs.len(), + at, + drop_metric_name, + |id, contract| { + let schema = &contract.schema; + let index = sources + .iter() + .position(|source| *source == id) + .ok_or_else(|| miss("native source is unbound"))?; + let node = entry.nodes.get(&inputs[index]); + if let Some(asap_types::query_plan::QueryPlanNode::Logical { operator, .. }) = node { + let (client, endpoint) = raw_endpoint + .ok_or_else(|| miss("query-time raw input has no Prometheus endpoint"))?; + let at = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; + return Ok(BoundInput::Lazy(super::super::raw_source::bind( + contract, operator, at, client, endpoint, + )?)); + } + let Some(asap_types::query_plan::QueryPlanNode::ReadMaterialization { binding }) = node + else { + return Err(miss("native stored source has no deployed summary binding")); + }; + let store = store.ok_or_else(|| { + EngineError::capability_miss("native_stored", "summary store unavailable") + })?; + let end = i64::try_from(at).map_err(|_| miss("native timestamp overflow"))?; + let start = at + .checked_sub(binding.window_ms) + .ok_or_else(|| miss("native window underflow"))?; + let address = asap_types::sds::StoredSummaryKey { + plan_id, + plan_version, + stored_output_id: binding.stored_output_reference.stored_output_id, + population: std::collections::BTreeMap::new(), + window: asap_types::sds::HalfOpenTimeRange { + start_ms: start as i64, + end_ms: end, + }, + }; + store + .read_bound_native_summary( + &address, + &binding.stored_output_reference, + schema.clone(), + max_bytes as usize, + ) + .map(BoundInput::Rows) + .map_err(|error| match error { + crate::storage_engines::sketch_db::index::NativeReadError::Unavailable( + message, + ) => miss(message), + crate::storage_engines::sketch_db::index::NativeReadError::Physical(error) => { + EngineError::from(error) + } + }) + }, + ) } /// Stored and protocol inputs are read before execution; query-time raw @@ -784,6 +793,7 @@ fn execute_batches( max_bytes: u64, input_count: usize, at: u64, + drop_metric_name: bool, mut input_batch: impl FnMut( u64, &asap_physical_operators::physical_planner::InputContract, @@ -870,7 +880,11 @@ fn execute_batches( let Value::Utf8(encoded) = &row[identity] else { return Err(miss("invalid physical series identity")); }; - decode_series_identity(encoded).map_err(EngineError::from)? + let mut labels = decode_series_identity(encoded).map_err(EngineError::from)?; + if drop_metric_name { + labels.remove("__name__"); + } + labels } else { batch .schema() @@ -935,7 +949,7 @@ mod request_contract_tests { Default::default(), vec![0], )?; - execute_batches(&program, 64 * 1024, 1, 0, |_, _| { + execute_batches(&program, 64 * 1024, 1, 0, false, |_, _| { Batch::try_new(schema.clone(), vec![vec![Value::Float64(1.0)]]) .map(BoundInput::Rows) .map_err(EngineError::from) diff --git a/data_plane/src/query_engines/asap_query_engine/raw_source.rs b/data_plane/src/query_engines/asap_query_engine/raw_source.rs index fa72e089f..b3d84484c 100644 --- a/data_plane/src/query_engines/asap_query_engine/raw_source.rs +++ b/data_plane/src/query_engines/asap_query_engine/raw_source.rs @@ -596,6 +596,13 @@ mod tests { inputs: vec![QueryNodeId(0)], source_nodes: vec![slot], max_bytes, + drop_metric_name: !control_plane::query_plan::result_keeps_metric_name( + &control_plane::query_parser::parse_query_expr_canonical( + QUERY, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(), + ), }, ), ]), @@ -655,6 +662,24 @@ mod tests { server.abort(); } + // PromQL drops the metric name from a function result; the raw rows keep + // it in their series identity, so the adapter removes it. + #[tokio::test] + async fn raw_program_results_drop_the_metric_name() { + let (endpoint, _, server) = prometheus(200, matrix(), std::time::Duration::ZERO).await; + let result = engine(endpoint).execute_at(QUERY, AT as u64).await.unwrap(); + let crate::query_engines::query_result::QueryResult::Vector(result) = result else { + panic!("expected an instant vector") + }; + assert_eq!(result.values.len(), 2); + for point in &result.values { + let keys = point.label_keys_override.as_ref().unwrap(); + assert!(!keys.iter().any(|key| key == "__name__"), "{keys:?}"); + assert!(keys.iter().any(|key| key == "instance"), "{keys:?}"); + } + server.abort(); + } + // An installed raw input fails the query when its endpoint fails or is not // configured; it is never answered as an empty vector. #[tokio::test] diff --git a/docs/developer_docs/control-plane/physical-compiler.md b/docs/developer_docs/control-plane/physical-compiler.md index 0404c1395..1769efaf5 100644 --- a/docs/developer_docs/control-plane/physical-compiler.md +++ b/docs/developer_docs/control-plane/physical-compiler.md @@ -231,7 +231,9 @@ decoded values is one `PhysicalFragment` per query-time region, compiled by contracts. Before binding, `preserve_uncompiled_computation_roots` checks that compilation; a root Planner cannot compile, or one whose region reads a raw PromQL selector, keeps no state and forwards the whole query as -`ExactFallback`. +`ExactFallback`. A query-time program over raw selectors (`Physical`) records +whether PromQL drops `__name__` from its result, since Planner keeps it in the +series identity. Graph traversal is separate from node definitions and store semantics. Activation validates roots, edges, bindings, reachability, and cycles. From 4f3c436a6e11f398ea906a1931127cd007c8c99d Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 07:22:47 +0000 Subject: [PATCH 3/4] fix: forward uncompiled queries without stranding state; tighten readout checks Review follow-ups: - A query whose readouts have no maintained state is forwarded whole in every mode; a query that compiled state but cannot bind it fails the candidate instead of maintaining that state unread. Previously a non-mixed deployment failed to compile, e.g. a ratio of two grouped rates whose states were not selected. - Pruned joins read both sides as stored state; a pruned join below a region root is not compiled, since its contract binds only at the root. - An instant count, which has no local exact readout, is not an input to a Planner fragment. - Range-maximum states require a named metric. - first_over_time, filters, relabeling and limitk keep __name__; series that collide once it is dropped are rejected, as PromQL does. Co-Authored-By: Claude Opus 5.5 --- control_plane/src/physical/compiler.rs | 53 ++++++++++---- control_plane/src/query_plan.rs | 73 +++++++++++++++++-- control_plane/src/query_plan/query_time.rs | 7 +- .../logical_dag/native_values.rs | 8 ++ .../asap_query_engine/raw_source.rs | 17 +++++ 5 files changed, 134 insertions(+), 24 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 1ca317c9f..43a911c80 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2418,10 +2418,11 @@ impl DeploymentPlanCompiler { } }, ) { - // A selected state without a binding, or computation Planner - // cannot compile over readouts, leaves no local plan. + // A readout with no maintained state leaves no local plan; + // the query is forwarded whole. A query that did compile + // state must not maintain it unread, so it fails instead. Err(crate::query_plan::QueryPlanError::UnsupportedNode(_)) - if request.allow_mixed_summary_and_exact_execution => + if executable_dags[query_index].is_none() => { query_node_bindings.retain(|(index, _), _| *index != query_index); Ok(exact_query_entry(query, canonical.clone(), instant)) @@ -4483,17 +4484,14 @@ fn collect_selected_materializations( grouping.clone(), selected, )?; - // Explicit external authoritative values do not need duplicate local state. - if !composable { - walk( - values, - readout, - composable, - native_maintenance, - grouping.clone(), - selected, - )?; - } + walk( + values, + readout, + composable, + native_maintenance, + grouping.clone(), + selected, + )?; } SummaryExpr::ValueOperation { child, .. } => { walk( @@ -5125,6 +5123,32 @@ pub(crate) mod tests { } } + // Without mixed execution, a computation Planner can compile over readouts + // whose states the deployment does not select forwards whole instead of + // failing the deployment. + #[test] + fn unselected_states_under_compilable_computation_forward_whole() { + let query = "sum by (job) (rate(a[5m])) / sum by (job) (rate(b[5m]))"; + let workload = request("ratio", query); + assert!(!workload.allow_mixed_summary_and_exact_execution); + let plan = DeploymentPlanCompiler + .compile_promql(workload, environment(10_000)) + .unwrap(); + let entry = plan.query_plan.lookup(query).unwrap(); + if entry.materialization_bindings().is_empty() { + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + )); + assert!(plan.precompute_plan.materializations.is_empty()); + } else { + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::PhysicalFragment { .. } + )); + } + } + // avg_over_time divides two per-series readouts. Planner does not yet // match per-series rows in a Binary, so no candidate keeps local state and // the exact engine evaluates the query. @@ -6204,6 +6228,7 @@ pub(crate) mod tests { )); assert_eq!(entry.canonical_query, query); assert!(entry.materialization_bindings().is_empty()); + assert!(plan.precompute_plan.materializations.is_empty()); } #[test] diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 929da269f..08f4776e6 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -208,16 +208,27 @@ pub(crate) fn compile_query_computation( continue; } let semantic = semantic.expect("readout has a semantic node"); + let source = exact_accumulator_value_source(semantic).unwrap_or(semantic); if matches!( semantic.expr, SummaryExpr::SummaryJoin { .. } | SummaryExpr::SummarySubtract { .. } | SummaryExpr::SummaryDelete { .. } - ) || node - .output_schema - .fields - .iter() - .any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))) + ) || (matches!( + source.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Count, + _ + ), + .. + } + ) && !exact_value_executable(source)) + || node + .output_schema + .fields + .iter() + .any(|field| !matches!(field.dtype, SummaryFamilyType::Plain(_))) { return Err(unsupported( "query-time computation consumes state that has no decoded readout".into(), @@ -229,6 +240,21 @@ pub(crate) fn compile_query_computation( ); frontier.insert(u64::from(id.0), Rc::clone(semantic)); } + // A pruning contract is bound only at the fragment root. + if dag.nodes.iter().any(|node| { + node.id != dag.root + && matches!( + node.payload, + Payload::RelationalJoin { + pruning: Some(_), + .. + } + ) + }) { + return Err(unsupported( + "a pruned join must be the root of its query-time region".into(), + )); + } let physical = compile(dag, contracts, &[u64::from(dag.root.0)]) .map_err(|error| unsupported(error.to_string()))?; let pruning = match &root.expr { @@ -653,8 +679,9 @@ where } /// Does PromQL keep `__name__` on this query's result series? Only a series -/// selector keeps it, through ordering, selection, subqueries and -/// `last_over_time`; functions, aggregations and arithmetic drop it. +/// selector keeps it, through ordering, selection, filtering, relabeling, +/// subqueries and `first_`/`last_over_time`; other functions, aggregations and +/// arithmetic drop it. pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bool { use planner_types::pre_asap::{AggIntent, QueryExpr}; match expr { @@ -663,10 +690,17 @@ pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bo | QueryExpr::TimeShift { child, .. } | QueryExpr::Sort { child, .. } | QueryExpr::Limit { child, .. } + | QueryExpr::Filter { child, .. } + | QueryExpr::PromqlSeriesSample { child, .. } + | QueryExpr::PromqlRelabel { child, .. } | QueryExpr::PromqlSubquery { child, .. } => result_keeps_metric_name(child), QueryExpr::Aggregate { child, measures, .. - } if matches!(measures.as_slice(), [AggIntent::LastOverTime]) => { + } if matches!( + measures.as_slice(), + [AggIntent::LastOverTime | AggIntent::FirstOverTime] + ) => + { result_keeps_metric_name(child) } _ => false, @@ -1156,6 +1190,29 @@ mod tests { assert!(fragments > 0); } + // Instant counts have no local exact readout, so a computation over one + // is not compiled locally; count_over_time keeps its readout. + #[test] + fn computation_over_instant_count_is_not_local() { + let selected = |query: &str| { + let canonical = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + crate::planner_selection::plan_test_query(&canonical).unwrap() + }; + let instant = selected("count(m) * 2"); + assert!(is_query_computation(&instant)); + let Err(error) = compile_query_computation(&instant) else { + panic!("instant count computation compiled locally"); + }; + assert!(error.to_string().contains("no decoded readout"), "{error}"); + let temporal = selected("sum(count_over_time(m[5m])) * 2"); + assert!(is_query_computation(&temporal)); + compile_query_computation(&temporal).unwrap(); + } + #[test] fn canonical_identity_ignores_formatting() { assert_eq!( diff --git a/control_plane/src/query_plan/query_time.rs b/control_plane/src/query_plan/query_time.rs index 39434e519..a035ff428 100644 --- a/control_plane/src/query_plan/query_time.rs +++ b/control_plane/src/query_plan/query_time.rs @@ -3,7 +3,7 @@ pub use asap_types::query_plan::query_time::*; use planner_types::post_asap::{ExactKind, SummaryExpr, SummaryFamilyType, SummaryNode}; -use planner_types::pre_asap::{QueryExpr, Reduction}; +use planner_types::pre_asap::{QueryExpr, Reduction, Source}; /// A per-series `max_over_time` over one plain range selector, the only /// maximum state the backend maintains. Planner distinguishes Max from Min @@ -30,7 +30,10 @@ pub(crate) fn is_range_max_materialization(node: &SummaryNode) -> bool { &child.expr, SummaryExpr::KeepPreAsap(expr) if matches!( expr.as_ref(), - QueryExpr::TimeRange { child, .. } if matches!(child.as_ref(), QueryExpr::Scan { .. }) + QueryExpr::TimeRange { child, .. } if matches!( + child.as_ref(), + QueryExpr::Scan { source: Source::TimeSeries { metric }, .. } if !metric.is_empty() + ) ) ) } diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs index 3a0585aa3..e62a0409a 100644 --- a/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag/native_values.rs @@ -859,6 +859,7 @@ fn execute_batches( .remove(0); let values = crate::query_engines::request::drive(async { let mut values = Vec::new(); + let mut renamed = std::collections::BTreeSet::new(); while let Some(batch) = stream.next().await { let batch = batch.map_err(EngineError::from)?; let identity = batch @@ -883,6 +884,13 @@ fn execute_batches( let mut labels = decode_series_identity(encoded).map_err(EngineError::from)?; if drop_metric_name { labels.remove("__name__"); + // PromQL rejects a result whose series collide once + // the name is dropped; the exact engine reports it. + if !renamed.insert(labels.clone()) { + return Err(miss( + "vector cannot contain metrics with the same labelset", + )); + } } labels } else { diff --git a/data_plane/src/query_engines/asap_query_engine/raw_source.rs b/data_plane/src/query_engines/asap_query_engine/raw_source.rs index b3d84484c..5928918d2 100644 --- a/data_plane/src/query_engines/asap_query_engine/raw_source.rs +++ b/data_plane/src/query_engines/asap_query_engine/raw_source.rs @@ -680,6 +680,23 @@ mod tests { server.abort(); } + // Series that collide once the name is dropped are a PromQL error, not two + // results with one label set. + #[tokio::test] + async fn raw_program_rejects_series_that_collide_without_the_metric_name() { + let body = serde_json::json!({"status": "success", "data": {"resultType": "matrix", "result": [ + {"metric": {"__name__": "m", "job": "api", "instance": "a"}, "values": [[900, "1"]]}, + {"metric": {"__name__": "n", "job": "api", "instance": "a"}, "values": [[900, "2"]]} + ]}}); + let (endpoint, _, server) = prometheus(200, body, std::time::Duration::ZERO).await; + let error = engine(endpoint) + .execute_at(QUERY, AT as u64) + .await + .unwrap_err(); + assert!(error.to_string().contains("same labelset"), "{error}"); + server.abort(); + } + // An installed raw input fails the query when its endpoint fails or is not // configured; it is never answered as an empty vector. #[tokio::test] From c1becfba893d2b5aa52f861b9c0006ab3f430e4c Mon Sep 17 00:00:00 2001 From: zzylol Date: Wed, 30 Sep 2026 07:40:43 +0000 Subject: [PATCH 4/4] test: pin whole-query forwarding; name the query when binding fails The non-mixed ratio test now requires forwarding with no maintained state, a binding failure names its query in the candidate error, and the metric name follows the left side of and/unless. Co-Authored-By: Claude Opus 5.5 --- control_plane/src/physical/compiler.rs | 22 ++++++--------- control_plane/src/query_plan.rs | 39 ++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 16 deletions(-) diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index 43a911c80..4be16cfd4 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -2427,7 +2427,10 @@ impl DeploymentPlanCompiler { query_node_bindings.retain(|(index, _), _| *index != query_index); Ok(exact_query_entry(query, canonical.clone(), instant)) } - other => other, + other => other.map_err(|error| CompileError::Query { + query_id: query.query_id.clone(), + reason: error.to_string(), + }), } }?; if frontend == QueryFrontend::MetricsQl { @@ -5135,18 +5138,11 @@ pub(crate) mod tests { .compile_promql(workload, environment(10_000)) .unwrap(); let entry = plan.query_plan.lookup(query).unwrap(); - if entry.materialization_bindings().is_empty() { - assert!(matches!( - &entry.nodes[&entry.root], - crate::query_plan::QueryPlanNode::ExactFallback { .. } - )); - assert!(plan.precompute_plan.materializations.is_empty()); - } else { - assert!(matches!( - &entry.nodes[&entry.root], - crate::query_plan::QueryPlanNode::PhysicalFragment { .. } - )); - } + assert!(matches!( + &entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + )); + assert!(plan.precompute_plan.materializations.is_empty()); } // avg_over_time divides two per-series readouts. Planner does not yet diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 08f4776e6..8db586aa6 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -680,10 +680,12 @@ where /// Does PromQL keep `__name__` on this query's result series? Only a series /// selector keeps it, through ordering, selection, filtering, relabeling, -/// subqueries and `first_`/`last_over_time`; other functions, aggregations and -/// arithmetic drop it. +/// subqueries, `first_`/`last_over_time` and the left side of `and`/`unless`; +/// other functions, aggregations and arithmetic drop it. A comparison keeps it +/// unless it has `bool`, which the IR does not record; Planner compiles no +/// comparison, so the rule treats them as dropping it. pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bool { - use planner_types::pre_asap::{AggIntent, QueryExpr}; + use planner_types::pre_asap::{AggIntent, BinaryOpKind, PromQLVectorSetOpKind, QueryExpr}; match expr { QueryExpr::Scan { .. } => true, QueryExpr::TimeRange { child, .. } @@ -694,6 +696,11 @@ pub fn result_keeps_metric_name(expr: &planner_types::pre_asap::QueryExpr) -> bo | QueryExpr::PromqlSeriesSample { child, .. } | QueryExpr::PromqlRelabel { child, .. } | QueryExpr::PromqlSubquery { child, .. } => result_keeps_metric_name(child), + QueryExpr::BinaryOp { + op: BinaryOpKind::Set(PromQLVectorSetOpKind::And | PromQLVectorSetOpKind::Unless), + lhs, + .. + } => result_keeps_metric_name(lhs), QueryExpr::Aggregate { child, measures, .. } if matches!( @@ -1213,6 +1220,32 @@ mod tests { compile_query_computation(&temporal).unwrap(); } + // PromQL keeps `__name__` only on selections of a series selector. + #[test] + fn metric_name_follows_promql_result_rules() { + for (query, keeps) in [ + ("m", true), + ("m offset 5m", true), + ("sort(m)", true), + ("topk(2, m)", true), + ("last_over_time(m[5m])", true), + ("m and n", true), + ("rate(m[5m])", false), + ("quantile_over_time(0.5, m[5m])", false), + ("m * 2", false), + ("sum by (job) (m)", false), + ("topk(2, rate(m[5m]))", false), + ("max_over_time(rate(m[1m])[5m:1m])", false), + ] { + let expr = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + assert_eq!(result_keeps_metric_name(&expr), keeps, "{query}"); + } + } + #[test] fn canonical_identity_ignores_formatting() { assert_eq!(