diff --git a/crates/asap-aware-mapping/src/accuracy/reconciliation.rs b/crates/asap-aware-mapping/src/accuracy/reconciliation.rs index 62a82ec26..9d4722a6c 100644 --- a/crates/asap-aware-mapping/src/accuracy/reconciliation.rs +++ b/crates/asap-aware-mapping/src/accuracy/reconciliation.rs @@ -3,7 +3,7 @@ //! //! ## The gap this closes //! -//! `asap_types::pre_asap::cse::share_common_sub_dags` (pre-ASAP CSE) only +//! `asap_types::ir::cse::share_common_sub_dags` (pre-ASAP CSE) only //! ever merges two sub-DAGs that are *exactly* [`PartialEq`]-equal, //! including their [`AggIntent`]'s `accuracy: AccuracyTarget` field. Two //! otherwise-identical aggregates that differ *only* in how tight an @@ -303,7 +303,7 @@ impl AccuracyReconciliationStrategy { /// /// Also requires the candidate's own *output* schema to carry a provable /// unique key ([`Schema::has_unique_key`]) — the exact legality gate - /// `pre_asap::cse::share_common_sub_dags` already applies to its own + /// `ir::cse::share_common_sub_dags` already applies to its own /// sharing decisions, and [`crate::rollup::RollupStrategy`] already /// reuses verbatim for the identical reason (see that module's /// `is_legal_rollup_source` doc, point 4): a producer's output is only diff --git a/crates/asap-aware-mapping/src/explanation.rs b/crates/asap-aware-mapping/src/explanation.rs index 810c7c3e7..f29a46c6f 100644 --- a/crates/asap-aware-mapping/src/explanation.rs +++ b/crates/asap-aware-mapping/src/explanation.rs @@ -700,7 +700,7 @@ mod tests { /// A sketch-applicable `Aggregate` reachable via two paths that CSE /// collapses onto one `Rc` — the same `median(x) == median(x)` shape - /// `pre_asap::cse`'s own `single_query_shares_its_own_repeated_sub-DAG` + /// `ir::cse`'s own `single_query_shares_its_own_repeated_sub_dag` /// test uses — must be reported once, not once per path: it is exactly /// one [`crate::replacement::TargetSubDAGCandidates`], keyed by `Rc` pointer identity, /// not one per path that reaches it. diff --git a/crates/asap-aware-mapping/src/lib.rs b/crates/asap-aware-mapping/src/lib.rs index 0b84000a2..8f6ffed47 100644 --- a/crates/asap-aware-mapping/src/lib.rs +++ b/crates/asap-aware-mapping/src/lib.rs @@ -22,9 +22,9 @@ //! It depends only on the IR crate, never on a front end — the layering //! invariant (arrows point up) holds here too. //! -//! Post-lowering **canonicalization** is *not* here: it landed in -//! `asap_types::pre_asap::canonicalize`, run inside the shared `resolve_root` -//! so every front end normalizes before the pre-ASAP IR leaves resolution +//! Post-lowering **canonicalization** is *not* here: it lives in +//! `asap_types::ir::canonicalize`, run inside `asap_frontend_common`'s shared +//! `resolve_root` so every front end normalizes before the IR leaves resolution //! (issue #34, closed). //! //! ## Planning workflows @@ -83,7 +83,7 @@ //! re-deriving it from an already-computed, strictly finer sibling //! `Aggregate` over identical child IR instead of an independent pass //! over the raw source — the cross-aggregate sibling of -//! `pre_asap::cse::share_common_sub_dags`'s identical-sub-DAG sharing. +//! `ir::cse::share_common_sub_dags`'s identical-sub-DAG sharing. //! [`rollup::is_legal_rollup_source`] is the standalone legality predicate //! other axes (e.g. issue #256's `GroupingStrategy`) are expected to //! consult directly, so it and this module's `RollupStrategy` can never @@ -115,7 +115,7 @@ //! //! | Term | Meaning | Entry point | //! |---|---|---| -//! | Schema resolution | Derive input schemas and resolve column names to positions | `asap_types::pre_asap::SchemaResolver::resolve_schema`, `resolve_root` | +//! | Schema resolution | Derive input schemas and resolve column names to positions | `asap_frontend_common::schema_resolver::SchemaResolver::resolve_schema`, `asap_frontend_common::resolve::resolve_root` | //! | Realization | Enumerate ranked physical forms for one aggregate intent | `replacement::realizations_for_intent` | //! | Replacement | Construct each candidate summary sub-DAG | [`replacement::ASAPStrategies`] | //! | Search | Enumerate and compare alternatives across a workload | [`replacement::search_workload`] | diff --git a/crates/asap-aware-mapping/src/recurrence.rs b/crates/asap-aware-mapping/src/recurrence.rs index c9b677c86..f5348921a 100644 --- a/crates/asap-aware-mapping/src/recurrence.rs +++ b/crates/asap-aware-mapping/src/recurrence.rs @@ -1136,7 +1136,7 @@ mod tests { /// Like `scan()`, plus a "job" label column to group by — CSE's /// sharing legality gate requires a provable unique key /// (`Schema::has_unique_key`), and an *ungrouped* aggregate's empty - /// `by` reports none (see `asap_types::pre_asap::cse`'s own "Legality" + /// `by` reports none (see `asap_types::ir::cse`'s own "Legality" /// module docs); grouping by a label column gives `sum_agg()` below a /// real one, matching the pattern /// `replacement.rs`'s own CSE fixtures already use (`metric_scan`/`agg` @@ -1437,7 +1437,7 @@ mod tests { /// Issue #287 review (lower-priority item): a parent referencing the /// same shared child twice (`BinaryOp{lhs: X, rhs: X}`, the same shape - /// `pre_asap::cse`'s own within-one-query sharing collapses onto one + /// `ir::cse`'s own within-one-query sharing collapses onto one /// `Rc`) must credit that child with 2 contributions per repeating /// root, matching how `TargetSubDAGCandidates::consumer_count` already counts that /// exact structural occurrence twice — not 1, which a plain diff --git a/crates/asap-aware-mapping/src/replacement.rs b/crates/asap-aware-mapping/src/replacement.rs index b3c7fdd8d..9ea0cd07a 100644 --- a/crates/asap-aware-mapping/src/replacement.rs +++ b/crates/asap-aware-mapping/src/replacement.rs @@ -77,7 +77,7 @@ //! binds (single intent, no `HAVING`), every entry becomes its own bound //! candidate. //! - [`SharedSubDAGStrategy`] wraps -//! `asap_types::pre_asap::cse::share_common_sub_dags`'s sharing decision. +//! `asap_types::ir::cse::share_common_sub_dags`'s sharing decision. //! Wherever a [`TargetSubDAG`] already has two or more consumers (i.e. //! `share_common_sub_dags` already collapsed two or more workload //! locations onto the same `Rc` — [`discover_targets`] below @@ -144,7 +144,7 @@ //! 1. **Per-target candidates, not flat plans.** [`TargetSubDAGCandidates`] //! stores the alternatives for one distinct [`TargetSubDAG`] (identified by //! its own `Rc` pointer identity — the same currency -//! [`asap_types::pre_asap::cse::share_common_sub_dags`] already +//! [`asap_types::ir::cse::share_common_sub_dags`] already //! established across the workload) holding every //! [`ReplacementSubDAG`] alternative discovered for it. [`CandidateLogicalASAPDAGs`] is //! a collection of these groups, keyed by `TargetSubDAG` — a candidate @@ -153,8 +153,8 @@ //! targets are just two different entries in two different groups, //! sharing every other node in the workload by construction (they *are* //! the same `Rc`s — nothing was copied to make a second "plan"). -//! 2. **Dedup by structural hash + `PartialEq`, reusing `pre_asap::cse`'s own -//! discipline.** [`asap_types::pre_asap::cse::structural_hash`] (made +//! 2. **Dedup by structural hash + `PartialEq`, reusing `ir::cse`'s own +//! discipline.** [`asap_types::ir::cse::structural_hash`] (made //! `pub` for exactly this reuse) is only ever a candidate-narrowing //! filter; [`TargetSubDAGCandidates::add_candidate`]'s actual duplicate check is //! `OperatorNode`'s derived `PartialEq` — the same "hash is a filter, @@ -175,7 +175,7 @@ //! line above stands for: every `TargetSubDAG` this pass discovers is one //! iteration of that loop. It walks every workload root's whole DAG (the //! same **relational-skeleton** operator-child scope -//! `asap_types::pre_asap::cse::share_common_sub_dags` itself uses — see +//! `asap_types::ir::cse::share_common_sub_dags` itself uses — see //! that module's "Algorithm" section), discovering one `TargetSubDAG` per //! distinct `Rc` and a *real* `consumer_count`: how many operator-child //! positions anywhere in the workload reference that exact `Rc`, not just diff --git a/crates/asap-aware-mapping/src/rollup.rs b/crates/asap-aware-mapping/src/rollup.rs index b1e1201b0..69a5bfb09 100644 --- a/crates/asap-aware-mapping/src/rollup.rs +++ b/crates/asap-aware-mapping/src/rollup.rs @@ -20,7 +20,7 @@ //! "Rolling up aggregations on a fine-grained group by to get a //! coarse-grained group by (like AHA)," alongside "CSE across aggregations, //! and group by key management" — this strategy is the *cross-aggregate* -//! sibling of `pre_asap::cse::share_common_sub_dags`'s *identical*-sub-DAG +//! sibling of `ir::cse::share_common_sub_dags`'s *identical*-sub-DAG //! sharing: CSE shares two structurally-*equal* aggregates onto one `Rc`; //! this strategy relates two structurally-*different* (differently grouped) //! aggregates over the same shared source. @@ -207,7 +207,7 @@ fn rollup_combinator(intent: &AggIntent, finer_measure_col: ColumnId) -> Option< /// 4. `finer_output_schema` (the finer aggregate's own *output* schema, not /// the shared child's) carries a provable unique key /// ([`Schema::has_unique_key`]) — **the exact legality gate -/// `pre_asap::cse::share_common_sub_dags` already applies to its own +/// `ir::cse::share_common_sub_dags` already applies to its own /// sharing decisions**, reused verbatim here rather than re-invented: /// `share_common_sub_dags`'s own doc ("Legality: gated by /// `Schema::unique_keys`") states a producer's output is only safely diff --git a/crates/asap-physical-operators/tests/blocking_resources.rs b/crates/asap-physical-operators/tests/blocking_resources.rs index 66702afdb..09cee7a4e 100644 --- a/crates/asap-physical-operators/tests/blocking_resources.rs +++ b/crates/asap-physical-operators/tests/blocking_resources.rs @@ -8,7 +8,7 @@ use asap_physical_operators::{ }; use futures::{executor::block_on, FutureExt, StreamExt}; use planner_types::ir::Predicate; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{Field, FieldDataType}, pre_asap::{DataType, JoinKind, ScalarValue}, @@ -62,7 +62,7 @@ fn cross_join() -> Operator { schema(1), schema(1), JoinKind::Cross, - &Predicate(QueryExpr::Literal(ScalarValue::Boolean(true))), + &Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))), schema(2), ) .unwrap() diff --git a/crates/asap-physical-operators/tests/physical_dag.rs b/crates/asap-physical-operators/tests/physical_dag.rs index 7ae992df0..e38bbd5ad 100644 --- a/crates/asap-physical-operators/tests/physical_dag.rs +++ b/crates/asap-physical-operators/tests/physical_dag.rs @@ -15,7 +15,7 @@ use planner_types::ir::physical_export::{ use planner_types::ir::ASAPOp; use planner_types::ir::BinaryOperator; use planner_types::ir::NonASAPOp; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{ExactKind, ExactParams, Field, FieldDataType}, pre_asap::DataType, @@ -889,11 +889,11 @@ fn planner_expressions_preserve_collection_and_nullable_types() { }, false, )]); - let access = QueryExpr::FunctionCall { + let access = ScalarExpr::FunctionCall { name: "asap_element_access".into(), args: vec![ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Utf8("count".into())), + ScalarExpr::Column(0), + ScalarExpr::Literal(ScalarValue::Utf8("count".into())), ], }; let project = Operator::project( @@ -926,11 +926,11 @@ fn planner_expressions_preserve_collection_and_nullable_types() { .unwrap(); let projected = project.schema(); dag.add(1, vec![0], project).unwrap(); - let predicate = QueryExpr::Compare { + let predicate = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Ge, - right: Box::new(QueryExpr::Literal(ScalarValue::Int64(1))), + right: Box::new(ScalarExpr::Literal(ScalarValue::Int64(1))), }; dag.add( 2, @@ -944,9 +944,9 @@ fn planner_expressions_preserve_collection_and_nullable_types() { .unwrap(); let rows = run(&dag, 2, query()); assert!(matches!(rows.as_slice(),[row] if matches!(row.as_slice(),[Value::Int64(7)]))); - let unknown = QueryExpr::FunctionCall { + let unknown = ScalarExpr::FunctionCall { name: "unregistered_function".into(), - args: vec![QueryExpr::Column(0)], + args: vec![ScalarExpr::Column(0)], }; assert!(CompiledExpression::compile(&unknown, &input_schema).is_err()); } @@ -958,11 +958,11 @@ fn native_relational_join_kinds_preserve_unmatched_rows() { use planner_types::pre_asap::{CompareOpKind, JoinKind}; let input = schema(&[("key", DataType::Int64, true)]); - let predicate = Predicate(QueryExpr::Compare { + let predicate = Predicate(ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Eq, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }); for (kind, count) in [ (JoinKind::Inner, 1), diff --git a/crates/asap-physical-operators/tests/physical_semantics.rs b/crates/asap-physical-operators/tests/physical_semantics.rs index b5ffbc8ae..20f02f76e 100644 --- a/crates/asap-physical-operators/tests/physical_semantics.rs +++ b/crates/asap-physical-operators/tests/physical_semantics.rs @@ -12,7 +12,7 @@ use futures::{executor::block_on, StreamExt}; use planner_types::ir::physical_export::{PhysicalASAPDAGNode, PhysicalASAPOperatorPayload}; use planner_types::ir::NonASAPOp; use planner_types::ir::Predicate; -use planner_types::ir::ScalarExpr as QueryExpr; +use planner_types::ir::ScalarExpr; use planner_types::{ post_asap::{Field, FieldDataType}, pre_asap::{CompareOpKind, DataType, JoinKind}, @@ -78,11 +78,11 @@ fn keys(rows: &[Vec]) -> Vec>> { .collect() } fn eq_predicate() -> Predicate { - Predicate(QueryExpr::Compare { + Predicate(ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Eq, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }) } fn join(left: Vec, right: Vec, kind: JoinKind, keyed: bool) -> Vec> { @@ -340,7 +340,7 @@ fn aggregate_empty_and_all_null_follow_asap_contract() { fn projection_rejects_expression_bound_to_another_schema() { let original = schema(&[("a", DataType::Int64, false), ("b", DataType::Int64, false)]); let current = schema(&[("a", DataType::Int64, false)]); - let expr = CompiledExpression::compile(&QueryExpr::Column(1), &original).unwrap(); + let expr = CompiledExpression::compile(&ScalarExpr::Column(1), &original).unwrap(); assert!(Operator::project(current, vec![("b".into(), Expression::planner(expr))]).is_err()); } @@ -415,11 +415,11 @@ fn planner_comparisons_handle_nan_without_execution_errors() { CompareOpKind::Gt, CompareOpKind::Ge, ] { - let expression = QueryExpr::Compare { + let expression = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: op.clone(), - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }; let compiled = CompiledExpression::compile(&expression, &input).unwrap(); for row in [ @@ -483,11 +483,11 @@ fn mixed_numeric_comparisons_preserve_large_integer_precision() { ("a", DataType::Int64, false), ("b", DataType::Float64, false), ]); - let expr = QueryExpr::Compare { + let expr = ScalarExpr::Compare { semantics: planner_types::ir::ExprSemantics::Sql, - left: Box::new(QueryExpr::Column(0)), + left: Box::new(ScalarExpr::Column(0)), op: CompareOpKind::Gt, - right: Box::new(QueryExpr::Column(1)), + right: Box::new(ScalarExpr::Column(1)), }; let compiled = CompiledExpression::compile(&expr, &input).unwrap(); for (a, b, expected) in [ @@ -508,11 +508,11 @@ fn boolean_truth_tables_agree_between_expression_paths() { for and in [true, false] { for a in [None, Some(false), Some(true)] { for b in [None, Some(false), Some(true)] { - let parts = vec![QueryExpr::Column(0), QueryExpr::Column(1)]; + let parts = vec![ScalarExpr::Column(0), ScalarExpr::Column(1)]; let planner = if and { - QueryExpr::BoolAnd(parts) + ScalarExpr::BoolAnd(parts) } else { - QueryExpr::BoolOr(parts) + ScalarExpr::BoolOr(parts) }; let native = if and { Expression::And( diff --git a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs index 2a0d7bc18..477d8fa36 100644 --- a/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs +++ b/crates/asap-physical-operators/tests/planspace_series_identity_heap.rs @@ -4,7 +4,7 @@ //! ranking, or workload Cartesian expansion. Placement variants are not listed. mod common; use common::compile_physical_asap_dag; -use planner_types::ir::OperatorNode as QueryExpr; +use planner_types::ir::OperatorNode; use asap_aware_mapping::{ accuracy::{AccuracyEvidenceProvider, DefaultAccuracyModel, PropagationStats}, @@ -28,7 +28,7 @@ use std::rc::Rc; struct Evidence; impl AccuracyEvidenceProvider for Evidence { - fn topk_max_distinct_items(&self, _: &QueryExpr) -> Option { + fn topk_max_distinct_items(&self, _: &OperatorNode) -> Option { Some(1000) } fn propagation_stats( @@ -67,7 +67,7 @@ impl ReplacementStrategy for LogicalOnly { } } -fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { +fn lower(query: &str, accuracy: &AccuracyTarget) -> Rc { let workload = PlanningWorkload { query_workload: QueryWorkload { language: QueryLanguage::PromQL, diff --git a/crates/devtools/tests/viewer_contract.rs b/crates/devtools/tests/viewer_contract.rs new file mode 100644 index 000000000..e11aea837 --- /dev/null +++ b/crates/devtools/tests/viewer_contract.rs @@ -0,0 +1,278 @@ +//! `tools/dag-viewer` ↔ `asap_types::dag_export` contract: the viewer's +//! `KIND_CATEGORY_JSON` must categorize exactly the `kind` strings +//! [`asap_types::dag_export::export`] can emit — `Operator::kind_name()` of +//! every `NonASAPOp` and `ASAPOp` variant — no more (a stale kind the IR no +//! longer has) and no less (an exported kind the viewer would render +//! uncategorized). + +use std::collections::{BTreeMap, BTreeSet}; +use std::rc::Rc; +use std::time::Duration; + +use asap_types::ir::{ + ASAPOp, BinaryOperator, NonASAPOp, Operator, OperatorNode, Predicate, ScalarExpr, TimeRangeKind, +}; +use asap_types::post_asap::maintained_population::{ + MaintainedPopulation, PopulationInput, PopulationStatistic, +}; +use asap_types::post_asap::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; +use asap_types::pre_asap::{ + ArithmeticOpKind, BinaryOpKind, ColumnRef, DataType, FieldDataType, GroupKeys, JoinKind, + Reduction, RelationalSetOpKind, SampleKind, ScalarValue, Schema, Source, TimeShift, + WindowFuncKind, +}; + +/// One instance of every `NonASAPOp` variant, children elided. The +/// exhaustive match fails to compile when a variant is added, until a sample +/// for it is added here too. +fn non_asap_samples() -> Vec> { + let pred = || Predicate(ScalarExpr::Literal(ScalarValue::Boolean(true))); + let samples = vec![ + NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::default(), + }, + NonASAPOp::Values { + rows: vec![], + schema: Schema::default(), + }, + NonASAPOp::Filter { + pred: pred(), + child: (), + }, + NonASAPOp::Project { + cols: vec![], + qualifier: None, + child: (), + }, + NonASAPOp::Aggregate { + reduction: Reduction::by(vec![]), + measures: vec![], + output_names: vec![], + filters: vec![], + having: None, + child: (), + }, + NonASAPOp::Join { + kind: JoinKind::Inner, + pred: pred(), + left: (), + right: (), + }, + NonASAPOp::SetOp { + kind: RelationalSetOpKind::Union, + all: true, + left: (), + right: (), + }, + NonASAPOp::Concat { + children: vec![], + discriminator_unique_key: None, + }, + NonASAPOp::Dedup { + cols: vec![], + child: (), + }, + NonASAPOp::Sort { + keys: vec![], + partition_by: GroupKeys::default(), + child: (), + }, + NonASAPOp::Limit { + n: None, + offset: 0, + partition_by: GroupKeys::default(), + child: (), + }, + NonASAPOp::BinaryOp { + operator: BinaryOperator { + checked_relative_division: false, + checked_finite_division: false, + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + vector_match: None, + }, + return_bool: false, + lhs: (), + rhs: (), + }, + NonASAPOp::SQLWindowFunc { + func: WindowFuncKind::RowNumber, + args: vec![], + partition_by: GroupKeys::default(), + order_by: vec![], + frame: None, + output_name: "w".into(), + child: (), + }, + NonASAPOp::TimeRange { + range: Duration::from_secs(60), + kind: TimeRangeKind::Range, + child: (), + }, + NonASAPOp::TimeShift { + shift: TimeShift::default(), + child: (), + }, + NonASAPOp::PromqlVectorFromScalar(ScalarExpr::Literal(ScalarValue::Float64(1.0))), + NonASAPOp::PromqlRelabel { + dst: "l".into(), + value: ScalarExpr::Literal(ScalarValue::Utf8("v".into())), + child: (), + }, + NonASAPOp::PromqlInfoEnrich { + selector: vec![], + child: (), + }, + NonASAPOp::PromqlSeriesSample { + by: GroupKeys::default(), + kind: SampleKind::LimitK(1), + child: (), + }, + NonASAPOp::PromqlSubquery { + range: Duration::from_secs(60), + resolution: None, + child: (), + }, + ]; + for sample in &samples { + match sample { + NonASAPOp::Scan { .. } + | NonASAPOp::Values { .. } + | NonASAPOp::Filter { .. } + | NonASAPOp::Project { .. } + | NonASAPOp::Aggregate { .. } + | NonASAPOp::Join { .. } + | NonASAPOp::SetOp { .. } + | NonASAPOp::Concat { .. } + | NonASAPOp::Dedup { .. } + | NonASAPOp::Sort { .. } + | NonASAPOp::Limit { .. } + | NonASAPOp::BinaryOp { .. } + | NonASAPOp::SQLWindowFunc { .. } + | NonASAPOp::TimeRange { .. } + | NonASAPOp::TimeShift { .. } + | NonASAPOp::PromqlVectorFromScalar(_) + | NonASAPOp::PromqlRelabel { .. } + | NonASAPOp::PromqlInfoEnrich { .. } + | NonASAPOp::PromqlSeriesSample { .. } + | NonASAPOp::PromqlSubquery { .. } => {} + } + } + samples +} + +/// One instance of every `ASAPOp` variant, children elided; see +/// [`non_asap_samples`]. +fn asap_samples() -> Vec> { + let family = FieldDataType::Plain(DataType::Float64); + let rows = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::default(), + })) + .unwrap(); + let samples = vec![ + ASAPOp::SummaryAgg { + child: (), + family: family.clone(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + filter: None, + }, + ASAPOp::SummaryEstimate { + summary_input: (), + query: SketchStatistic::FrequencyL2, + }, + ASAPOp::FinalizeExactAccumulator { child: () }, + ASAPOp::MaintainPopulation { + child: (), + population: MaintainedPopulation { + input: PopulationInput::Rows { + input: Rc::clone(&rows), + value_column: 0, + grouping: GroupKeys::default(), + }, + max_k: 1, + quantiles: false, + }, + }, + ASAPOp::EvaluatePopulation { + child: (), + evaluation: PopulationStatistic::Sum, + }, + ASAPOp::SummaryMerge { children: vec![] }, + ASAPOp::SummarySubtract { + left: (), + right: (), + }, + ASAPOp::SummaryDelete { + summary_input: (), + key: 0, + }, + ASAPOp::SummaryJoin { + outer: (), + inner: (), + key: 0, + family, + }, + ASAPOp::Extension { + child: (), + name: "x".into(), + }, + ]; + for sample in &samples { + match sample { + ASAPOp::SummaryAgg { .. } + | ASAPOp::SummaryEstimate { .. } + | ASAPOp::FinalizeExactAccumulator { .. } + | ASAPOp::MaintainPopulation { .. } + | ASAPOp::EvaluatePopulation { .. } + | ASAPOp::SummaryMerge { .. } + | ASAPOp::SummarySubtract { .. } + | ASAPOp::SummaryDelete { .. } + | ASAPOp::SummaryJoin { .. } + | ASAPOp::Extension { .. } => {} + } + } + samples +} + +/// The viewer's `kind -> category` table, parsed out of the JS source the +/// same way the viewer itself does (`JSON.parse(KIND_CATEGORY_JSON)`). +fn viewer_kind_categories() -> BTreeMap { + const START: &str = "const KIND_CATEGORY_JSON = `"; + let source = include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../../tools/dag-viewer/node-style.js" + )); + let json = source + .split_once(START) + .expect("node-style.js must declare KIND_CATEGORY_JSON") + .1 + .split_once("`;") + .expect("KIND_CATEGORY_JSON must be a template literal") + .0; + serde_json::from_str(json).expect("KIND_CATEGORY_JSON must be valid JSON") +} + +#[test] +fn viewer_categorizes_exactly_the_exported_node_kinds() { + let non_asap = non_asap_samples(); + let asap = asap_samples(); + let expected: BTreeSet<&str> = non_asap + .iter() + .map(NonASAPOp::kind_name) + .chain(asap.iter().map(ASAPOp::kind_name)) + .collect(); + assert_eq!( + expected.len(), + non_asap.len() + asap.len(), + "exported kind names must be unique" + ); + let categories = viewer_kind_categories(); + let actual: BTreeSet<&str> = categories.keys().map(String::as_str).collect(); + + assert_eq!(actual, expected); +} diff --git a/crates/frontend-sql/src/sql/clickhouse_ast.rs b/crates/frontend-sql/src/sql/clickhouse_ast.rs index f9246d22a..78e1694c4 100644 --- a/crates/frontend-sql/src/sql/clickhouse_ast.rs +++ b/crates/frontend-sql/src/sql/clickhouse_ast.rs @@ -6,7 +6,7 @@ use datafusion::sql::sqlparser::ast::{ }; use std::ops::ControlFlow; -use super::collection_planning::MAP_PLANNING_NAME; +use super::collection_udfs::MAP_UDF_NAME; pub(super) fn normalize(statement: &mut Statement) { struct PreserveNames; @@ -66,7 +66,7 @@ pub(super) fn normalize(statement: &mut Statement) { return ControlFlow::Continue(()); }; if unquoted_name(&function.name).is_some_and(|name| name.eq_ignore_ascii_case("map")) { - function.name = ObjectName::from(vec![Ident::new(MAP_PLANNING_NAME)]); + function.name = ObjectName::from(vec![Ident::new(MAP_UDF_NAME)]); return ControlFlow::Continue(()); } if !unquoted_name(&function.name).is_some_and(|name| name.eq_ignore_ascii_case("modulo")) diff --git a/crates/frontend-sql/src/sql/collection_planning.rs b/crates/frontend-sql/src/sql/collection_udfs.rs similarity index 71% rename from crates/frontend-sql/src/sql/collection_planning.rs rename to crates/frontend-sql/src/sql/collection_udfs.rs index 62bbbbb6d..63c862b2b 100644 --- a/crates/frontend-sql/src/sql/collection_planning.rs +++ b/crates/frontend-sql/src/sql/collection_udfs.rs @@ -1,5 +1,9 @@ -//! DataFusion planning adapters. Types come from the canonical signature rules; -//! physical evaluation deliberately remains the query engine's responsibility. +//! Planning-only DataFusion UDFs for ClickHouse's collection functions: +//! `map(...)`, `mapConcat`, `arrayElement` and `tupleElement` over Map, List +//! and Struct values. They let DataFusion's SQL planner resolve these calls, +//! with return types from the canonical signature rules; lowering turns them +//! into IR scalar functions, and DataFusion never executes them (evaluation +//! is the query engine's responsibility). use super::types::{arrow_to_dtype, dtype_to_arrow, scalar_value_to_asap}; use asap_types::ir::scalar::{element_access_type, struct_field_type}; use asap_types::ir::ScalarExpr; @@ -18,10 +22,10 @@ use std::sync::Arc; /// The name ClickHouse's `map(...)` is planned under. DataFusion's SQL planner /// reserves `map` for its own constructor (and rejects `map()`), so /// `clickhouse_ast::normalize` renames the call and lowering restores `map`. -pub(super) const MAP_PLANNING_NAME: &str = "asap_map_construct"; +pub(super) const MAP_UDF_NAME: &str = "asap_map_construct"; #[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum PlanningFunction { +enum CollectionFunction { Map(MapScalarFunction), Element, StructField, @@ -30,28 +34,28 @@ enum PlanningFunction { pub(super) fn register(context: &SessionContext) { for (name, function) in [ ( - MAP_PLANNING_NAME, - PlanningFunction::Map(MapScalarFunction::Construct), + MAP_UDF_NAME, + CollectionFunction::Map(MapScalarFunction::Construct), ), ( "mapconcat", - PlanningFunction::Map(MapScalarFunction::Concat), + CollectionFunction::Map(MapScalarFunction::Concat), ), - ("arrayelement", PlanningFunction::Element), - ("tupleelement", PlanningFunction::StructField), + ("arrayelement", CollectionFunction::Element), + ("tupleelement", CollectionFunction::StructField), ] { - context.register_udf(ScalarUDF::from(CollectionPlanningFunction { + context.register_udf(ScalarUDF::from(CollectionUdf { name, function, signature: match function { - PlanningFunction::Map(MapScalarFunction::Construct) => Signature::one_of( + CollectionFunction::Map(MapScalarFunction::Construct) => Signature::one_of( vec![TypeSignature::Exact(vec![]), TypeSignature::VariadicAny], Volatility::Immutable, ), - PlanningFunction::Map(MapScalarFunction::Access) - | PlanningFunction::Element - | PlanningFunction::StructField => Signature::any(2, Volatility::Immutable), - PlanningFunction::Map(MapScalarFunction::Concat) => { + CollectionFunction::Map(MapScalarFunction::Access) + | CollectionFunction::Element + | CollectionFunction::StructField => Signature::any(2, Volatility::Immutable), + CollectionFunction::Map(MapScalarFunction::Concat) => { Signature::variadic_any(Volatility::Immutable) } }, @@ -59,20 +63,20 @@ pub(super) fn register(context: &SessionContext) { } } #[derive(Debug, PartialEq, Eq)] -struct CollectionPlanningFunction { +struct CollectionUdf { name: &'static str, - function: PlanningFunction, + function: CollectionFunction, signature: Signature, } // `function` is determined by `name` at registration, so hashing the name and // signature agrees with the derived `Eq`. -impl Hash for CollectionPlanningFunction { +impl Hash for CollectionUdf { fn hash(&self, state: &mut H) { self.name.hash(state); self.signature.hash(state); } } -impl CollectionPlanningFunction { +impl CollectionUdf { fn output( &self, args: &[DataType], @@ -90,7 +94,7 @@ impl CollectionPlanningFunction { .collect::>>()?; let (dtype, nullable) = if matches!( self.function, - PlanningFunction::Element | PlanningFunction::StructField + CollectionFunction::Element | CollectionFunction::StructField ) { // DataFusion asks for argument-dependent types before canonical // expression binding. Reuse the shared resolver over typed argument @@ -116,11 +120,11 @@ impl CollectionPlanningFunction { }) .collect::>>()?; match self.function { - PlanningFunction::Element => element_access_type(&args, &schema), - PlanningFunction::StructField => struct_field_type(&args, &schema), - PlanningFunction::Map(_) => unreachable!(), + CollectionFunction::Element => element_access_type(&args, &schema), + CollectionFunction::StructField => struct_field_type(&args, &schema), + CollectionFunction::Map(_) => unreachable!(), } - } else if let PlanningFunction::Map(function) = self.function { + } else if let CollectionFunction::Map(function) = self.function { function.output_type(&inputs) } else { unreachable!() @@ -129,7 +133,7 @@ impl CollectionPlanningFunction { Ok((dtype_to_arrow(&dtype), nullable)) } } -impl ScalarUDFImpl for CollectionPlanningFunction { +impl ScalarUDFImpl for CollectionUdf { fn name(&self) -> &str { self.name } @@ -162,7 +166,10 @@ impl ScalarUDFImpl for CollectionPlanningFunction { Ok(Arc::new(ArrowField::new(self.name, dtype, nullable))) } fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { - Err(DataFusionError::NotImplemented("collection planning adapter cannot execute; use a capable query engine or external exact sub-DAG".into())) + Err(DataFusionError::NotImplemented( + "collection UDF cannot execute; use a capable query engine or external exact sub-DAG" + .into(), + )) } } @@ -170,10 +177,10 @@ impl ScalarUDFImpl for CollectionPlanningFunction { mod tests { use super::*; #[test] - fn planning_adapter_explicitly_refuses_physical_execution() { - let adapter = CollectionPlanningFunction { - name: MAP_PLANNING_NAME, - function: PlanningFunction::Map(MapScalarFunction::Construct), + fn collection_udf_explicitly_refuses_physical_execution() { + let udf = CollectionUdf { + name: MAP_UDF_NAME, + function: CollectionFunction::Map(MapScalarFunction::Construct), signature: Signature::any(0, Volatility::Immutable), }; let args = ScalarFunctionArgs { @@ -184,10 +191,10 @@ mod tests { config_options: Default::default(), }; assert!(matches!( - adapter.invoke_with_args(args), + udf.invoke_with_args(args), Err(DataFusionError::NotImplemented(_)) )); - let result = adapter.return_type(&[]).unwrap(); + let result = udf.return_type(&[]).unwrap(); let (expected, _) = MapScalarFunction::Construct.output_type(&[]).unwrap(); assert_eq!(result, dtype_to_arrow(&expected)); } diff --git a/crates/frontend-sql/src/sql/expr.rs b/crates/frontend-sql/src/sql/expr.rs index 1ff5dcc51..92360bcec 100644 --- a/crates/frontend-sql/src/sql/expr.rs +++ b/crates/frontend-sql/src/sql/expr.rs @@ -226,7 +226,7 @@ impl SqlLowerer<'_> { "asap_element_access".into() } else if sf.func.name().eq_ignore_ascii_case("tupleelement") { "asap_struct_field".into() - } else if sf.func.name() == super::collection_planning::MAP_PLANNING_NAME { + } else if sf.func.name() == super::collection_udfs::MAP_UDF_NAME { "map".into() } else { sf.func.name().to_string() diff --git a/crates/frontend-sql/src/sql/mod.rs b/crates/frontend-sql/src/sql/mod.rs index ad0f1c41e..3f873d071 100644 --- a/crates/frontend-sql/src/sql/mod.rs +++ b/crates/frontend-sql/src/sql/mod.rs @@ -75,7 +75,7 @@ use asap_types::workload::SqlDialect; use crate::error::SqlError as LoweringError; mod clickhouse_ast; -mod collection_planning; +mod collection_udfs; mod expr; mod types; @@ -250,7 +250,7 @@ impl<'a> SqlLowerer<'a> { let config = SessionConfig::new().set_str("datafusion.sql_parser.dialect", dialect_name); let ctx = SessionContext::new_with_config(config); if matches!(self.dialect, SqlDialect::ClickhouseSQL) { - collection_planning::register(&ctx); + collection_udfs::register(&ctx); } // A catalog key like "bgp.bgp_updates" schema-qualifies the table // (e.g. a ClickHouse database name). DataFusion requires the parent diff --git a/crates/types/Cargo.toml b/crates/types/Cargo.toml index 8572854a9..0e10ccdf4 100644 --- a/crates/types/Cargo.toml +++ b/crates/types/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" # execution logic — removed, no real implementor existed; see issue #190). # No internal deps. [dependencies] -# "rc" — QueryExpr's child fields are Rc> (issue #212, #222: +# "rc" — OperatorNode child fields are Rc (issue #212, #222: # shared sub-expressions), and Rc's Serialize/Deserialize impls live behind # this feature flag. dag_export.rs / DAGNode already flatten the DAG to a # node+edge list for JSON export, so this does not change wire format — a diff --git a/crates/types/src/ir/aggregate_schema.rs b/crates/types/src/ir/aggregate_schema.rs index 10b33d2ae..4c868f554 100644 --- a/crates/types/src/ir/aggregate_schema.rs +++ b/crates/types/src/ir/aggregate_schema.rs @@ -55,7 +55,7 @@ fn per_series_reduction_schema( /// The output schema of an `Aggregate { reduction, measures }` over `in_schema` — /// the **single** canonical derivation shared by /// [`NonASAPOp::output_schema`](crate::ir::NonASAPOp::output_schema)'s -/// `Aggregate` arm and the HAVING-resolution path (`column_resolution::output_schema_for_aggregate`), +/// `Aggregate` arm and the HAVING-resolution path (`asap_frontend_common::resolve`), /// so the two can never drift (issue #41). /// /// `Reduction::PerEntity` selects the label-preserving diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 688204a87..c7ba61f18 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -1,5 +1,4 @@ -//! Unified operator and scalar representation from #511. -//! Legacy consumers remain on their existing representation until the planner cutover. +//! The operator IR from #511: one operator DAG for every planning stage. pub mod aggregate_schema; pub mod asap; pub mod error; diff --git a/crates/types/src/ir/operator_properties.rs b/crates/types/src/ir/operator_properties.rs index 4737e6278..7a42aad90 100644 --- a/crates/types/src/ir/operator_properties.rs +++ b/crates/types/src/ir/operator_properties.rs @@ -1,8 +1,555 @@ -//! Operator parameters shared with the existing dag during migration. -//! Definitions move here when legacy dag consumers are removed. -pub use crate::pre_asap::query_expr::{ - AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, - InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, - Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, - WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; +//! Supporting parameter types used inside operator payloads. +//! +//! For example, `Aggregate.by` uses [`GroupKeys`], a join chooses [`JoinKind`], +//! and a SQL window carries [`WindowFrame`]. These types describe what an +//! operator does. Derived node metadata (schema, guarantee, timing) lives on +//! [`super::OperatorNode`], not in this module. +use crate::pre_asap::{ArithmeticOpKind, ColumnId, ColumnRef, CompareOpKind, ScalarValue}; +use serde::{Deserialize, Serialize}; +/// The column-reference type an operator parameter is generic over: +/// positional [`ColumnId`] once bound, name-based [`ColumnRef`] before. +pub trait ColState: + Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de> +{ +} + +impl ColState for ColumnId {} + +impl ColState for ColumnRef {} + +// ── Leaf / supporting types ─────────────────────────────────────────────────── + +/// Positional grouping keys, shared by every "operate per group" operator: +/// `Aggregate.by` (reduce per group), `Sort.partition_by` (rank per group — +/// including generic `topk`/`bottomk`), and `SQLWindowFunc.partition_by` (window +/// per group). One spelling so grouping has a single home to evolve. Empty +/// (and `by`) = no grouping (a global operation). +/// +/// Heavy-hitter `AggIntent::TopK` carries its grouping here too, via the +/// enclosing `Aggregate.by` (issue #13) — so reduce, rank, and window groupings +/// all share this one type. +/// +/// ## `by` vs `without` (issue #39) +/// +/// The stored [`keys`](Self::keys) are **kept** labels for `by(...)` and +/// **excluded** labels for `without(...)`. PromQL's `without(labels)` groups by +/// every label *except* those listed; the complement can't be enumerated at +/// lowering time under an open (usage-derived) schema, so it is deferred to the +/// runtime — the excluded positions are stored, the kept set stays open. Only +/// `Aggregate` ever produces the `without` form; `Sort` / `SQLWindowFunc` / +/// `PromqlSeriesSample` groupings are always `by`. +/// +/// Serialises as a bare array for the (overwhelmingly common) `by` case — +/// wire-compatible with the `Vec` this field held before — and as +/// `{"without": [...]}` for the exclusion case. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct GroupKeys { + keys: Vec, + without: bool, +} + +// Not `#[derive(Default)]`: derive would add a `C: Default` bound, but an +// empty key set needs nothing from `C` — `ColumnRef` has no meaningful +// default anyway. +impl Default for GroupKeys { + fn default() -> Self { + Self { + keys: Vec::new(), + without: false, + } + } +} + +impl GroupKeys { + /// An empty key set — a global (ungrouped) operation. + pub fn none() -> Self { + Self::default() + } + /// `by(keys)` — group by exactly these columns. + pub fn by(keys: Vec) -> Self { + Self { + keys, + without: false, + } + } + /// `without(keys)` — group by every label *except* these (issue #39). The + /// kept set is runtime-resolved; only the excluded positions are stored. + pub fn without(keys: Vec) -> Self { + Self { + keys, + without: true, + } + } + /// Whether this is a `without(...)` exclusion grouping. + pub fn is_without(&self) -> bool { + self.without + } + /// The named keys — kept labels for `by`, excluded labels for `without`. + pub fn keys(&self) -> &[C] { + &self.keys + } +} + +impl std::ops::Deref for GroupKeys { + type Target = [C]; + fn deref(&self) -> &Self::Target { + &self.keys + } +} + +impl From> for GroupKeys { + fn from(keys: Vec) -> Self { + Self::by(keys) + } +} + +impl FromIterator for GroupKeys { + fn from_iter>(iter: I) -> Self { + Self::by(iter.into_iter().collect()) + } +} + +impl<'a, C> IntoIterator for &'a GroupKeys { + type Item = &'a C; + type IntoIter = std::slice::Iter<'a, C>; + fn into_iter(self) -> Self::IntoIter { + self.keys.iter() + } +} + +/// Compare directly against a `Vec` so call sites and tests can keep +/// writing `keys == vec![..]` / `assert_eq!(keys, &vec![..])`. A `without` +/// grouping never equals a bare `by` list. +impl PartialEq> for GroupKeys { + fn eq(&self, other: &Vec) -> bool { + !self.without && &self.keys == other + } +} + +/// (De)serialise as a bare array for `by`, or `{"without": [...]}` for the +/// exclusion form — keeping the `by` wire format identical to the old newtype. +/// Borrowed for `Serialize` (no `C: Clone` needed to write one out), owned for +/// `Deserialize` (there's nothing to borrow from). +#[derive(Serialize)] +#[serde(untagged)] +enum GroupKeysReprRef<'a, C> { + By(&'a [C]), + Without { without: &'a [C] }, +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum GroupKeysRepr { + By(Vec), + Without { without: Vec }, +} + +impl Serialize for GroupKeys { + fn serialize(&self, serializer: S) -> Result { + if self.without { + GroupKeysReprRef::Without { + without: self.keys.as_slice(), + } + .serialize(serializer) + } else { + GroupKeysReprRef::By(self.keys.as_slice()).serialize(serializer) + } + } +} + +impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { + fn deserialize>(deserializer: D) -> Result { + Ok(match GroupKeysRepr::deserialize(deserializer)? { + GroupKeysRepr::By(keys) => Self::by(keys), + GroupKeysRepr::Without { without } => Self::without(without), + }) + } +} + +/// Which data model a `Source` / `AggIntent` operates over. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum DataModel { + TimeSeries, + Tabular, + Any, +} + +/// The leaf data source of a `Scan`. The schema itself rides on the +/// `Scan.schema` field (SchemaResolver-built); `Source` carries only the leaf's +/// identity. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum Source { + /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. + TimeSeries { metric: String }, + /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. + Table { table_ref: String }, +} + +/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's +/// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison +/// operator has exactly one representation (and one `Display`) across the IR. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum BinaryOpKind { + /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `ScalarExpr::Arithmetic`). + Arithmetic(ArithmeticOpKind), + /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared + /// with `ScalarExpr::Compare`). PromQL keeps the matched series whose + /// comparison holds. + Compare(CompareOpKind), + /// PromQL comparison with the `bool` modifier: every matched series + /// yields 1 or 0 and loses its metric name. A separate variant, not a + /// flag, because only comparisons take `bool`. + CompareBool(CompareOpKind), + /// PromQL vector-set operation. + Set(PromQLVectorSetOpKind), +} + +impl std::fmt::Display for BinaryOpKind { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + BinaryOpKind::Arithmetic(op) => write!(f, "{op}"), + BinaryOpKind::Compare(op) => write!(f, "{op}"), + BinaryOpKind::CompareBool(op) => write!(f, "{op} bool"), + BinaryOpKind::Set(PromQLVectorSetOpKind::And) => f.write_str("AND"), + BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => f.write_str("OR"), + BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) => f.write_str("unless"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum JoinKind { + Inner, + Left, + Right, + Full, + Cross, + /// Left semi-join — each left row that has **at least one** match, once. + /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)` (issue #111). + /// + /// Output schema is the **left's alone**; the right side is a filter, not a + /// source of columns. The join predicate still resolves against the + /// concatenated `left ++ right` schema — its scope is deliberately wider + /// than the node's output. + Semi, + /// Left anti-join — each left row with **no** match. `WHERE NOT EXISTS (…)`. + /// Same schema rule as [`JoinKind::Semi`]. + /// + /// Note this is *not* `NOT IN (SELECT …)`: under SQL's three-valued logic a + /// NULL on the right makes `NOT IN` yield no rows at all, where an anti-join + /// yields every left row. The SQL front end rejects `NOT IN (subquery)` + /// rather than lower it here. + Anti, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum RelationalSetOpKind { + Union, + Intersect, + Except, +} + +/// PromQL vector-set operator used by [`BinaryOpKind::Set`]. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum PromQLVectorSetOpKind { + And, + Or, + Unless, +} + +/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming +/// time `Window`: this is an analytic frame over already-materialised rows. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFuncKind { + RowNumber, + Rank, + DenseRank, + Lag, + Lead, + /// ClickHouse `lagInFrame`/`leadInFrame`: unlike [`Lag`](Self::Lag)/[`Lead`](Self::Lead), + /// these respect the window frame bounds (NULL/default past the frame edge) + /// rather than reaching arbitrarily far back/forward. Kept as distinct + /// variants so the frame clause is never silently discarded by conflating + /// them with `Lag`/`Lead` (#267). `WindowFuncKind` still has no frame + /// representation, so today these lower and behave exactly like + /// `Lag`/`Lead` — the tag is correct, the frame-respecting behavior isn't + /// implemented yet. See #231 for modeling window frames properly. + LagInFrame, + LeadInFrame, + FirstValue, + LastValue, + /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. + NthValue(Option), + Sum, + Avg, + Count, + Min, + Max, +} + +/// A window's frame-spec (`ROWS`/`RANGE BETWEEN … AND …`) — which rows around +/// the current one an analytic window function reads. `GROUPS` is rejected at +/// lowering time (issue #268): every SQL corpus in this repo uses only `ROWS`, +/// and nothing downstream interprets frame semantics yet, so it isn't worth +/// modelling untested. +/// +/// Meaningless (but harmless) on the rank-only and navigation functions +/// (`ROW_NUMBER`/`RANK`/`DENSE_RANK`/`LAG`/`LEAD`), which ignore the frame per +/// SQL semantics — DataFusion still attaches one, stored here verbatim. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct WindowFrame { + pub units: WindowFrameUnits, + pub start_bound: WindowFrameBound, + pub end_bound: WindowFrameBound, +} + +/// A finite window-frame displacement. Intervals are normalized to Arrow's +/// month/day/nanosecond representation so SQL `RANGE INTERVAL ...` bounds +/// survive lowering without leaking DataFusion types into the canonical IR. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameOffset { + Scalar(ScalarValue), + Interval { + months: i32, + days: i32, + nanoseconds: i64, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameUnits { + /// Boundaries count physical rows: `ROWS BETWEEN 2 PRECEDING AND CURRENT ROW`. + Rows, + /// Boundaries count by value-distance on the (single) `ORDER BY` column: + /// `RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW`. + Range, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowFrameBound { + /// `UNBOUNDED PRECEDING` is + /// `Preceding(WindowFrameOffset::Scalar(ScalarValue::Null))`. + Preceding(WindowFrameOffset), + CurrentRow, + /// `UNBOUNDED FOLLOWING` is + /// `Following(WindowFrameOffset::Scalar(ScalarValue::Null))`. + Following(WindowFrameOffset), +} + +/// A symbolic label matcher on the **info metric** side of an +/// [`crate::ir::NonASAPOp::PromqlInfoEnrich`] (issue #84). Unlike a `Scan` predicate it is not +/// resolved positionally — it references the info metric's labels (`__name__` +/// picks the metric, the rest constrain data labels), which aren't in the input +/// vector's schema; the post-ASAP realization pass applies it against the info metric. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct InfoMatcher { + pub label: String, + /// One of `Eq` / `Ne` / `Regex` / `NotRegex` (PromQL `=`/`!=`/`=~`/`!~`). + pub op: CompareOpKind, + pub value: String, +} + +/// Series-sampling selection mode (PromQL `limitk` / `limit_ratio`, issue #86). +/// A [`crate::ir::NonASAPOp::PromqlSeriesSample`] keeps a *subset of whole series*, unchanged — it does +/// not rank or reduce, so it is distinct from `TopK` and from `Sort → Limit`. +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SampleKind { + /// `limitk(k, v)` — up to `k` series per group. Which series survive is + /// deterministic across evaluations but otherwise unspecified (no ordering). + LimitK(usize), + /// `limit_ratio(r, v)` — a deterministic `r`-fraction of series per group. + /// `r ∈ [-1, 1]`; a negative `r` selects the complementary fraction. + LimitRatio(f64), +} + +/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorMatch { + pub kind: VectorMatchKind, + pub labels: Vec, + pub grouping: Option, +} + +/// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead +/// of the query evaluation time (issue #40). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum AtModifier { + /// `@ start()` — the query range's start instant. + Start, + /// `@ end()` — the query range's end instant. + End, + /// `@ ` — an absolute instant, milliseconds since the Unix epoch (may be + /// negative). PromQL writes the timestamp in seconds; the front end scales it. + Timestamp(i64), +} + +/// PromQL per-selector **time-shift** modifiers — `offset` and `@` (issue #40). +/// Neither changes a selector's *schema*; both move *when* it is evaluated, so +/// the shift is a pass-through wrapper ([`crate::ir::NonASAPOp::TimeShift`]) over the +/// selector rather than a new leaf shape. The runtime resolves the anchor and +/// applies the offset. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] +pub struct TimeShift { + /// `offset ` as signed milliseconds — a positive value shifts the + /// lookback *back* in time (`offset 5m`), a negative value shifts it + /// *forward* (`offset -5m`). `0` = no offset. + pub offset_ms: i64, + /// `@` anchor; `None` = evaluate at the query time. + pub at: Option, +} + +impl TimeShift { + /// Whether this shift is the identity (no `offset`, no `@`) — the state of + /// every selector that carries neither modifier. + pub fn is_identity(&self) -> bool { + self.offset_ms == 0 && self.at.is_none() + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum VectorMatchKind { + On, + Ignoring, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct VectorGrouping { + pub side: GroupSide, + pub labels: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub enum GroupSide { + Left, + Right, +} + +// ── Intent algebra IR ──────────────────────────────────────────────────────── + +/// What kind of computation an `Aggregate` node performs — orthogonal to +/// *which* columns it groups by (that's still [`GroupKeys`], inside +/// `Reduce`). Explicit, decided once by whichever pass constructs the node +/// (structural, at front-end lowering time), rather than inferred downstream from +/// whether a grouping-key list happens to be empty or from a neighboring +/// node's shape. See design proposal #165. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub enum Reduction { + /// Collapses input rows via `by` — `by`/`without` semantics are exactly + /// [`GroupKeys`]'s. May still collapse every row into one (an empty, + /// non-`without` `by`) — that's a genuine reduction with zero grouping + /// columns, not "no grouping concept." + Reduce(GroupKeys), + /// No grouping concept at all: preserves one output row per input + /// entity (e.g. a per-series windowed computation with no `by(...)` + /// clause to begin with, because there's no aggregation operator here + /// for such a clause to attach to). Never merges across entities, and + /// never collapses an entity's own row structure (e.g. a time axis) — + /// unlike `Reduce(GroupKeys::without(vec![]))` ("group by every + /// label"), which is still a genuine reduction and does collapse it. + PerEntity, +} + +impl Reduction { + /// Shorthand for the common case — group by these (possibly empty) + /// keys, kept rather than excluded. + pub fn by(keys: Vec) -> Self { + Self::Reduce(GroupKeys::by(keys)) + } + + /// The grouping keys, if this is a genuine reduction — `None` for + /// `PerEntity`, which has no grouping-keys concept to report. + pub fn group_keys(&self) -> Option<&GroupKeys> { + match self { + Self::Reduce(by) => Some(by), + Self::PerEntity => None, + } + } + + /// The grouping keys, panicking if this is `PerEntity` — for call sites + /// (tests, mostly) that already know, from the shape they built or are + /// asserting on, that this must be a genuine reduction. Prefer + /// [`group_keys`](Self::group_keys) wherever the caller can't assume that. + pub fn expect_reduce(&self) -> &GroupKeys { + match self { + Self::Reduce(by) => by, + Self::PerEntity => panic!("expected Reduction::Reduce, got PerEntity"), + } + } +} + +/// A caller-proven compound unique key for a [`crate::ir::NonASAPOp::Concat`] (issue +/// #228) — built only via [`ConcatDiscriminatorKey::new`], never by naming `discriminator` directly +/// in a struct literal (both fields are private): from *other Rust code*, +/// the only way to end up with one of these is to hand over a specific +/// column as the discriminator, by name, at the call site. +/// +/// Caveat: this is a Rust-API-level guarantee, not a data-level one. The +/// derived `Deserialize` impl below builds a `ConcatDiscriminatorKey` +/// directly from field values, bypassing `new()`. Deserialization is therefore +/// equivalent to a caller supplying the assertion directly; it does not prove +/// either fact below. An external boundary accepting IR data must +/// reject this field or validate both obligations before treating it as +/// uniqueness evidence. +/// +/// # Soundness +/// +/// `Concat`'s default (see its own doc) is to drop `unique_keys` +/// unconditionally, because a key unique **within** one branch is not unique +/// **across** the concatenation unless the branches' value sets for that key +/// are provably disjoint — nothing about matching schemas or matching +/// per-branch keys establishes that on its own. Two different branches can +/// trivially emit the same `inner_key` value (e.g. two PromQL +/// `histogram_quantiles` branches keyed on `(host, le)` can both produce a +/// `(host, le)` pair for different φ). +/// +/// Prepending `discriminator` restores a compound key only when two facts +/// hold: `inner_key` uniquely identifies rows **within every branch**, and +/// `discriminator`'s value is **guaranteed to differ between branches** — a +/// literal the producer just tagged the branch with (PromQL φ riding along via +/// [`crate::ir::NonASAPOp::PromqlRelabel`], a Postgres-style synthetic `GROUPING()` id +/// for `ROLLUP`/`CUBE`, …), never something inferred structurally from the +/// branches' own data — then `discriminator` alone partitions rows into +/// disjoint sets independent of what the branches actually contain, so +/// `(discriminator, inner_key)` is sound even when otherwise-identical +/// `inner_key` values occur in different branches. Neither fact is verified +/// here; both are part of the caller-proven claim. +/// +/// This is a **caller-proven claim, not something `Concat` can verify**: +/// nothing stops a caller from asserting a discriminator that in fact +/// repeats across branches, in which case the resulting `unique_keys` claim +/// is simply wrong — `output_schema` trusts it without checking. The +/// obligation is on the constructor call site, exactly as it is on +/// [`crate::ir::NonASAPOp::Dedup`]'s `cols` or any other unverified `unique_keys` +/// producer in this module. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] +pub struct ConcatDiscriminatorKey { + discriminator: C, + inner_key: Vec, +} + +impl ConcatDiscriminatorKey { + /// The only constructor — `discriminator` must be named explicitly by + /// the caller. See the type's doc for the soundness obligation this + /// puts on that caller. + pub fn new(discriminator: C, inner_key: Vec) -> Self { + Self { + discriminator, + inner_key, + } + } + + pub fn discriminator(&self) -> &C { + &self.discriminator + } + + pub fn inner_key(&self) -> &[C] { + &self.inner_key + } +} diff --git a/crates/types/src/ir/schema_support.rs b/crates/types/src/ir/schema_support.rs index c710c8654..95c42070d 100644 --- a/crates/types/src/ir/schema_support.rs +++ b/crates/types/src/ir/schema_support.rs @@ -1,5 +1,14 @@ //! Series-identity realization for the unified dag. use crate::pre_asap::schema::*; +/// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before +/// candidate search. `closed` describes physical columns here: the final +/// column contains every dynamic source label. It does not assert that the +/// query's projected labels are the full label set. +/// +/// This realization supports explicit `by` grouping and per-series computation. +/// Operators that rewrite or implicitly match dynamic label sets require their +/// own realization; they must not accidentally treat the opaque identity as a +/// user label or silently discard it. pub fn with_promql_series_identity( root: &std::rc::Rc, ) -> Result, String> { diff --git a/crates/types/src/ir/timing.rs b/crates/types/src/ir/timing.rs index 29f1f7d8f..e4ff6a538 100644 --- a/crates/types/src/ir/timing.rs +++ b/crates/types/src/ir/timing.rs @@ -511,15 +511,19 @@ fn validate_non_asap( return Err(ExecutionDataStateError::InvalidCheckedDivision); } if timing == ExecutionTiming::IngestionTime { + // The opaque series identity is a non-null Utf8 key; every + // other column is a non-null Float64 value or timestamp. let plain_float_or_ts = |schema: &Schema| { schema.fields.iter().all(|field| { !field.nullable - && (matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) || (field.name - == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY - && field.dtype == FieldDataType::Plain(DataType::Utf8))) + && if field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY { + field.dtype == FieldDataType::Plain(DataType::Utf8) + } else { + matches!( + field.dtype, + FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) + ) + } }) }; let float_count = node @@ -923,4 +927,99 @@ mod tests { } validate_default(&corr_over(3), ExecutionTiming::QueryTime).unwrap(); } + // Typed derived updates retain one opaque series identity only when both + // operands cover the same series; arbitrary labels are never admitted. + #[test] + fn maintenance_binary_accepts_only_well_typed_series_identity() { + use crate::ir::non_asap::BinaryOperator; + use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind}; + let identity = crate::pre_asap::schema::PROMQL_SERIES_IDENTITY; + // A finalized per-series Sum of `metric`'s rows. + let operand = |metric: &str, schema: &Schema| { + let rows = OperatorNode::new_shared(crate::ir::Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { + metric: metric.into(), + }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let mut state = schema.clone(); + state.fields[0].dtype = exact_sum(); + let sum = Rc::new(OperatorNode::with_schema( + crate::ir::Operator::ASAP(ASAPOp::SummaryAgg { + child: rows, + family: exact_sum(), + input: SummaryUpdate::column(ColumnRef::SampleValue), + reduction: Reduction::PerEntity, + grouping: GroupingStrategy::default(), + filter: None, + }), + state, + )); + Rc::new(OperatorNode::with_schema( + crate::ir::Operator::ASAP(ASAPOp::FinalizeExactAccumulator { child: sum }), + schema.clone(), + )) + }; + let validate = |schema: Schema, rhs: &str| { + let binary = Rc::new(OperatorNode::with_schema( + crate::ir::Operator::NonASAP(NonASAPOp::BinaryOp { + operator: BinaryOperator { + kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), + vector_match: None, + checked_relative_division: false, + checked_finite_division: false, + }, + return_bool: false, + lhs: operand("m", &schema), + rhs: operand(rhs, &schema), + }), + schema, + )); + validate_default(&placed_at_ingestion(binary), ExecutionTiming::IngestionTime) + }; + let mut schema = Schema::lifted( + vec![ + Field::plain("value", DataType::Float64, false), + Field::plain("ts", DataType::Timestamp, false), + ], + None, + ); + schema.time_index = Some(1); + assert!(validate(schema.clone(), "m").is_ok()); + assert!( + validate(schema.clone(), "n").is_ok(), + "no identity to align" + ); + schema + .fields + .push(Field::plain(identity, DataType::Utf8, false)); + assert!(validate(schema.clone(), "m").is_ok()); + assert_eq!( + validate(schema.clone(), "n"), + Err(ExecutionDataStateError::InvalidMaintenanceBinary), + "different selectors may cover different series" + ); + for mutation in 0..4 { + let mut invalid = schema.clone(); + match mutation { + 0 => invalid.fields[2].nullable = true, + 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), + 2 => invalid.fields.push(invalid.fields[2].clone()), + _ => invalid.fields[2].name = "label".into(), + } + assert_eq!( + validate(invalid, "m"), + Err(ExecutionDataStateError::InvalidMaintenanceBinary) + ); + } + } } diff --git a/crates/types/src/lib.rs b/crates/types/src/lib.rs index 9fc450c7f..8c1eeb921 100644 --- a/crates/types/src/lib.rs +++ b/crates/types/src/lib.rs @@ -1,26 +1,22 @@ //! `asap-types` — shared vocabulary for the whole workspace. //! -//! Merges the former `asap-ir` crate (the pre-ASAP intent algebra, -//! workload/batch types, and DAG export) with the data-type-only modules of -//! the former `asap-sketch` crate (the post-ASAP sketch-bound IR types, -//! under [`post_asap`]). -//! -//! - [`pre_asap`] / [`types`] / [`workload`] / [`dag_export`] — the -//! pre-ASAP IR: language-agnostic query intent, independent of any -//! sketch decision. -//! - [`post_asap`] — the post-ASAP IR: sketch-bound types -//! ([`post_asap::sketch`], [`post_asap::expr`], [`post_asap::schema`]) -//! that commit to a concrete `SummaryKind`/`SummaryParams` realization. -//! No execution logic lives in this workspace (see issue #190) — a -//! downstream deployment crate is expected to supply that. -//! [`post_asap::query_time`] is the one exception, folder-separated from -//! the rest of `post_asap` on purpose: pure, sketch-object-agnostic -//! posterior error-bound math (issue #239) that a future real sketch -//! runtime's readout path can call directly — see that module's docs -//! for the planning-time/execution-time boundary and why it's unwired -//! today. +//! - [`ir`] — the unified operator IR: one operator language before and +//! after ASAP optimization ([`ir::OperatorNode`]), plus its passes +//! (canonicalize, CSE, timing) and the wire export ([`ir::physical_export`]). +//! - [`pre_asap`] — the shared field vocabulary the IR's operators are +//! built from (grouping keys, reductions, sources, aggregation intents, +//! scalar literal / operator kinds, [`pre_asap::Schema`]). +//! - [`post_asap`] — summary-state types (families, kinds, parameters, +//! grouping strategy), accuracy guarantees, and the execution-timing +//! vocabulary. No execution logic lives in this workspace (issue #190). +//! [`post_asap::query_time`] holds pure posterior error-bound math +//! (issue #239) a future sketch runtime's evaluation path can call; see its +//! docs for why it is unwired today. +//! - [`types`] / [`workload`] / [`parsed_workload`] / [`dag_export`] / +//! [`cost`] / [`resources`] — workload, batch, export and cost types. pub mod cost; pub mod dag_export; +pub mod ir; pub mod parsed_workload; pub mod post_asap; pub mod pre_asap; @@ -28,5 +24,3 @@ pub mod resources; pub mod serde_f64; pub mod types; pub mod workload; - -pub mod ir; diff --git a/crates/types/src/post_asap/cse.rs b/crates/types/src/post_asap/cse.rs deleted file mode 100644 index 55889008c..000000000 --- a/crates/types/src/post_asap/cse.rs +++ /dev/null @@ -1,442 +0,0 @@ -//! Structural sharing for a selected workload in one execution/data scope. -//! -//! This is not candidate selection or a cross-request cache. Callers opt into -//! common producer execution only after agreeing on lifecycle and data scope. -//! Typed equality includes schemas, guarantees and complete source expressions. - -use std::collections::HashMap; -use std::rc::Rc; - -use super::{SummaryExpr, SummaryNode}; - -/// Numeric PartialEq alone conflates signed zeros. The serialized check is -/// additional evidence, never a replacement for typed equality (JSON maps -/// nonfinite floats to null). Keep this rule local to structural sharing. -fn same_value(left: &T, right: &T) -> bool { - left == right - && match (serde_json::to_string(left), serde_json::to_string(right)) { - (Ok(left), Ok(right)) => left == right, - _ => false, - } -} - -/// Children have already been interned. Comparing their identities avoids -/// recursively expanding a shared DAG once for every path to each descendant. -fn same_node(left: &SummaryNode, right: &SummaryNode) -> bool { - use SummaryExpr::*; - let expression_equal = match (&left.expr, &right.expr) { - (KeepPreAsap(a), KeepPreAsap(b)) => Rc::ptr_eq(a, b) || same_value(a, b), - ( - BinaryOp { - lhs: al, - rhs: ar, - operator: ao, - timing: at, - }, - BinaryOp { - lhs: bl, - rhs: br, - operator: bo, - timing: bt, - }, - ) => Rc::ptr_eq(al, bl) && Rc::ptr_eq(ar, br) && ao == bo && at == bt, - ( - ValueOperation { - child: ac, - operation: ao, - timing: at, - }, - ValueOperation { - child: bc, - operation: bo, - timing: bt, - }, - ) => Rc::ptr_eq(ac, bc) && same_value(ao, bo) && at == bt, - ( - RelationalJoin { - left: al, - right: ar, - kind: ak, - pred: ap, - pruning: ax, - }, - RelationalJoin { - left: bl, - right: br, - kind: bk, - pred: bp, - pruning: bx, - }, - ) => { - Rc::ptr_eq(al, bl) - && Rc::ptr_eq(ar, br) - && ak == bk - && same_value(ap, bp) - && same_value(ax, bx) - } - ( - SummaryAgg { - child: ac, - family: af, - input: ai, - reduction: ar, - grouping: ag, - filter: afl, - }, - SummaryAgg { - child: bc, - family: bf, - input: bi, - reduction: br, - grouping: bg, - filter: bfl, - }, - ) => { - Rc::ptr_eq(ac, bc) - && af == bf - && same_value(ai, bi) - && ar == br - && ag == bg - && same_value(afl, bfl) - } - ( - SummaryJoin { - outer: ao, - inner: ai, - key: ak, - family: af, - }, - SummaryJoin { - outer: bo, - inner: bi, - key: bk, - family: bf, - }, - ) => Rc::ptr_eq(ao, bo) && Rc::ptr_eq(ai, bi) && ak == bk && af == bf, - ( - SummarySubtract { - left: al, - right: ar, - }, - SummarySubtract { - left: bl, - right: br, - }, - ) => Rc::ptr_eq(al, bl) && Rc::ptr_eq(ar, br), - ( - SummaryEstimate { - summary_input: ai, - query: aq, - }, - SummaryEstimate { - summary_input: bi, - query: bq, - }, - ) => Rc::ptr_eq(ai, bi) && same_value(aq, bq), - ( - SummaryDelete { - summary_input: ai, - key: ak, - }, - SummaryDelete { - summary_input: bi, - key: bk, - }, - ) => Rc::ptr_eq(ai, bi) && ak == bk, - ( - SummaryMerge { - children: a, - timing: at, - }, - SummaryMerge { - children: b, - timing: bt, - }, - ) => at == bt && a.len() == b.len() && a.iter().zip(b).all(|(a, b)| Rc::ptr_eq(a, b)), - // Keep this exhaustive on the left: new variants require a sharing rule. - ( - KeepPreAsap(_) - | BinaryOp { .. } - | ValueOperation { .. } - | RelationalJoin { .. } - | SummaryAgg { .. } - | SummaryJoin { .. } - | SummarySubtract { .. } - | SummaryEstimate { .. } - | SummaryDelete { .. } - | SummaryMerge { .. }, - _, - ) => false, - }; - expression_equal && left.schema == right.schema && same_value(&left.guarantee, &right.guarantee) -} - -/// Intern equal selected sub-DAGs across roots while preserving every root ID. -/// -/// Only structural equality is used: no grouping, parameter, accuracy or source -/// coercions are performed. All roots must belong to the same data snapshot or -/// maintenance scope. Downstream realization must still check physical -/// implementation compatibility. Use separate calls for independent executions. -/// -/// When the selected states are identical, this is the planner's -/// summary-capability rule (#509 Pass 2): one summary build node feeds every -/// readout it supports, e.g. one KLL for p50 and p99, or one UnivMon for -/// distinct count, entropy and L2. Candidate generation sizes a variant for -/// the strictest sibling consumer so differing accuracy targets can reach -/// identical states here. -pub fn share_common_summary_sub_dags( - roots: Vec<(Id, Rc)>, -) -> Vec<(Id, Rc)> { - fn visit( - node: &Rc, - seen: &mut HashMap>, - pool: &mut Vec>, - ) -> Rc { - let identity = Rc::as_ptr(node) as usize; - if let Some(node) = seen.get(&identity) { - return Rc::clone(node); - } - let mut result = node.as_ref().clone(); - match &mut result.expr { - SummaryExpr::KeepPreAsap(_) => {} - SummaryExpr::SummaryAgg { child, .. } => *child = visit(child, seen, pool), - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - *lhs = visit(lhs, seen, pool); - *rhs = visit(rhs, seen, pool); - } - - SummaryExpr::ValueOperation { child, .. } => *child = visit(child, seen, pool), - SummaryExpr::RelationalJoin { left, right, .. } => { - *left = visit(left, seen, pool); - *right = visit(right, seen, pool); - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - *outer = visit(outer, seen, pool); - *inner = visit(inner, seen, pool); - } - SummaryExpr::SummarySubtract { left, right } => { - *left = visit(left, seen, pool); - *right = visit(right, seen, pool); - } - SummaryExpr::SummaryEstimate { summary_input, .. } - | SummaryExpr::SummaryDelete { summary_input, .. } => { - *summary_input = visit(summary_input, seen, pool); - } - SummaryExpr::SummaryMerge { children, .. } => { - for child in children { - *child = visit(child, seen, pool); - } - } - } - let result = match pool.iter().find(|existing| same_node(existing, &result)) { - Some(existing) => Rc::clone(existing), - None => { - let result = Rc::new(result); - pool.push(Rc::clone(&result)); - result - } - }; - seen.insert(identity, Rc::clone(&result)); - result - } - let mut seen = HashMap::new(); - let mut pool = Vec::new(); - roots - .into_iter() - .map(|(id, root)| (id, visit(&root, &mut seen, &mut pool))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::post_asap::{ResultGuarantee, Schema}; - use crate::pre_asap::{QueryExpr, ScalarValue}; - - fn leaf(value: f64) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::Literal(ScalarValue::Float64( - value, - )))), - schema: Schema::lifted(vec![], None), - guarantee: Some(ResultGuarantee::exact("fixture")), - }) - } - - // Equal separately constructed roots preserve both IDs but share identity. - #[test] - fn shares_equal_roots_and_preserves_ids() { - let roots = share_common_summary_sub_dags(vec![("a", leaf(1.0)), ("b", leaf(1.0))]); - assert_eq!(roots[0].0, "a"); - assert_eq!(roots[1].0, "b"); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - - // A diamond is retained across the returned roots, not copied per consumer. - #[test] - fn shares_children_across_distinct_roots() { - let merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - timing: crate::post_asap::ExecutionTiming::IngestionTime, - children: vec![leaf(1.0), leaf(2.0)], - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }); - let roots = share_common_summary_sub_dags(vec![(0, leaf(1.0)), (1, merge)]); - let SummaryExpr::SummaryMerge { children, .. } = &roots[1].1.expr else { - panic!() - }; - assert!(Rc::ptr_eq(&roots[0].1, &children[0])); - assert!(!Rc::ptr_eq(&children[0], &children[1])); - } - - // Unknown guarantees must not be replaced by an equal expression's exact guarantee. - #[test] - fn distinct_guarantees_and_values_are_not_shared() { - let mut unknown = leaf(1.0).as_ref().clone(); - unknown.guarantee = None; - let roots = share_common_summary_sub_dags(vec![ - (0, leaf(1.0)), - (1, Rc::new(unknown)), - (2, leaf(2.0)), - ]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[2].1)); - assert!(roots[1].1.guarantee.is_none()); - } - - // Sharing must preserve IEEE signed zero, including inside exact expressions. - #[test] - fn signed_zero_is_not_coalesced() { - for values in [[0.0, -0.0], [-0.0, 0.0]] { - let roots = - share_common_summary_sub_dags(vec![(0, leaf(values[0])), (1, leaf(values[1]))]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - for ((_, root), expected) in roots.iter().zip(values) { - let SummaryExpr::KeepPreAsap(expr) = &root.expr else { - panic!() - }; - let QueryExpr::Literal(ScalarValue::Float64(actual)) = expr.as_ref() else { - panic!() - }; - assert_eq!(actual.to_bits(), expected.to_bits()); - assert_eq!(1.0 / actual, 1.0 / expected); - } - } - } - - // Exact expression wrappers must retain signed zero too; JSON's null - // encoding of nonfinite floats must never become the equality decision. - #[test] - fn nested_values_and_nonfinite_values_remain_distinct() { - let wrapped = |value| { - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(Rc::new(QueryExpr::promql_scalar(value))), - ..leaf(1.0).as_ref().clone() - }) - }; - for (a, b) in [ - (0.0, -0.0), - (f64::INFINITY, f64::NEG_INFINITY), - (f64::NAN, f64::NAN), - ] { - let roots = share_common_summary_sub_dags(vec![(0, wrapped(a)), (1, wrapped(b))]); - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - let roots = share_common_summary_sub_dags(vec![ - (0, wrapped(f64::INFINITY)), - (1, wrapped(f64::INFINITY)), - ]); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - } - - // Distinct quantile readouts share only a compatible typed sketch producer. - #[test] - fn quantile_roots_share_producer_but_not_readout_or_parameters() { - use crate::post_asap::{ - FieldDataType, GroupingStrategy, SketchAlgorithm, SketchKind, SketchParams, - SketchStatistic, SummaryUpdate, - }; - use crate::pre_asap::{ColumnRef, Reduction}; - fn readout(q: f64, alpha: f64) -> Rc { - let producer = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: leaf(1.0), - family: FieldDataType::Sketch( - SketchKind::new( - SketchAlgorithm::DDSketch, - SketchParams::DDSketch { alpha }, - ), - GroupingStrategy::default(), - ), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }); - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: producer, - query: SketchStatistic::Quantile { q }, - }, - schema: Schema::lifted(vec![], None), - guarantee: None, - }) - } - let roots = share_common_summary_sub_dags(vec![ - ("p95", readout(0.95, 0.01)), - ("p99", readout(0.99, 0.01)), - ("strict", readout(0.95, 0.001)), - ]); - let producer = |root: &Rc| match &root.expr { - SummaryExpr::SummaryEstimate { summary_input, .. } => Rc::clone(summary_input), - _ => panic!(), - }; - assert!(!Rc::ptr_eq(&roots[0].1, &roots[1].1)); - assert!(Rc::ptr_eq(&producer(&roots[0].1), &producer(&roots[1].1))); - assert!(!Rc::ptr_eq(&producer(&roots[0].1), &producer(&roots[2].1))); - } - - // Fifty unique input nodes must not require walking an expanded 2^24 DAG. - // The timeout is a coarse runaway guard, not a performance SLA. - #[test] - fn shared_diamond_does_not_expand_during_comparison() { - let (done, completion) = std::sync::mpsc::channel(); - let worker = std::thread::spawn(move || { - fn diamond() -> Rc { - let mut current = leaf(1.0); - for _ in 0..24 { - current = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - timing: super::super::ExecutionTiming::QueryTime, - lhs: Rc::clone(¤t), - rhs: current, - operator: super::super::BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: crate::pre_asap::BinaryOpKind::Arithmetic( - crate::pre_asap::ArithmeticOpKind::Add, - ), - vector_match: None, - }, - }, - schema: super::super::Schema::lifted(vec![], None), - guarantee: None, - }); - } - current - } - let roots = share_common_summary_sub_dags(vec![(0, diamond()), (1, diamond())]); - assert!(Rc::ptr_eq(&roots[0].1, &roots[1].1)); - done.send(()).unwrap(); - }); - completion - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("comparison expanded the shared DAG"); - worker.join().unwrap(); - } -} diff --git a/crates/types/src/post_asap/execution_data_state.rs b/crates/types/src/post_asap/execution_data_state.rs index 629465354..8b1b197de 100644 --- a/crates/types/src/post_asap/execution_data_state.rs +++ b/crates/types/src/post_asap/execution_data_state.rs @@ -1,56 +1,14 @@ -//! Execution-data-state contract for mixed exact/summary plans (issue #171). +//! Execution timing and data-state vocabulary of the operator IR. //! -//! A post-ASAP DAG mixes two very different moments of execution: the -//! **update/ingest path** (rows arrive, maintained summary state is updated) -//! and **query evaluation** (maintained state is read out and a final result -//! is produced). A plan that places a query-time residual *underneath* a -//! maintained summary is not merely expensive — it is unexecutable, because -//! the maintenance loop has no readout values to feed into that summary. -//! [`SummaryExpr::ValueOperation`] represents such work without inventing a -//! node per function or use case. Its [`ExecutionTiming`] makes placement -//! explicit and independent of the semantic [`ValueOperation`]. -//! -//! [`ExecutionDataState`] is what a node's output *is*, at which data_state; -//! [`validate_execution_data_states`] checks every edge of a DAG against the -//! rules below at plan construction, returning a typed [`ExecutionDataStateError`] rather -//! than deferring to a runtime failure. -//! -//! ## Edge rules -//! -//! | Parent | Accepts from `child` | -//! |---|---| -//! | `SummaryAgg.child` | Rows or exact accumulator state at either phase. The initial construction phase follows the input; deployment assigns final phases. | -//! | `SummaryEstimate.summary_input` | Summary state at either phase (any family). Initial readout produces `QUERY_ROWS`. | -//! | `SummaryJoin.outer/inner` | `INGESTION_ROWS` or `INGESTION_SUMMARY`; never a read-time data_state. | -//! | `SummarySubtract`/`SummaryDelete` | `INGESTION_SUMMARY`. | -//! | `SummaryMerge` | Summary state at its explicit ingestion or read timing. | -//! | `ValueOperation.child` with `IngestionTime` | `INGESTION_ROWS`; explicit `FinalizeExactAccumulator` also accepts exact accumulator state. Produces `INGESTION_ROWS`. | -//! | `ValueOperation.child` with `QueryTime` | `QUERY_ROWS`. Produces `QUERY_ROWS`. | -//! -//! ## `KeepPreAsap` declares its data_state through the derivation -//! -//! A [`SummaryExpr::KeepPreAsap`] leaf is a raw pre-ASAP computation that a -//! runtime can execute at either time: as maintenance input beneath a -//! `SummaryAgg`/maintenance-time `ValueOperation`, or as a query-time fallback -//! beneath a read-time `ValueOperation` (or at the root). It carries no timing -//! field of its own -//! — every existing consumer pattern-matches the one-field shape — so its -//! data_state is *assigned* by [`validate_execution_data_states`] from the edge that -//! reaches it and reported in the returned [`ExecutionDataStateAssignment`]. What it may -//! not do is stay ambiguous inside one mixed plan: the same `Rc` -//! reached once as update input and once as query-time fallback is -//! [`ExecutionDataStateError::AmbiguousKeepPreAsap`], because no single execution of that -//! sub-DAG can serve both roles. - -use std::collections::HashMap; -use std::rc::Rc; +//! [`ExecutionTiming`] says when a node's value is produced (ingestion vs. +//! query time); [`ExecutionDataState`] pairs it with the [`DataPrimitive`] +//! the edge carries (raw values vs. summary state). The rules that assign +//! and check them over a DAG live in [`crate::ir::timing`], which reports +//! violations as [`ExecutionDataStateError`]. use thiserror::Error; -use super::expr::{ExactOperation, SummaryExpr, SummaryNode, ValueOperation}; -use crate::pre_asap::schema::FieldDataType; - -use crate::pre_asap::query_expr::{aggregate_output_schema, Predicate, QueryExprError}; +use crate::ir::SchemaDerivationError; use crate::pre_asap::schema::Schema; /// When a post-ASAP value is produced. @@ -65,9 +23,6 @@ pub enum ExecutionTiming { } impl ExecutionTiming { - pub fn is_query_time(&self) -> bool { - *self == Self::QueryTime - } pub fn as_str(self) -> &'static str { match self { Self::IngestionTime => "ingestion_time", @@ -79,7 +34,7 @@ impl ExecutionTiming { /// The primitive representation carried by a post-ASAP edge. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] pub enum DataPrimitive { - /// Directly usable values, including approximate summary readouts. + /// Directly usable values, including approximate summary evaluations. /// This does not imply original input data or an exact guarantee. Raw, SummaryState, @@ -123,67 +78,27 @@ impl std::fmt::Display for ExecutionDataState { } } -/// Which parent/edge a [`ExecutionDataStateError`] is about — the variant name of the -/// parent `SummaryExpr` plus its field, for a message a plan author can act -/// on. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum ExecutionDataStateEdge { - SummaryAggChild, - SummaryEstimateInput, - SummaryJoinInput, - SummarySubtractInput, - SummaryDeleteInput, - SummaryMergeInput, - ValueOperationChild, -} - -impl ExecutionDataStateEdge { - fn describe(self) -> &'static str { - match self { - Self::SummaryAggChild => "SummaryAgg.child", - Self::SummaryEstimateInput => "SummaryEstimate.summary_input", - Self::SummaryJoinInput => "SummaryJoin.{outer,inner}", - Self::SummarySubtractInput => "SummarySubtract.{left,right}", - Self::SummaryDeleteInput => "SummaryDelete.summary_input", - Self::SummaryMergeInput => "SummaryMerge.children[]", - Self::ValueOperationChild => "ValueOperation.child", - } - } -} - /// A plan-construction-time data_state violation. Typed (not a string) so a /// strategy can degrade to a conservative fallback on the specific variant /// it expects, and so tests can assert the *reason* a plan was rejected. #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum ExecutionDataStateError { - #[error("operator reached with conflicting execution timings: {first:?} and {second:?}")] - ConflictingTiming { - first: ExecutionDataState, - second: ExecutionDataState, - }, - #[error("{operator} node has no execution timing")] - UntimedNode { operator: &'static str }, - #[error("evaluation value under maintenance: {edge} received {child}")] - EvaluationUnderMaintenance { - edge: &'static str, - child: ExecutionDataState, - }, - #[error("invalid maintained-population maintenance/readout contract")] + #[error("invalid maintained-population maintenance/evaluation contract")] InvalidMaintainedPopulation, - /// A query-time value (`SummaryEstimate` / read-time `ValueOperation` output) + /// A query-time value (a `SummaryEstimate` or query-time operator output) /// placed beneath a maintained summary — the one shape issue #171's /// data_state split exists to make unrepresentable. #[error( - "readout value under maintenance: {edge} received a {child} input, but a maintained \ + "evaluation value under maintenance: {edge} received a {child} input, but a maintained \ summary can only consume update-path values (or exact accumulator state)" )] - ReadoutUnderMaintenance { + EvaluationUnderMaintenance { edge: &'static str, child: ExecutionDataState, }, /// Any other edge whose child data_state the parent does not accept /// (e.g. plain update rows fed straight into a `SummaryEstimate`, or a - /// sketch's opaque state fed into a read-time `ValueOperation`). + /// sketch's opaque state fed into a query-time operator). #[error("{edge} does not accept a {child} input")] IllegalChildDataState { edge: &'static str, @@ -197,13 +112,10 @@ pub enum ExecutionDataStateError { composed into another maintained summary" )] UnsupportedStateComposition { family: String }, - /// One shared `KeepPreAsap` node reached both as update-path raw input - /// and as a query-time fallback — see the module docs. - #[error( - "KeepPreAsap sub-DAG is data_state-ambiguous: reached as {first} and as {second} in the same \ - plan" - )] - AmbiguousKeepPreAsap { + /// One shared node assigned two different execution timings by its + /// consumers; no single execution of it can serve both. + #[error("shared node is assigned conflicting timings: {first} and {second}")] + ConflictingTiming { first: ExecutionDataState, second: ExecutionDataState, }, @@ -215,623 +127,44 @@ pub enum ExecutionDataStateError { InvalidMaintenanceBinary, #[error("checked division requires one valid guard on a read-time division operator")] InvalidCheckedDivision, - /// An `ExactOperation` whose input columns are not all `Plain` at its + /// An exact operator whose input columns are not all `Plain` at its /// declared data_state. #[error("exact operator consumes non-plain column {column:?} ({dtype})")] NonPlainOperand { column: String, dtype: String }, - /// A reserved ASAP operator (`SummaryMerge`, `SummarySubtract`, - /// `SummaryDelete`, `SummaryJoin`, `Extension`) in an executable plan. - #[error("{operator} is a reserved operator with no execution contract yet")] + /// An ASAP operator with no execution timing contract yet in an + /// executable plan: `SummaryMerge` (schema and coverage are implemented) + /// and the reserved `SummarySubtract`, `SummaryDelete`, `SummaryJoin` + /// and `Extension`. + #[error("{operator} has no execution timing contract yet")] UnimplementedOperator { operator: &'static str }, + /// A node reached by export without a timing: the lifecycle timing pass + /// was not applied to the DAG first. + #[error("{operator} node has no execution timing; apply lifecycle timings before export")] + UntimedNode { operator: &'static str }, /// The exported physical DAG failed its own validation, e.g. an empty /// batch or a hand-timed plan the timing pass would have rejected. #[error("invalid physical ASAP DAG: {0}")] InvalidPhysicalDAG(#[from] crate::ir::physical_export::PhysicalASAPDAGValidationError), } -/// The data_state assigned to every node of a validated plan, keyed by -/// `Rc` pointer identity — the explicit per-node "execution_data_state" a -/// runtime or a DAG export reads instead of re-deriving it. For every -/// non-`KeepPreAsap` node this equals [`produced_data_state`]; for a -/// `KeepPreAsap` leaf it is the data_state the reaching edge assigned. -#[derive(Debug, Clone, Default)] -pub struct ExecutionDataStateAssignment { - domains: HashMap<*const SummaryNode, ExecutionDataState>, -} - -impl ExecutionDataStateAssignment { - /// The data_state assigned to `node`, if it was part of the validated plan. - pub fn data_state_of(&self, node: &Rc) -> Option { - self.domains.get(&Rc::as_ptr(node)).copied() - } - - /// The data_state assigned to the node at `ptr` — for callers walking a plan - /// by reference rather than by `Rc`. - pub fn data_state_of_ptr(&self, ptr: *const SummaryNode) -> Option { - self.domains.get(&ptr).copied() - } -} - -/// Initial layout proposed by semantic realization, not a restriction on physical -/// operator placement. `PostAsapDAG::with_execution_phases` assigns the final -/// phase independently of payload kind. Returns `None` for -/// [`SummaryExpr::KeepPreAsap`], whose data_state is assigned by the edge reaching -/// it (see the module docs). -pub fn produced_data_state(expr: &SummaryExpr) -> Option { - Some(match expr { - SummaryExpr::KeepPreAsap(_) => return None, - SummaryExpr::BinaryOp { timing, .. } => ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::Raw, - }, - SummaryExpr::RelationalJoin { .. } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::SummaryAgg { child, .. } => ExecutionDataState { - timing: produced_data_state(&child.expr) - .map_or(ExecutionTiming::IngestionTime, |state| state.timing), - primitive: DataPrimitive::SummaryState, - }, - SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } => ExecutionDataState::INGESTION_SUMMARY, - SummaryExpr::SummaryMerge { timing, .. } => ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::SummaryState, - }, - SummaryExpr::SummaryEstimate { .. } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::ValueOperation { timing, .. } => match timing { - ExecutionTiming::IngestionTime => ExecutionDataState::INGESTION_ROWS, - ExecutionTiming::QueryTime => ExecutionDataState::QUERY_ROWS, - }, - }) -} - -/// Is `family` the exact-accumulator family whose partial state *is* the -/// value — the one summary state a `SummaryAgg` may re-accumulate? -fn is_exact_accumulator_state(schema: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &schema.fields { - match &field.dtype { - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) => {} - other => { - return Err(ExecutionDataStateError::UnsupportedStateComposition { - family: format!("{other:?}"), - }) - } - } - } - Ok(()) -} - -/// Validate every edge of the DAG rooted at `root` against the module-level -/// rules, returning each node's assigned data_state on success. Shared -/// `Rc`s are visited once per reaching edge (the assignment is -/// per node, so a conflict between two edges is what -/// [`ExecutionDataStateError::AmbiguousKeepPreAsap`] detects). -pub fn validate_execution_data_states( - root: &Rc, -) -> Result { - // The root may be a readable value or bare maintained state (a - // deployment may hand an `ExactAggregate` accumulator straight to a - // consumer) — only an update-path-only root is meaningless. - let root_domain = match produced_data_state(&root.expr) { - None => ExecutionDataState::QUERY_ROWS, - Some(ExecutionDataState::INGESTION_ROWS) => { - return Err(ExecutionDataStateError::MaintenanceRowsAtRoot) - } - Some(data_state) => data_state, - }; - validate_execution_data_states_at(root, root_domain) -} - -/// [`validate_execution_data_states`] for a *sub*-plan whose root is known to -/// sit at `data_state` — e.g. a maintenance-time `ValueOperation` about to be placed beneath a -/// `SummaryAgg`, which would be rejected as a whole-plan root but is a -/// legal update-path input. Validates every edge beneath `root` exactly -/// as the whole-plan entry point does. -pub fn validate_execution_data_states_at( - root: &Rc, - data_state: ExecutionDataState, -) -> Result { - let mut assignment = ExecutionDataStateAssignment::default(); - visit(root, data_state, &mut assignment)?; - Ok(assignment) -} - -/// The source rows whose series a maintenance operand has one row for: a -/// finalized per-series Sum or Count of those rows, or aligned arithmetic of -/// operands over the same rows. Each emits exactly the series with a sample. -fn per_series_rows(node: &SummaryNode) -> Option<&crate::pre_asap::QueryExpr> { - use crate::post_asap::ExactKind; - match &node.expr { - SummaryExpr::ValueOperation { - child, - operation: ValueOperation::FinalizeExactAccumulator, - .. - } => match &child.expr { - SummaryExpr::SummaryAgg { - child, - family: FieldDataType::ExactAggregate(ExactKind::Sum | ExactKind::Count, _), - reduction: crate::pre_asap::query_expr::Reduction::PerEntity, - .. - } => match &child.expr { - SummaryExpr::KeepPreAsap(rows) => Some(rows.as_ref()), - _ => None, - }, - _ => None, - }, - SummaryExpr::BinaryOp { - lhs, - rhs, - timing: ExecutionTiming::IngestionTime, - .. - } => { - let rows = per_series_rows(lhs)?; - (per_series_rows(rhs) == Some(rows)).then_some(rows) - } - _ => None, - } -} - -/// Record `data_state` for `node` (detecting a conflicting earlier assignment -/// for a `KeepPreAsap`), then check and recurse into every child edge. -fn visit( - node: &Rc, - data_state: ExecutionDataState, - assignment: &mut ExecutionDataStateAssignment, -) -> Result<(), ExecutionDataStateError> { - let ptr = Rc::as_ptr(node); - if let Some(previous) = assignment.domains.get(&ptr) { - if *previous != data_state { - return Err(ExecutionDataStateError::AmbiguousKeepPreAsap { - first: *previous, - second: data_state, - }); - } - // Already validated through another edge with the same data_state. - return Ok(()); - } - assignment.domains.insert(ptr, data_state); - - match &node.expr { - SummaryExpr::KeepPreAsap(_) => Ok(()), - SummaryExpr::BinaryOp { - lhs, - rhs, - timing, - operator, - } => { - if (operator.checked_relative_division && operator.checked_finite_division) - || (operator.checked_relative_division || operator.checked_finite_division) - && (*timing != ExecutionTiming::QueryTime - || !matches!( - operator.kind, - crate::pre_asap::BinaryOpKind::Arithmetic( - crate::pre_asap::ArithmeticOpKind::Div - ) - )) - { - return Err(ExecutionDataStateError::InvalidCheckedDivision); - } - if *timing == ExecutionTiming::IngestionTime { - use crate::pre_asap::{BinaryOpKind, DataType}; - if operator.vector_match.is_some() - || !matches!(operator.kind, BinaryOpKind::Arithmetic(_)) - || lhs.schema != rhs.schema - || lhs.schema != node.schema - // The opaque identity is a key, not an extra maintenance value. - || node.schema.fields.iter().filter(|field| { - field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY - }).count() > 1 - // Maintenance arithmetic pairs every row by identity, while - // Prometheus drops unmatched series; it is exact only when - // both operands provably produce the same series. - || node.schema.fields.iter().any(|field| { - field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY - }) && per_series_rows(lhs).is_none_or(|rows| per_series_rows(rhs) != Some(rows)) - || !node.schema.fields.iter().all(|field| { - !field.nullable - && if field.name == crate::pre_asap::schema::PROMQL_SERIES_IDENTITY { - field.dtype == FieldDataType::Plain(DataType::Utf8) - } else { - matches!( - field.dtype, - FieldDataType::Plain(DataType::Float64 | DataType::Timestamp) - ) - } - }) - || node - .schema - .fields - .iter() - .filter(|field| { - matches!(field.dtype, FieldDataType::Plain(DataType::Float64)) - }) - .count() - != 1 - { - return Err(ExecutionDataStateError::InvalidMaintenanceBinary); - } - } - let expected = ExecutionDataState { - timing: *timing, - primitive: DataPrimitive::Raw, - }; - for input in [lhs, rhs] { - let state = produced_data_state(&input.expr).unwrap_or(expected); - if state != expected { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "BinaryOp operand", - child: state, - }); - } - visit(input, state, assignment)?; - } - Ok(()) - } - - SummaryExpr::RelationalJoin { left, right, .. } => { - for input in [left, right] { - let state = - produced_data_state(&input.expr).unwrap_or(ExecutionDataState::QUERY_ROWS); - if state != ExecutionDataState::QUERY_ROWS { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: "RelationalJoin input", - child: state, - }); - } - visit(input, state, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryAgg { child, .. } => { - let child_domain = child_domain( - child, - ExecutionDataStateEdge::SummaryAggChild, - |avail| match avail { - ExecutionDataState::INGESTION_ROWS | ExecutionDataState::QUERY_ROWS => Ok(()), - state if state.primitive == DataPrimitive::SummaryState => { - is_exact_accumulator_state(&child.schema) - } - other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { - edge: ExecutionDataStateEdge::SummaryAggChild.describe(), - child: other, - }), - }, - )?; - visit(child, child_domain, assignment) - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - for input in [outer, inner] { - let s = child_domain(input, ExecutionDataStateEdge::SummaryJoinInput, |avail| { - match avail { - ExecutionDataState::INGESTION_ROWS - | ExecutionDataState::INGESTION_SUMMARY => Ok(()), - other => Err(ExecutionDataStateError::ReadoutUnderMaintenance { - edge: ExecutionDataStateEdge::SummaryJoinInput.describe(), - child: other, - }), - } - })?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummarySubtract { left, right } => { - for input in [left, right] { - let s = state_only(input, ExecutionDataStateEdge::SummarySubtractInput)?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryDelete { summary_input, .. } => { - let s = state_only(summary_input, ExecutionDataStateEdge::SummaryDeleteInput)?; - visit(summary_input, s, assignment) - } - SummaryExpr::SummaryMerge { children, timing } => { - for input in children { - let s = child_domain(input, ExecutionDataStateEdge::SummaryMergeInput, |state| { - if state.primitive == DataPrimitive::SummaryState - && (*timing == ExecutionTiming::QueryTime || state.timing == *timing) - { - Ok(()) - } else { - Err(ExecutionDataStateError::IllegalChildDataState { - edge: ExecutionDataStateEdge::SummaryMergeInput.describe(), - child: state, - }) - } - })?; - visit(input, s, assignment)?; - } - Ok(()) - } - SummaryExpr::SummaryEstimate { summary_input, .. } => { - let s = state_only(summary_input, ExecutionDataStateEdge::SummaryEstimateInput)?; - visit(summary_input, s, assignment) - } - SummaryExpr::ValueOperation { - child, - operation, - timing, - } => { - // Population timing is a lifecycle decision: a retained population - // is maintained at ingestion time, an ephemeral one is rebuilt - // from raw input per query. Its input and readout contracts are - // structural and hold either way. - let valid_population = match operation { - ValueOperation::MaintainPopulation { population } => { - matches!(&child.expr, SummaryExpr::KeepPreAsap(input) if population.matches_input(input)) - } - ValueOperation::ReadPopulation { readout } => { - *timing == ExecutionTiming::QueryTime - && matches!(&child.expr, SummaryExpr::ValueOperation { operation: ValueOperation::MaintainPopulation { population }, .. } if population.supports(readout)) - } - _ => true, - }; - if !valid_population { - return Err(ExecutionDataStateError::InvalidMaintainedPopulation); - } - let required = match timing { - ExecutionTiming::IngestionTime => ExecutionDataState::INGESTION_ROWS, - ExecutionTiming::QueryTime => ExecutionDataState::QUERY_ROWS, - }; - let s = produced_data_state(&child.expr).unwrap_or(required); - let exact_readout = (*timing == ExecutionTiming::QueryTime - || matches!(operation, ValueOperation::FinalizeExactAccumulator)) - && s.primitive == DataPrimitive::SummaryState - && (*timing == ExecutionTiming::QueryTime || s.timing == *timing) - && is_exact_accumulator_state(&child.schema).is_ok(); - // A query-time readout may read a population retained at ingestion. - let population_readout = matches!(operation, ValueOperation::ReadPopulation { .. }) - && *timing == ExecutionTiming::QueryTime - && matches!( - &child.expr, - SummaryExpr::ValueOperation { - operation: ValueOperation::MaintainPopulation { .. }, - .. - } - ); - if s != required && !exact_readout && !population_readout { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: ExecutionDataStateEdge::ValueOperationChild.describe(), - child: s, - }); - } - check_plain_operands(operation, &child.schema)?; - visit(child, s, assignment) - } - } -} - -/// The data_state `child` takes as a direct input of `parent`, without -/// validating legality — `child`'s own produced data_state, or for a -/// `KeepPreAsap` leaf the data_state `parent`'s edge assigns it (update-path raw -/// input under maintenance-time operation edges, query-time fallback under a -/// a read-time operation, and — meaninglessly, but for a stable answer — maintenance rows -/// under a state-only edge). For DAG export and other reporting that needs -/// an explicit per-node data_state even on a plan that -/// [`validate_execution_data_states`] would reject. -pub fn assigned_child_data_state(parent: &SummaryExpr, child: &SummaryNode) -> ExecutionDataState { - if let Some(avail) = produced_data_state(&child.expr) { - return avail; - } - match parent { - SummaryExpr::ValueOperation { - timing: ExecutionTiming::QueryTime, - .. - } - | SummaryExpr::BinaryOp { - timing: ExecutionTiming::QueryTime, - .. - } => ExecutionDataState::QUERY_ROWS, - SummaryExpr::KeepPreAsap(_) - | SummaryExpr::BinaryOp { - timing: ExecutionTiming::IngestionTime, - .. - } - | SummaryExpr::RelationalJoin { .. } - | SummaryExpr::SummaryAgg { .. } - | SummaryExpr::SummaryJoin { .. } - | SummaryExpr::SummarySubtract { .. } - | SummaryExpr::SummaryDelete { .. } - | SummaryExpr::SummaryEstimate { .. } - | SummaryExpr::SummaryMerge { .. } - | SummaryExpr::ValueOperation { - timing: ExecutionTiming::IngestionTime, - .. - } => ExecutionDataState::INGESTION_ROWS, - } -} - -/// The data_state `child` takes on `edge`: its own produced data_state -/// (checked via `accept`), or — for a `KeepPreAsap` leaf — the data_state the -/// edge assigns it, derived from what that edge accepts. -fn child_domain( - child: &Rc, - edge: ExecutionDataStateEdge, - accept: impl Fn(ExecutionDataState) -> Result<(), ExecutionDataStateError>, -) -> Result { - match produced_data_state(&child.expr) { - Some(avail) => { - accept(avail)?; - Ok(avail) - } - None => { - // A raw pre-ASAP sub-DAG executes at whichever data_state its consumer - // needs: update-path input for maintenance-time operation edges, - // query-time fallback for a read-time edge. State-only edges - // can't consume plain rows at all. - let assigned = match edge { - ExecutionDataStateEdge::SummaryAggChild - | ExecutionDataStateEdge::SummaryJoinInput - | ExecutionDataStateEdge::ValueOperationChild => ExecutionDataState::INGESTION_ROWS, - ExecutionDataStateEdge::SummaryEstimateInput - | ExecutionDataStateEdge::SummarySubtractInput - | ExecutionDataStateEdge::SummaryDeleteInput - | ExecutionDataStateEdge::SummaryMergeInput => { - return Err(ExecutionDataStateError::IllegalChildDataState { - edge: edge.describe(), - child: ExecutionDataState::INGESTION_ROWS, - }) - } - }; - accept(assigned)?; - Ok(assigned) - } - } -} - -fn state_only( - child: &Rc, - edge: ExecutionDataStateEdge, -) -> Result { - child_domain(child, edge, |avail| match avail { - state - if state.primitive == DataPrimitive::SummaryState - && (state.timing == ExecutionTiming::IngestionTime - || matches!(edge, ExecutionDataStateEdge::SummaryEstimateInput)) => - { - Ok(()) - } - other => Err(ExecutionDataStateError::IllegalChildDataState { - edge: edge.describe(), - child: other, - }), - }) -} - -/// The exact operator must consume only `Plain` columns of its input: for -/// an `Aggregate` payload, every grouping key and every measure's input -/// column. -fn check_plain_operands( - op: &ValueOperation, - input: &Schema, -) -> Result<(), ExecutionDataStateError> { - if matches!( - op, - ValueOperation::Sort { .. } - | ValueOperation::Limit { .. } - | ValueOperation::Project { .. } - | ValueOperation::Filter { .. } - | ValueOperation::FinalizeExactAccumulator - ) { - return check_plain_or_exact_values(input); - } - let ValueOperation::Exact(op) = op else { - return check_all_plain(input); - }; - let ExactOperation::Aggregate { - reduction, - measures, - filters, - .. - } = op; - let mut referenced: Vec = reduction - .group_keys() - .map(|keys| keys.keys().to_vec()) - .unwrap_or_default(); - for m in measures { - referenced.extend(m.input_cols()); - } - for Predicate(f) in filters.iter().flatten() { - referenced.extend(f.columns_referenced().into_iter().copied()); - } - // With no explicit input column (the PromQL sample-value convention) - // the operator reads every non-key column, so all must be plain. - let implicit = measures.iter().any(|m| m.input_cols().is_empty()); - for (i, field) in input.fields.iter().enumerate() { - if !(implicit || referenced.contains(&i)) { - continue; - } - if !matches!(field.dtype, FieldDataType::Plain(_)) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_plain_or_exact_values(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!( - field.dtype, - FieldDataType::Plain(_) | FieldDataType::ExactAggregate(..) - ) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -fn check_all_plain(input: &Schema) -> Result<(), ExecutionDataStateError> { - for field in &input.fields { - if !matches!(field.dtype, FieldDataType::Plain(_)) { - return Err(ExecutionDataStateError::NonPlainOperand { - column: field.name.clone(), - dtype: format!("{:?}", field.dtype), - }); - } - } - Ok(()) -} - -/// `schema` with its reuse metadata dropped, or `None` if any field carries -/// summary state — the shape an exact operator reads. -pub fn plain_schema(schema: &Schema) -> Option { - schema - .is_all_plain() - .then(|| Schema::lifted(schema.fields.clone(), schema.time_index)) -} - /// `schema` as a summary-planning node output: fields and time axis kept, /// unique keys dropped, closed. pub fn lift_plain(schema: &Schema) -> Schema { Schema::lifted(schema.fields.clone(), schema.time_index) } -/// Output schema of `op` applied to a child whose edge carries `input` — -/// the same canonical derivation the pre-ASAP `Aggregate` node uses, so an -/// exact `ValueOperation` never disagrees with the pre-ASAP -/// target it was lowered from. `Err` when the child carries non-plain -/// state the operator cannot read. -pub fn exact_operation_output_schema( - op: &ExactOperation, - input: &Schema, -) -> Result { - let plain = plain_schema(input).ok_or(ExactOperationSchemaError::NonPlainInput)?; - let ExactOperation::Aggregate { - reduction, - measures, - output_names, - .. - } = op; - let out = aggregate_output_schema(&plain, reduction, measures, output_names)?; - Ok(lift_plain(&out)) -} - -/// Why [`exact_operation_output_schema`] could not derive a schema. +/// Why an exact operator's output schema could not be derived. #[derive(Debug, Error)] pub enum ExactOperationSchemaError { #[error("exact operator input carries summary state, not plain columns")] NonPlainInput, #[error("schema derivation failed: {0}")] - Schema(#[from] QueryExprError), - #[error("schema derivation failed: {0}")] - Derivation(#[from] crate::ir::SchemaDerivationError), + Schema(#[from] SchemaDerivationError), } #[cfg(test)] mod tests { use super::*; - use crate::post_asap::{ExactKind, ExactParams, GroupingStrategy, SketchStatistic}; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::ColumnRef; - use crate::pre_asap::query_expr::{QueryExpr, Reduction, Source}; - use crate::pre_asap::schema::{DataType, Field}; /// Both execution phases use raw values, distinct from maintained state. #[test] @@ -849,347 +182,6 @@ mod tests { assert_eq!(DataPrimitive::SummaryState.as_str(), "summary_state"); } - fn scan() -> Rc { - Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("zone", DataType::Utf8, true), - ], - 0, - vec![], - ), - }) - } - - fn keep() -> Rc { - let s = scan(); - let schema = lift_plain(&s.output_schema().unwrap()); - Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(s), - schema, - guarantee: None, - }) - } - - fn plain(names: &[&str]) -> Schema { - Schema::lifted( - names - .iter() - .map(|n| Field { - name: (*n).into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }) - .collect(), - None, - ) - } - - fn agg(child: Rc, family: FieldDataType) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child, - family: family.clone(), - input: crate::post_asap::SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted( - vec![Field { - name: "state".into(), - dtype: family, - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }) - } - - fn kll() -> FieldDataType { - use crate::post_asap::{SketchAlgorithm, SketchKind, SketchParams}; - FieldDataType::Sketch( - SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), - GroupingStrategy::default(), - ) - } - - fn estimate(child: Rc) -> Rc { - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryEstimate { - summary_input: child, - query: SketchStatistic::Quantile { q: 0.99 }, - }, - schema: plain(&["quantile_0_99"]), - guarantee: None, - }) - } - - fn max_op() -> ExactOperation { - ExactOperation::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - having: None, - filters: vec![], - } - } - - #[test] - fn keep_pre_asap_under_summary_agg_is_update_input() { - let leaf = keep(); - let root = agg(Rc::clone(&leaf), kll()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&leaf), - Some(ExecutionDataState::INGESTION_ROWS) - ); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::INGESTION_SUMMARY) - ); - } - - // Typed derived updates retain one opaque series identity only when both - // operands cover the same series; arbitrary labels are never admitted. - #[test] - fn maintenance_binary_accepts_only_well_typed_series_identity() { - use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind}; - let identity = crate::pre_asap::schema::PROMQL_SERIES_IDENTITY; - // A finalized per-series Sum of `metric`'s rows. - let operand = |metric: &str, schema: &Schema| { - let rows = Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { - metric: metric.into(), - }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - }); - let mut state = schema.clone(); - state.fields[0].dtype = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let sum = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child: Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(rows), - schema: schema.clone(), - guarantee: None, - }), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: crate::post_asap::SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::PerEntity, - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: state, - guarantee: None, - }); - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: sum, - operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::IngestionTime, - }, - schema: schema.clone(), - guarantee: None, - }) - }; - let validate = |schema: Schema, rhs: &str| { - let binary = Rc::new(SummaryNode { - expr: SummaryExpr::BinaryOp { - lhs: operand("m", &schema), - rhs: operand(rhs, &schema), - timing: ExecutionTiming::IngestionTime, - operator: crate::post_asap::BinaryOperator { - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - checked_relative_division: false, - checked_finite_division: false, - }, - }, - schema, - guarantee: None, - }); - validate_execution_data_states(&estimate(agg(binary, kll()))).map(|_| ()) - }; - let mut schema = plain(&["value"]); - schema.fields.push(Field { - table: None, - name: "ts".into(), - dtype: FieldDataType::Plain(DataType::Timestamp), - nullable: false, - }); - schema.time_index = Some(1); - assert!(validate(schema.clone(), "m").is_ok()); - assert!( - validate(schema.clone(), "n").is_ok(), - "no identity to align" - ); - schema.fields.push(Field { - table: None, - name: identity.into(), - dtype: FieldDataType::Plain(DataType::Utf8), - nullable: false, - }); - assert!(validate(schema.clone(), "m").is_ok()); - assert_eq!( - validate(schema.clone(), "n"), - Err(ExecutionDataStateError::InvalidMaintenanceBinary), - "different selectors may cover different series" - ); - for mutation in 0..4 { - let mut invalid = schema.clone(); - match mutation { - 0 => invalid.fields[2].nullable = true, - 1 => invalid.fields[2].dtype = FieldDataType::Plain(DataType::Timestamp), - 2 => invalid.fields.push(invalid.fields[2].clone()), - _ => invalid.fields[2].name = "label".into(), - } - assert_eq!( - validate(invalid, "m"), - Err(ExecutionDataStateError::InvalidMaintenanceBinary) - ); - } - } - - #[test] - fn exact_accumulator_state_may_feed_another_summary_agg() { - let inner = agg( - keep(), - FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - ); - let root = estimate(agg(inner, kll())); - assert!(validate_execution_data_states(&root).is_ok()); - } - - #[test] - fn readout_can_feed_summary_construction_at_query_time() { - let inner = estimate(agg(keep(), kll())); - let summary = agg(inner, kll()); - let root = estimate(summary.clone()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&summary).unwrap().timing, - ExecutionTiming::QueryTime - ); - } - - #[test] - fn query_time_operation_over_readout_is_legal_and_root_is_readout() { - let inner = estimate(agg(keep(), kll())); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::QUERY_ROWS) - ); - } - - #[test] - fn non_exact_operator_uses_the_same_read_domain_contract() { - let inner = estimate(agg(keep(), kll())); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Extension { - name: "approximate_calibration".into(), - }, - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["calibrated"]), - guarantee: None, - }); - - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&root), - Some(ExecutionDataState::QUERY_ROWS) - ); - } - - #[test] - fn query_time_values_can_feed_query_time_summary_construction() { - let inner = estimate(agg(keep(), kll())); - let post = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = agg(post, kll()); - let root = estimate(root); - validate_execution_data_states(&root).unwrap(); - } - - #[test] - fn function_under_summary_agg_is_legal_but_not_at_root() { - let operation = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: keep(), - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::IngestionTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - assert_eq!( - validate_execution_data_states(&operation).err(), - Some(ExecutionDataStateError::MaintenanceRowsAtRoot) - ); - let root = estimate(agg(Rc::clone(&operation), kll())); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&operation), - Some(ExecutionDataState::INGESTION_ROWS) - ); - } - - #[test] - fn function_over_readout_is_rejected() { - let inner = estimate(agg(keep(), kll())); - let operation = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: inner, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::IngestionTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = agg(operation, kll()); - assert!(matches!( - validate_execution_data_states(&root), - Err(ExecutionDataStateError::IllegalChildDataState { - edge: "ValueOperation.child", - child: ExecutionDataState::QUERY_ROWS - }) - )); - } - #[test] fn execution_phase_wire_names_are_ingestion_and_query_time() { for (phase, name) in [ @@ -1206,154 +198,4 @@ mod tests { assert!(serde_json::from_str::("\"maintenance_time\"").is_err()); assert!(serde_json::from_str::("\"MaintenanceTime\"").is_err()); } - - #[test] - fn summary_merge_runs_at_ingestion_or_query_time() { - for timing in [ExecutionTiming::IngestionTime, ExecutionTiming::QueryTime] { - let input = agg(keep(), kll()); - let merged = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![input.clone()], - timing, - }, - schema: input.schema.clone(), - guarantee: None, - }); - let root = estimate(merged.clone()); - let assignment = validate_execution_data_states(&root).unwrap(); - assert_eq!( - assignment.data_state_of(&merged), - Some(ExecutionDataState { - timing, - primitive: DataPrimitive::SummaryState, - }) - ); - let exported = crate::post_asap::compile_post_asap_dag(&root).unwrap(); - assert!(exported.nodes.iter().any(|node| matches!(node.payload, - crate::post_asap::PostAsapOperatorPayload::SummaryMerge - if node.output_state.timing == timing))); - } - } - - #[test] - fn ingestion_merge_cannot_depend_on_query_execution() { - let input = agg(keep(), kll()); - let query_merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![input.clone()], - timing: ExecutionTiming::QueryTime, - }, - schema: input.schema.clone(), - guarantee: None, - }); - let ingestion_merge = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - children: vec![query_merge], - timing: ExecutionTiming::IngestionTime, - }, - schema: input.schema.clone(), - guarantee: None, - }); - assert!(validate_execution_data_states(&ingestion_merge).is_err()); - } - - #[test] - fn a_shared_keep_pre_asap_reached_in_two_domains_is_ambiguous() { - // One raw sub-DAG used both as update input (under a SummaryAgg) and - // as a query-time fallback (under an ExactRead) — no single - // execution can serve both, so the plan is rejected. - let shared = keep(); - let maintained = estimate(agg(Rc::clone(&shared), kll())); - let post_over_raw = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: Rc::clone(&shared), - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::SummaryMerge { - timing: ExecutionTiming::IngestionTime, - children: vec![ - Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: maintained, - operation: ValueOperation::Exact(max_op()), - timing: ExecutionTiming::QueryTime, - }, - schema: plain(&["max"]), - guarantee: None, - }), - post_over_raw, - ], - }, - schema: plain(&["max"]), - guarantee: None, - }); - // SummaryMerge only accepts state, so this fails earlier for a - // different reason; probe the ambiguity through a direct visit. - let mut assignment = ExecutionDataStateAssignment::default(); - visit(&shared, ExecutionDataState::INGESTION_ROWS, &mut assignment).unwrap(); - assert_eq!( - visit(&shared, ExecutionDataState::QUERY_ROWS, &mut assignment), - Err(ExecutionDataStateError::AmbiguousKeepPreAsap { - first: ExecutionDataState::INGESTION_ROWS, - second: ExecutionDataState::QUERY_ROWS, - }) - ); - assert!(validate_execution_data_states(&root).is_err()); - } - - // Both paired operands must be plain; an unrelated state column is not an input. - #[test] - fn pearson_corr_checks_both_operand_states() { - let operation = ValueOperation::Exact(ExactOperation::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::PearsonCorr { left: 0, right: 1 }], - output_names: vec![], - having: None, - filters: vec![], - }); - for operand in [0, 1] { - let mut input = plain(&["x", "y", "unused"]); - input.fields[operand].dtype = kll(); - assert!(matches!( - check_plain_operands(&operation, &input), - Err(ExecutionDataStateError::NonPlainOperand { .. }) - )); - } - let mut input = plain(&["x", "y", "unused"]); - input.fields[2].dtype = kll(); - check_plain_operands(&operation, &input).unwrap(); - } - - #[test] - fn exact_operator_schema_matches_pre_asap_aggregate_derivation() { - let child_schema = lift_plain(&scan().output_schema().unwrap()); - let op = ExactOperation::Aggregate { - reduction: Reduction::by(vec![2]), - measures: vec![AggIntent::Max { col: None }], - output_names: vec![], - having: None, - filters: vec![], - }; - let out = exact_operation_output_schema(&op, &child_schema).unwrap(); - let names: Vec<_> = out.fields.iter().map(|f| f.name.as_str()).collect(); - assert_eq!(names, vec!["zone", "max"]); - assert!(out - .fields - .iter() - .all(|f| matches!(f.dtype, FieldDataType::Plain(_)))); - } - - #[test] - fn exact_operator_rejects_non_plain_input() { - let state = agg(keep(), kll()); - assert!(matches!( - exact_operation_output_schema(&max_op(), &state.schema), - Err(ExactOperationSchemaError::NonPlainInput) - )); - } } diff --git a/crates/types/src/post_asap/expr.rs b/crates/types/src/post_asap/expr.rs deleted file mode 100644 index 732a09db5..000000000 --- a/crates/types/src/post_asap/expr.rs +++ /dev/null @@ -1,303 +0,0 @@ -use super::ExecutionTiming; -use std::rc::Rc; - -use super::guarantee::ResultGuarantee; -use super::sketch::{GroupingStrategy, SketchStatistic, SummaryUpdate}; -use crate::pre_asap::agg_intent::AggIntent; -use crate::pre_asap::query_expr::Predicate; -use crate::pre_asap::schema::{FieldDataType, Schema}; -use crate::pre_asap::{ - BinaryOpKind, ColumnRef, GroupKeys, JoinKind, ProjectItem, QueryExpr, Reduction, SortKey, - VectorMatch, -}; - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[non_exhaustive] -pub enum ExactOperation { - Aggregate { - reduction: Reduction, - measures: Vec, - output_names: Vec, - /// Per-measure row predicates parallel to `measures`, positional - /// against the child's output rows — the same contract as - /// `QueryExpr::Aggregate.filters` (issue #466). - #[serde(default)] - filters: Vec>, - having: Option, - }, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[non_exhaustive] -pub enum ValueOperation { - /// Maintain the full declared population, including membership changes, - /// so removing a TopK member can promote another. - MaintainPopulation { - population: super::maintained_population::MaintainedPopulation, - }, - /// Read an aggregate or TopK prefix from the maintained population. - ReadPopulation { - readout: super::maintained_population::PopulationStatistic, - }, - Exact(ExactOperation), - /// Read an exact accumulator's state as its finalized scalar value. - /// - /// Exact accumulators do not need an estimator, but the explicit node - /// marks the maintenance-to-read boundary before query-time operators - /// such as PromQL binary arithmetic, sorting, and limiting. - FinalizeExactAccumulator, - /// Query-time column projection. SQL lowering retains the SELECT list as - /// a `Project` above its aggregate, so the post-ASAP DAG must preserve - /// its expressions, aliases, and optional derived-table qualifier while - /// allowing the aggregate child to be planned independently. - Project { - cols: Vec, - qualifier: Option, - }, - /// Query-time row filtering. The predicate remains positional against - /// the child's output schema and is evaluated only after any summary - /// state below it has been read out to rows. - Filter { - pred: Predicate, - }, - /// Query-time ordering of the child's value rows. This is deliberately - /// distinct from frequency-sketch heavy-hitter readout: PromQL `topk` - /// ranks the values produced by its child at the evaluation timestamp. - Sort { - keys: Vec, - partition_by: GroupKeys, - }, - /// Query-time row selection, normally composed over [`Self::Sort`] for - /// PromQL `topk`/`bottomk` and SQL `ORDER BY … LIMIT`. - Limit { - n: usize, - offset: usize, - /// Apply the offset and limit independently to each group. - partition_by: GroupKeys, - }, - Extension { - name: String, - }, -} - -/// Whether a candidate-membership sidecar is proven to contain every true -/// top-k key or is an explicitly approximate optimization. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum CandidateCompleteness { - Certified { guarantee: ResultGuarantee }, - BestEffort { guarantee: Option }, -} - -// ── Post-ASAP DAG node ─────────────────────────────────────────────────────── - -/// A node in the post-ASAP DAG: wraps the expression and its derived output -/// schema so every edge carries a typed schema. `Schema` may contain -/// summary-state-typed columns (`FieldDataType`'s non-`Plain` variants); -/// the pre-ASAP `Schema` cannot. -#[derive(Debug, Clone, PartialEq)] -pub struct SummaryNode { - pub expr: SummaryExpr, - /// Output schema of `expr` — the schema of the data flowing on the edge - /// leading *from* this node to its parent(s). - pub schema: Schema, - /// The machine-readable accuracy guarantee of the *value* this node - /// produces (issue #172) — `Some` on every finalized, caller-visible - /// value: a `SummaryEstimate` readout, an `ExactAggregate`-family - /// `SummaryAgg` (its state *is* the value), or a `KeepPreAsap` sub-DAG - /// (executed exactly). `None` on raw summary state — a sketch-family - /// `SummaryAgg`, `SummaryMerge`, `SummarySubtract`, `SummaryDelete`, - /// `SummaryJoin` — whose guarantee only exists once something reads it - /// out; and `None` on a readout of a family the plugged-in - /// `AccuracyModel` has no local guarantee for (`Sample`/`Wavelet`/ - /// `StatModel`), which a fail-closed consumer must treat as "unknown", - /// never as exact. - pub guarantee: Option, -} - -// ── Post-ASAP sketch-bound IR ──────────────────────────────────────────────── - -/// Sketch-bound IR produced by post-ASAP binding and final selection. Binding -/// rules selectively replace logical aggregates and joins in the pre-ASAP -/// `QueryExpr` with summary-bound counterparts. Final selection can retain -/// supported read-time value operations around independently planned children; -/// other unsupported sub-DAGs pass through as `KeepPreAsap(Rc)`. -/// -/// Traversing from the root node yields a DAG; shared sub-expressions appear -/// as multiple `Rc` references to the same `SummaryNode`. -#[derive(Debug, Clone, PartialEq)] -pub enum SummaryExpr { - /// A pre-ASAP sub-DAG kept as-is because it has no selected implementation - /// or supported residual decomposition. Output schema is the inner node's - /// schema, lifted to `Schema` with all fields as - /// `FieldDataType::Plain`. - KeepPreAsap(Rc), - - /// A PromQL binary operation whose operands were planned independently. - /// This keeps realizable summary/readout leaves visible instead of - /// hiding the complete expression inside `KeepPreAsap`. - BinaryOp { - timing: ExecutionTiming, - lhs: Rc, - rhs: Rc, - operator: BinaryOperator, - }, - - /// Plain-row semantics composed with a post-ASAP child. Timing is an - /// independent physical choice, not part of the operation's identity. - ValueOperation { - child: Rc, - operation: ValueOperation, - timing: super::execution_data_state::ExecutionTiming, - }, - - /// Read-time relational join over two row-producing children. This is - /// distinct from [`SummaryJoin`](Self::SummaryJoin), which combines - /// summary states for join estimation during maintenance. - RelationalJoin { - left: Rc, - right: Rc, - kind: JoinKind, - pred: Predicate, - /// Optional proof for candidate pruning; ranking remains a separate operation. - pruning: Option, - }, - - /// Summary aggregation. Post-ASAP binding chose `family` — which - /// summary family (exact accumulator, sketch, sample, wavelet, or - /// statistical model) and its `(kind, params)` — from the catalog for - /// `AggIntent` under `DeploymentConstraints`. - /// Output schema: grouping columns (verbatim) + one field carrying - /// partial summary state per group, typed `family`. - SummaryAgg { - child: Rc, - /// Which summary family realizes this aggregation, and that - /// family's own `(kind, params)`. Never `FieldDataType::Plain` - /// — this node always produces summary state, not a plain value. - family: FieldDataType, - /// Optional multidimensional item identity and the observation/update - /// weight fed into each state update. Subpopulation semantics remain - /// on `reduction`; physical sharing remains on `grouping`. - input: SummaryUpdate, - /// How this aggregation's output rows relate to `child`'s — the - /// same [`Reduction`] the pre-ASAP `Aggregate` node it was bound - /// from carried (issue #165), reused verbatim rather than - /// flattened to a bare `Vec`. `Reduction::Reduce(by)` - /// with an empty `by` is a genuine full reduction (merge every - /// candidate into one group); `Reduction::PerEntity` has no - /// grouping concept at all (never merge across entities) — the - /// two collapsed to the same ambiguous `by: []` before this field - /// existed (issue #163). - reduction: Reduction, - /// How this aggregation's summary state is physically instantiated - /// across `reduction`'s subpopulations — one independent instance - /// per `by` key (today's only behavior, and this field's default), - /// or one shared Hydra-family structure serving all of them (issue - /// #256). Lives here, next to `reduction`, for planning, and is also - /// encoded in sketch-valued `family`/output-schema state so merges - /// can reject incompatible layouts. `reduction` is the field that - /// carries the `by` keys this axis's legality depends on (a - /// `SharedMultiSubpopulation` choice only makes sense when - /// `reduction` actually has a subpopulation concept — see - /// `asap_aware_mapping::grouping`'s module docs for the legality - /// rules). Every existing producer of a `SummaryAgg` sets this to - /// `GroupingStrategy::PerSubpopulationInstance` (its `Default`), - /// so no existing behavior changes. - grouping: GroupingStrategy, - /// Row predicate gating this summary's updates (issue #466): only - /// rows where it is `TRUE` update the state; grouping keys are - /// still read from every row. Positional against `child`'s output. - /// A field rather than a `Filter` child so summaries that differ - /// only in predicate can still share one child. No binding rule - /// sets it yet — a filtered pre-ASAP measure stays `KeepPreAsap` — - /// so every producer today writes `None`. - filter: Option, - }, - - /// Summary-aware join (KMV / theta for join-cardinality; join-sample for - /// sampling). Emitted only when a `Bind*OnJoin` rule fires. - /// Output schema: one field typed `family`, read by a downstream - /// `SummaryEstimate`. - SummaryJoin { - outer: Rc, - inner: Rc, - key: ColumnRef, - /// Never `FieldDataType::Plain` — see [`SummaryAgg::family`](SummaryExpr::SummaryAgg). - family: FieldDataType, - }, - - /// Subtract one summary from another. Valid only for families with a - /// linear-inverse property (CMS, theta, count-based). Catalog flag - /// `subtractable` must be true for the family. - /// Output schema: one field (same family + params as inputs). - SummarySubtract { - left: Rc, - right: Rc, - }, - - /// Delete a key from a summary (CMS update with −1, deletable Bloom - /// filter). Catalog flag `deletable` must be true. Output schema = - /// input schema unchanged in type (same field type as input). - SummaryDelete { - summary_input: Rc, - key: ColumnRef, - }, - - /// Read out a query result from a built summary. The summary-state field - /// type does *not* propagate downstream of an estimate — the output - /// schema is a regular row-shaped schema (Float64 for quantile, Int64 - /// for count/cardinality, `[(key, count)]` for top-k). - SummaryEstimate { - summary_input: Rc, - query: SketchStatistic, - }, - - /// ⊕ — union of summaries across stages / shards. Distinct from the - /// pre-ASAP `Concat` because summary union has type constraints: all - /// inputs must agree on `family` (kind + params) and the catalog flag - /// `mergeable` must be true. Inserted by a deployment's own stage - /// allocator (not modeled in this crate) on cut edges. - /// Output schema: one field (same family + params as inputs). - SummaryMerge { - children: Vec>, - timing: ExecutionTiming, - }, -} - -/// All semantics owned by a post-ASAP binary operator. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct BinaryOperator { - /// Execute division only for finite operands, a nonzero divisor, and a - /// normal finite result; otherwise use exact execution. Required by the - /// relative-value division certificate, including floating-point range. - #[serde(default)] - pub checked_relative_division: bool, - /// Conditional exact rewrites (such as temporal average from sum/count) - /// require finite operands and quotient. Zero/subnormal results are valid; - /// overflow must fall back to the original query rather than emit infinity. - #[serde(default)] - pub checked_finite_division: bool, - pub kind: BinaryOpKind, - /// `None` is the only currently supported vector/vector matching mode. - /// The field is retained so execution never has to recover semantics by - /// re-parsing PromQL. - pub vector_match: Option, -} - -impl BinaryOperator { - pub fn from_logical(operator: &crate::ir::BinaryOperator, return_bool: bool) -> Self { - use crate::pre_asap::BinaryOpKind as L; - Self { - kind: match &operator.kind { - L::Arithmetic(op) => BinaryOpKind::Arithmetic(op.clone()), - L::Compare(op) if return_bool => BinaryOpKind::CompareBool(op.clone()), - L::Compare(op) => BinaryOpKind::Compare(op.clone()), - L::CompareBool(op) => BinaryOpKind::CompareBool(op.clone()), - L::Set(op) => BinaryOpKind::Set(op.clone()), - }, - vector_match: operator.vector_match.clone(), - checked_relative_division: operator.checked_relative_division, - checked_finite_division: operator.checked_finite_division, - } - } -} diff --git a/crates/types/src/post_asap/maintained_population.rs b/crates/types/src/post_asap/maintained_population.rs index 577544bd7..5e64b5436 100644 --- a/crates/types/src/post_asap/maintained_population.rs +++ b/crates/types/src/post_asap/maintained_population.rs @@ -1,4 +1,4 @@ -//! Language-independent maintained populations and their readouts. +//! Language-independent maintained populations and their evaluations. //! Resource limits, ingestion placement and data structures belong to the executor. use serde::{Deserialize, Serialize}; @@ -35,82 +35,6 @@ pub enum PopulationStatistic { Average, } -impl CurrentSeriesInput { - /// Verify the named contract against the canonical maintenance input. - pub fn matches_input(&self, input: &crate::pre_asap::QueryExpr) -> bool { - use crate::pre_asap::{CompareOpKind, DataType, QueryExpr, ScalarValue, Source}; - // PromQL instant selectors carry an ingestion-interval `TimeRange` as - // their input scope. The population must use the same expiry horizon; - // shifted and otherwise transformed inputs still fail below. - let input = match input { - QueryExpr::TimeRange { range, child } - if self.lookback_ms > 0 - && *range == std::time::Duration::from_millis(self.lookback_ms) => - { - child.as_ref() - } - QueryExpr::TimeRange { .. } => return false, - other if self.lookback_ms == 300_000 => other, - _ => return false, - }; - let QueryExpr::Scan { - source: Source::TimeSeries { metric }, - predicates, - schema, - } = input - else { - return false; - }; - if self.metric.is_empty() - || *metric != self.metric - || (schema.closed && !schema.has_promql_series_identity()) - || schema.time_index.is_none() - { - return false; - } - if self.grouping.iter().any(|label| { - !schema - .fields - .iter() - .any(|c| c.name == *label && c.dtype == DataType::Utf8) - }) { - return false; - } - let mut matchers = Vec::new(); - for predicate in predicates { - let QueryExpr::Compare { left, op, right } = predicate.0.as_ref() else { - return false; - }; - let (QueryExpr::Column(col), QueryExpr::Literal(ScalarValue::Utf8(value))) = - (left.as_ref(), right.as_ref()) - else { - return false; - }; - let Some(column) = schema.fields.get(*col) else { - return false; - }; - if column.dtype != DataType::Utf8 { - return false; - } - let operation = match op { - CompareOpKind::Eq => CurrentSeriesMatch::Equal, - CompareOpKind::Ne => CurrentSeriesMatch::NotEqual, - CompareOpKind::Regex => CurrentSeriesMatch::Regex, - CompareOpKind::NotRegex => CurrentSeriesMatch::NotRegex, - _ => return false, - }; - matchers.push(CurrentSeriesMatcher { - label: column.name.clone(), - value: value.clone(), - operation, - }); - } - matchers.sort(); - matchers.dedup(); - self.matchers == matchers && self.grouping.windows(2).all(|w| w[0] < w[1]) - } -} - /// Membership is part of state identity. Table rows must never acquire implicit /// latest-per-series selection, stale markers, or a PromQL lookback. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -130,35 +54,6 @@ pub struct MaintainedPopulation { pub quantiles: bool, } -impl MaintainedPopulation { - pub fn matches_input(&self, input: &crate::pre_asap::QueryExpr) -> bool { - match &self.input { - PopulationInput::CurrentSeries(spec) => spec.matches_input(input), - PopulationInput::Rows { - input: expected, - value_column, - grouping, - } => { - use crate::pre_asap::{DataType, QueryExpr, Source}; - expected.as_ref() == input - && matches!(input, QueryExpr::Scan { source: Source::Table { .. }, schema, .. } - if schema.closed && schema.fields.get(*value_column).is_some_and(|c| c.dtype == DataType::Float64 && !c.nullable) - && !grouping.is_without() && grouping.keys().iter().all(|k| *k < schema.fields.len())) - } - } - } - - pub fn supports(&self, readout: &PopulationStatistic) -> bool { - match readout { - PopulationStatistic::Quantile { q } => self.quantiles && q.is_finite(), - PopulationStatistic::TopK { k } => *k <= self.max_k, - PopulationStatistic::Sum - | PopulationStatistic::Count - | PopulationStatistic::Average => true, - } - } -} - impl CurrentSeriesInput { /// Verify the named contract against the canonical maintenance input. pub fn matches_node(&self, input: &crate::ir::OperatorNode) -> bool { diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index b1a7da67a..e282fccad 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -1,39 +1,31 @@ -//! The post-ASAP IR: summary-bound types, distinct from -//! [`crate::pre_asap`]'s pre-ASAP IR. +//! Summary-state types and the execution-timing vocabulary of the operator +//! IR ([`crate::ir`]). //! -//! Where [`crate::pre_asap`] carries *intent* only ("compute a -//! quantile to ε accuracy"), this module is the summary-bound IR: the -//! summary family, kind/algorithm, and parameters are committed (one -//! `(Kind, Params)` pair per family — [`sketch::ExactKind`]/[`sketch::ExactParams`], +//! Where an intent ([`crate::pre_asap::AggIntent`]) says *what* to compute +//! ("a quantile to ε accuracy"), these types say *how* a summary realizes it: +//! the family, kind/algorithm and parameters are committed — one +//! `(Kind, Params)` pair per family ([`sketch::ExactKind`]/[`sketch::ExactParams`], //! [`sketch::SamplingKind`]/[`sketch::SamplingParams`], //! [`sketch::WaveletKind`]/[`sketch::WaveletParams`], -//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]), and -//! [`expr::SummaryNode`] / [`expr::SummaryExpr`] describe the summary -//! computation. The `Sketch` family is the one exception to that -//! one-pair-per-family shape: it nests a third level, [`sketch::SketchKind`] -//! (quantile/cardinality/frequency/top-k), which itself carries the -//! committed [`sketch::SketchAlgorithm`] and [`sketch::SketchParams`] — -//! `FieldDataType::Sketch(SketchKind, GroupingStrategy)`, not a flat -//! `(kind, params)` pair -//! — because `Sketch` is the one family with more than one algorithm per -//! purpose today; no other family needs that extra level yet. +//! [`sketch::StatModelKind`]/[`sketch::StatModelParams`]). The `Sketch` family +//! nests a third level, [`sketch::SketchKind`] (quantile/cardinality/ +//! frequency/top-k), carrying the committed [`sketch::SketchAlgorithm`] and +//! [`sketch::SketchParams`], because it is the one family with more than one +//! algorithm per purpose. //! -//! A second, orthogonal axis lives here too: [`sketch::GroupingStrategy`] -//! (issue #256) — *how many* physical instances of a chosen family/kind -//! exist across a grouped aggregate's `by` subpopulations -//! (`PerSubpopulationInstance`, today's only behavior, vs. -//! `SharedMultiSubpopulation`/Hydra — see [`sketch::HydraKind`]/ -//! [`sketch::HydraParams`]), carried on [`expr::SummaryExpr::SummaryAgg`] -//! alongside `reduction` and on sketch-valued edge types -//! — see `asap_aware_mapping::grouping`'s module docs for why. +//! [`sketch::GroupingStrategy`] is a second, orthogonal axis: how many +//! physical instances of a summary exist across a grouped aggregate's `by` +//! subpopulations (per-subpopulation vs. one shared Hydra instance — see +//! `asap_aware_mapping::grouping`). It rides on `ASAPOp::SummaryAgg` and on +//! sketch-valued edge types. +//! +//! The rest: accuracy guarantees ([`guarantee`]), maintained populations, +//! summary windows and maintenance lifecycle, and the execution timing / +//! data-state vocabulary ([`execution_data_state`]). -// Legacy summary IR: no longer re-exported; removed by the cleanup PR. -pub mod cse; pub mod execution_data_state; -pub mod expr; pub mod guarantee; pub mod maintained_population; -pub mod post_asap_dag; pub mod query_time; pub mod sketch; pub mod summary_maintenance; @@ -45,31 +37,10 @@ pub use execution_data_state::{ lift_plain, DataPrimitive, ExactOperationSchemaError, ExecutionDataState, ExecutionDataStateError, ExecutionTiming, }; -// Legacy summary IR names, kept for the legacy modules above only. -#[allow(unused_imports)] -pub(crate) use cse::share_common_summary_sub_dags; -#[allow(unused_imports)] -pub(crate) use execution_data_state::{ - assigned_child_data_state, exact_operation_output_schema, produced_data_state, - validate_execution_data_states, validate_execution_data_states_at, - ExecutionDataStateAssignment, -}; -#[allow(unused_imports)] -pub(crate) use expr::{ - BinaryOperator, CandidateCompleteness, ExactOperation, SummaryExpr, SummaryNode, ValueOperation, -}; pub use guarantee::{ AccuracyError, BoundExpr, CompositionOperator, ErrorMetric, GuaranteeSource, ProbabilityExpr, ResultGuarantee, }; -#[allow(unused_imports)] -pub(crate) use post_asap_dag::{ - compile_post_asap_dag, compile_post_asap_dag_with_node_ids, EdgeRole, - GroupingEdgeCompatibility, PostAsapDAG, PostAsapDAGCompilation, PostAsapDAGDocument, - PostAsapDAGEdge, PostAsapDAGNode, PostAsapDAGValidationError, PostAsapNodeId, - PostAsapNodeIdentityMap, PostAsapOperatorPayload, WindowEdgeCompatibility, - POST_ASAP_DAG_WIRE_VERSION, -}; pub use query_time::{ classic_cms_sizing, cms_posterior_error_bound, count_sketch_posterior_error_bound, cu_sketch_posterior_error_bound, traditional_a_priori_bound, diff --git a/crates/types/src/post_asap/post_asap_dag.rs b/crates/types/src/post_asap/post_asap_dag.rs deleted file mode 100644 index d489c055e..000000000 --- a/crates/types/src/post_asap/post_asap_dag.rs +++ /dev/null @@ -1,867 +0,0 @@ -//! Runtime-neutral post-ASAP DAG contract shared by precompute and query engines. - -use std::collections::HashMap; -use std::rc::Rc; - -use super::{ - validate_execution_data_states, ExecutionDataState, ExecutionDataStateError, ResultGuarantee, - Schema, SummaryExpr, SummaryNode, -}; -use super::{ - BinaryOperator, CandidateCompleteness, ExecutionTiming, FieldDataType, GroupingStrategy, - SketchStatistic, SummaryUpdate, ValueOperation, -}; -use crate::pre_asap::{ColumnRef, JoinKind, Predicate, QueryExpr, Reduction}; -use thiserror::Error; - -pub const POST_ASAP_DAG_WIRE_VERSION: u32 = 6; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum EdgeRole { - Input, - Left, - Right, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum GroupingEdgeCompatibility { - Identical, - ConsumerCoarsensProducer, - Incompatible, - NotApplicable, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub enum WindowEdgeCompatibility { - /// Physical lowering must prove equal pane/query phase or install an - /// exact boundary residual. The logical DAG alone cannot make that claim. - #[serde(rename = "RequiresAlignedPanePhaseOrExactBoundaryResidual")] - RequiresAlignedPanePhaseOrExactWindowEdgeResidual, - NotApplicable, -} - -/// Stable identity of a node within one exported post-ASAP semantic DAG. -#[derive( - Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, serde::Serialize, serde::Deserialize, -)] -#[serde(transparent)] -pub struct PostAsapNodeId(pub u32); - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] -pub enum PostAsapOperatorPayload { - Fallback { - expression: QueryExpr, - }, - Binary { - operator: BinaryOperator, - }, - Value { - operation: ValueOperation, - }, - RelationalJoin { - join_kind: JoinKind, - pred: Predicate, - pruning: Option, - }, - SummaryAgg { - family: FieldDataType, - input: SummaryUpdate, - reduction: Reduction, - grouping: GroupingStrategy, - /// See `SummaryExpr::SummaryAgg::filter`. Wire version 6 added it; - /// a version-5 reader would otherwise take a filtered summary as - /// unfiltered. - filter: Option, - }, - SummaryJoin { - key: ColumnRef, - family: FieldDataType, - }, - SummarySubtract, - SummaryDelete { - key: ColumnRef, - }, - SummaryEstimate { - query: SketchStatistic, - }, - SummaryMerge, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGNode { - pub id: PostAsapNodeId, - /// The payload variant is the sole operator identity (`payload.kind` in JSON). - pub payload: PostAsapOperatorPayload, - /// Phase is a placement choice for every operator, independent of payload kind. - pub output_state: ExecutionDataState, - pub output_schema: Schema, - pub guarantee: Option, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGEdge { - pub producer: PostAsapNodeId, - pub consumer: PostAsapNodeId, - pub role: EdgeRole, - pub intermediate_schema: Schema, - pub data_state: ExecutionDataState, - pub grouping: GroupingEdgeCompatibility, - pub window: WindowEdgeCompatibility, -} - -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAG { - pub nodes: Vec, - pub edges: Vec, - /// Semantic workload root. Physical query/precompute sinks are selected - /// downstream by the control plane. - pub root: PostAsapNodeId, -} - -/// Versioned transport envelope for a post-ASAP semantic DAG. -/// -/// Process boundaries exchange this envelope and call [`Self::validate`]. -#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct PostAsapDAGDocument { - pub schema_version: u32, - pub dag: PostAsapDAG, -} - -#[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum PostAsapDAGValidationError { - #[error("phase assignment must name every DAG node exactly once")] - IncompletePhaseAssignment, - #[error("ingestion node {consumer:?} depends on query node {producer:?}")] - QueryDependencyInIngestion { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("unsupported post-ASAP DAG schema version {0}")] - UnsupportedVersion(u32), - #[error("duplicate post-ASAP node id {0:?}")] - DuplicateNodeId(PostAsapNodeId), - #[error("post-ASAP DAG root {0:?} does not name a node")] - MissingRoot(PostAsapNodeId), - #[error("edge endpoint {0:?} does not name a node")] - MissingEdgeEndpoint(PostAsapNodeId), - #[error("edge {producer:?}->{consumer:?} schema differs from producer output")] - EdgeSchemaMismatch { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("edge {producer:?}->{consumer:?} data state differs from producer output")] - EdgeDataStateMismatch { - producer: PostAsapNodeId, - consumer: PostAsapNodeId, - }, - #[error("post-ASAP DAG contains a cycle")] - Cycle, - #[error("post-ASAP node {0:?} is not reachable from the root")] - UnreachableNode(PostAsapNodeId), - #[error("summary aggregate node {node:?} output schema does not contain its declared family")] - SummaryFamilySchemaMismatch { node: PostAsapNodeId }, - #[error( - "summary aggregate node {node:?} declares grouping inconsistent with its sketch state" - )] - SummaryGroupingMismatch { node: PostAsapNodeId }, -} - -impl PostAsapDAGDocument { - pub fn new(dag: PostAsapDAG) -> Self { - Self { - schema_version: POST_ASAP_DAG_WIRE_VERSION, - dag, - } - } - - pub fn validate(&self) -> Result<(), PostAsapDAGValidationError> { - if self.schema_version != POST_ASAP_DAG_WIRE_VERSION { - return Err(PostAsapDAGValidationError::UnsupportedVersion( - self.schema_version, - )); - } - self.dag.validate() - } -} - -impl PostAsapDAG { - /// Assign execution phases without changing operator semantics. Phase choices - /// do not prove deployment support: callers must bind concrete implementations - /// and storage boundaries before installing this plan. - pub fn with_execution_phases( - &self, - phases: &std::collections::BTreeMap, - ) -> Result { - self.validate()?; - if phases.len() != self.nodes.len() - || self.nodes.iter().any(|node| !phases.contains_key(&node.id)) - { - return Err(PostAsapDAGValidationError::IncompletePhaseAssignment); - } - let mut dag = self.clone(); - for node in &mut dag.nodes { - node.output_state.timing = phases[&node.id]; - } - let states: HashMap<_, _> = dag.nodes.iter().map(|n| (n.id, n.output_state)).collect(); - for edge in &mut dag.edges { - edge.data_state = states[&edge.producer]; - } - dag.validate()?; - Ok(dag) - } - - pub fn validate(&self) -> Result<(), PostAsapDAGValidationError> { - use std::collections::{HashMap, HashSet}; - let mut nodes = HashMap::new(); - for node in &self.nodes { - if nodes.insert(node.id, node).is_some() { - return Err(PostAsapDAGValidationError::DuplicateNodeId(node.id)); - } - if let PostAsapOperatorPayload::SummaryAgg { - family, grouping, .. - } = &node.payload - { - let mut found_family = false; - for field in &node.output_schema.fields { - if &field.dtype == family { - found_family = true; - } - if let FieldDataType::Sketch(_, schema_grouping) = &field.dtype { - if schema_grouping != grouping { - return Err(PostAsapDAGValidationError::SummaryGroupingMismatch { - node: node.id, - }); - } - } - } - if !found_family { - return Err(PostAsapDAGValidationError::SummaryFamilySchemaMismatch { - node: node.id, - }); - } - } - } - if !nodes.contains_key(&self.root) { - return Err(PostAsapDAGValidationError::MissingRoot(self.root)); - } - let mut children: HashMap> = HashMap::new(); - for edge in &self.edges { - let producer = nodes.get(&edge.producer).ok_or( - PostAsapDAGValidationError::MissingEdgeEndpoint(edge.producer), - )?; - if !nodes.contains_key(&edge.consumer) { - return Err(PostAsapDAGValidationError::MissingEdgeEndpoint( - edge.consumer, - )); - } - if producer.output_state.timing == ExecutionTiming::QueryTime - && nodes[&edge.consumer].output_state.timing == ExecutionTiming::IngestionTime - { - return Err(PostAsapDAGValidationError::QueryDependencyInIngestion { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.intermediate_schema != producer.output_schema { - return Err(PostAsapDAGValidationError::EdgeSchemaMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - if edge.data_state != producer.output_state { - return Err(PostAsapDAGValidationError::EdgeDataStateMismatch { - producer: edge.producer, - consumer: edge.consumer, - }); - } - children - .entry(edge.consumer) - .or_default() - .push(edge.producer); - } - fn visit( - id: PostAsapNodeId, - children: &HashMap>, - visiting: &mut HashSet, - visited: &mut HashSet, - ) -> bool { - if visited.contains(&id) { - return true; - } - if !visiting.insert(id) { - return false; - } - if children - .get(&id) - .into_iter() - .flatten() - .any(|child| !visit(*child, children, visiting, visited)) - { - return false; - } - visiting.remove(&id); - visited.insert(id); - true - } - if !visit( - self.root, - &children, - &mut HashSet::new(), - &mut HashSet::new(), - ) { - return Err(PostAsapDAGValidationError::Cycle); - } - let mut reachable = HashSet::new(); - fn mark( - id: PostAsapNodeId, - children: &HashMap>, - reachable: &mut HashSet, - ) { - if !reachable.insert(id) { - return; - } - for child in children.get(&id).into_iter().flatten() { - mark(*child, children, reachable); - } - } - mark(self.root, &children, &mut reachable); - if let Some(id) = nodes.keys().find(|id| !reachable.contains(id)) { - return Err(PostAsapDAGValidationError::UnreachableNode(*id)); - } - Ok(()) - } -} - -/// Compiler-local identity assignment. It deliberately retains `Rc` handles -/// and is not serialized; deployed artifacts persist the post-ASAP node ID -/// together with their physical materialization/query IDs. -#[derive(Debug, Clone)] -pub struct PostAsapNodeIdentityMap { - nodes_by_id: Vec>, -} - -impl PostAsapNodeIdentityMap { - pub fn node_id(&self, node: &Rc) -> Option { - self.nodes_by_id - .iter() - .position(|candidate| Rc::ptr_eq(candidate, node)) - .map(|id| PostAsapNodeId(id as u32)) - } - - pub fn summary_node(&self, id: PostAsapNodeId) -> Option<&Rc> { - self.nodes_by_id.get(id.0 as usize) - } -} - -#[derive(Debug, Clone)] -pub struct PostAsapDAGCompilation { - pub dag: PostAsapDAG, - pub node_ids: PostAsapNodeIdentityMap, -} - -pub fn compile_post_asap_dag( - root: &Rc, -) -> Result { - Ok(compile_post_asap_dag_with_node_ids(root)?.dag) -} - -pub fn compile_post_asap_dag_with_node_ids( - root: &Rc, -) -> Result { - let assignment = validate_execution_data_states(root)?; - let mut nodes = Vec::new(); - let mut edges = Vec::new(); - let mut ids = HashMap::new(); - let mut nodes_by_id = Vec::new(); - - fn visit( - node: &Rc, - assignment: &super::ExecutionDataStateAssignment, - ids: &mut HashMap<*const SummaryNode, PostAsapNodeId>, - nodes: &mut Vec, - edges: &mut Vec, - nodes_by_id: &mut Vec>, - ) -> PostAsapNodeId { - if let Some(id) = ids.get(&Rc::as_ptr(node)) { - return *id; - } - let children: Vec<(&Rc, EdgeRole)> = match &node.expr { - SummaryExpr::KeepPreAsap(_) => vec![], - SummaryExpr::BinaryOp { lhs, rhs, .. } => { - vec![(lhs, EdgeRole::Left), (rhs, EdgeRole::Right)] - } - - SummaryExpr::ValueOperation { child, .. } | SummaryExpr::SummaryAgg { child, .. } => { - vec![(child, EdgeRole::Input)] - } - SummaryExpr::RelationalJoin { left, right, .. } => { - vec![(left, EdgeRole::Left), (right, EdgeRole::Right)] - } - SummaryExpr::SummaryJoin { outer, inner, .. } => { - vec![(outer, EdgeRole::Left), (inner, EdgeRole::Right)] - } - SummaryExpr::SummarySubtract { left, right } => { - vec![(left, EdgeRole::Left), (right, EdgeRole::Right)] - } - SummaryExpr::SummaryDelete { summary_input, .. } - | SummaryExpr::SummaryEstimate { summary_input, .. } => { - vec![(summary_input, EdgeRole::Input)] - } - SummaryExpr::SummaryMerge { children, .. } => { - children.iter().map(|c| (c, EdgeRole::Input)).collect() - } - }; - let child_ids: Vec<_> = children - .iter() - .map(|(c, r)| (visit(c, assignment, ids, nodes, edges, nodes_by_id), *c, *r)) - .collect(); - let id = PostAsapNodeId(nodes.len() as u32); - let state = assignment - .data_state_of(node) - .expect("validated node has state"); - let payload = match &node.expr { - SummaryExpr::KeepPreAsap(expression) => PostAsapOperatorPayload::Fallback { - expression: (**expression).clone(), - }, - SummaryExpr::BinaryOp { operator, .. } => PostAsapOperatorPayload::Binary { - operator: operator.clone(), - }, - - SummaryExpr::ValueOperation { operation, .. } => PostAsapOperatorPayload::Value { - operation: operation.clone(), - }, - SummaryExpr::RelationalJoin { - kind, - pred, - pruning, - .. - } => PostAsapOperatorPayload::RelationalJoin { - join_kind: kind.clone(), - pred: pred.clone(), - pruning: pruning.clone(), - }, - SummaryExpr::SummaryAgg { - family, - input, - reduction, - grouping, - filter, - .. - } => PostAsapOperatorPayload::SummaryAgg { - family: family.clone(), - input: input.clone(), - reduction: reduction.clone(), - grouping: grouping.clone(), - filter: filter.clone(), - }, - SummaryExpr::SummaryJoin { key, family, .. } => PostAsapOperatorPayload::SummaryJoin { - key: key.clone(), - family: family.clone(), - }, - SummaryExpr::SummarySubtract { .. } => PostAsapOperatorPayload::SummarySubtract, - SummaryExpr::SummaryDelete { key, .. } => { - PostAsapOperatorPayload::SummaryDelete { key: key.clone() } - } - SummaryExpr::SummaryEstimate { query, .. } => { - PostAsapOperatorPayload::SummaryEstimate { - query: query.clone(), - } - } - SummaryExpr::SummaryMerge { .. } => PostAsapOperatorPayload::SummaryMerge, - }; - nodes.push(PostAsapDAGNode { - id, - payload, - output_state: state, - output_schema: node.schema.clone(), - guarantee: node.guarantee.clone(), - }); - nodes_by_id.push(Rc::clone(node)); - ids.insert(Rc::as_ptr(node), id); - for (producer, child, role) in child_ids { - let maintenance_dependency = nodes[producer.0 as usize].output_state.timing - == ExecutionTiming::IngestionTime - && nodes[id.0 as usize].output_state.timing == ExecutionTiming::IngestionTime; - let grouping = match (&child.expr, &node.expr) { - ( - SummaryExpr::SummaryAgg { - reduction: producer, - .. - }, - SummaryExpr::SummaryAgg { - reduction: consumer, - .. - }, - ) if producer == consumer => GroupingEdgeCompatibility::Identical, - ( - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::PerEntity, - .. - }, - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(_), - .. - }, - ) => GroupingEdgeCompatibility::ConsumerCoarsensProducer, - ( - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(producer), - .. - }, - SummaryExpr::SummaryAgg { - reduction: crate::pre_asap::Reduction::Reduce(consumer), - .. - }, - ) if !producer.is_without() - && !consumer.is_without() - && consumer.iter().all(|key| producer.contains(key)) => - { - GroupingEdgeCompatibility::ConsumerCoarsensProducer - } - (SummaryExpr::SummaryAgg { .. }, SummaryExpr::SummaryAgg { .. }) => { - GroupingEdgeCompatibility::Incompatible - } - _ => GroupingEdgeCompatibility::NotApplicable, - }; - edges.push(PostAsapDAGEdge { - producer, - consumer: id, - role, - intermediate_schema: child.schema.clone(), - // The whole-DAG validator owns contextual state assignment, - // especially for shared KeepPreAsap leaves. Export that - // authoritative result instead of independently deriving the - // edge state a second time. - data_state: assignment - .data_state_of(child) - .expect("validated child has data state"), - grouping, - window: if maintenance_dependency { - WindowEdgeCompatibility::RequiresAlignedPanePhaseOrExactWindowEdgeResidual - } else { - WindowEdgeCompatibility::NotApplicable - }, - }); - } - id - } - - let root = visit( - root, - &assignment, - &mut ids, - &mut nodes, - &mut edges, - &mut nodes_by_id, - ); - let dag = PostAsapDAG { nodes, edges, root }; - dag.validate() - .expect("compiler emits a valid post-ASAP DAG"); - Ok(PostAsapDAGCompilation { - dag, - node_ids: PostAsapNodeIdentityMap { nodes_by_id }, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::post_asap::{ - ExactKind, ExactParams, ExecutionTiming, FieldDataType, GroupingStrategy, SummaryUpdate, - ValueOperation, - }; - use crate::pre_asap::schema::{Field, Schema}; - use crate::pre_asap::{ColumnRef, DataType, QueryExpr, Reduction, Source}; - use std::collections::BTreeMap; - - #[test] - fn every_physical_payload_can_be_assigned_either_phase() { - use crate::post_asap::DataPrimitive; - use crate::pre_asap::{ArithmeticOpKind, BinaryOpKind, JoinKind, Predicate, ScalarValue}; - let family = FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum); - let predicate = Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))); - let payloads = vec![ - PostAsapOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(1)), - }, - PostAsapOperatorPayload::Binary { - operator: BinaryOperator { - checked_relative_division: false, - checked_finite_division: false, - kind: BinaryOpKind::Arithmetic(ArithmeticOpKind::Add), - vector_match: None, - }, - }, - PostAsapOperatorPayload::Value { - operation: ValueOperation::Limit { - n: 1, - offset: 0, - partition_by: Default::default(), - }, - }, - PostAsapOperatorPayload::RelationalJoin { - join_kind: JoinKind::Semi, - pred: predicate, - pruning: None, - }, - PostAsapOperatorPayload::SummaryAgg { - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - PostAsapOperatorPayload::SummaryJoin { - key: ColumnRef::SampleValue, - family: family.clone(), - }, - PostAsapOperatorPayload::SummarySubtract, - PostAsapOperatorPayload::SummaryDelete { - key: ColumnRef::SampleValue, - }, - PostAsapOperatorPayload::SummaryEstimate { - query: SketchStatistic::Cardinality, - }, - PostAsapOperatorPayload::SummaryMerge, - ]; - for payload in payloads { - // This checks physical identity and placement, not kernel availability. - let primitive = match &payload { - PostAsapOperatorPayload::Fallback { .. } - | PostAsapOperatorPayload::Binary { .. } - | PostAsapOperatorPayload::Value { .. } - | PostAsapOperatorPayload::RelationalJoin { .. } - | PostAsapOperatorPayload::SummaryEstimate { .. } => DataPrimitive::Raw, - PostAsapOperatorPayload::SummaryAgg { .. } - | PostAsapOperatorPayload::SummaryJoin { .. } - | PostAsapOperatorPayload::SummarySubtract - | PostAsapOperatorPayload::SummaryDelete { .. } - | PostAsapOperatorPayload::SummaryMerge => DataPrimitive::SummaryState, - }; - let dag = PostAsapDAG { - root: PostAsapNodeId(0), - edges: vec![], - nodes: vec![PostAsapDAGNode { - id: PostAsapNodeId(0), - payload: payload.clone(), - output_state: ExecutionDataState { - timing: ExecutionTiming::QueryTime, - primitive, - }, - output_schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: family.clone(), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }], - }; - for phase in [ExecutionTiming::IngestionTime, ExecutionTiming::QueryTime] { - let placed = dag - .with_execution_phases(&BTreeMap::from([(dag.root, phase)])) - .unwrap(); - assert_eq!(placed.nodes[0].payload, payload); - assert_eq!(placed.nodes[0].output_state.timing, phase); - let wire = serde_json::to_value(&placed).unwrap(); - assert!(wire["nodes"][0]["payload"].get("timing").is_none()); - assert_eq!(serde_json::from_value::(wire).unwrap(), placed); - } - assert!(dag.with_execution_phases(&BTreeMap::new()).is_err()); - } - } - - #[test] - fn phase_assignment_updates_edges_and_rejects_query_dependencies_in_ingestion() { - use crate::pre_asap::ScalarValue; - let schema = Schema::lifted(vec![], None); - let nodes = [0, 1] - .into_iter() - .map(|id| PostAsapDAGNode { - id: PostAsapNodeId(id), - payload: PostAsapOperatorPayload::Fallback { - expression: QueryExpr::Literal(ScalarValue::Int64(1)), - }, - output_state: ExecutionDataState::QUERY_ROWS, - output_schema: schema.clone(), - guarantee: None, - }) - .collect(); - let dag = PostAsapDAG { - nodes, - root: PostAsapNodeId(1), - edges: vec![PostAsapDAGEdge { - producer: PostAsapNodeId(0), - consumer: PostAsapNodeId(1), - role: EdgeRole::Input, - intermediate_schema: schema, - data_state: ExecutionDataState::QUERY_ROWS, - grouping: GroupingEdgeCompatibility::NotApplicable, - window: WindowEdgeCompatibility::NotApplicable, - }], - }; - let placed = dag - .with_execution_phases(&BTreeMap::from([ - (PostAsapNodeId(0), ExecutionTiming::IngestionTime), - (PostAsapNodeId(1), ExecutionTiming::QueryTime), - ])) - .unwrap(); - assert_eq!( - placed.edges[0].data_state.timing, - ExecutionTiming::IngestionTime - ); - assert_eq!(dag.edges[0].data_state.timing, ExecutionTiming::QueryTime); - assert!(matches!( - dag.with_execution_phases(&BTreeMap::from([ - (PostAsapNodeId(0), ExecutionTiming::QueryTime), - (PostAsapNodeId(1), ExecutionTiming::IngestionTime), - ])), - Err(PostAsapDAGValidationError::QueryDependencyInIngestion { .. }) - )); - } - - #[test] - fn exports_summary_over_summary_as_typed_precompute_edges() { - let scan = Rc::new(QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::new(vec![Field::plain("value", DataType::Float64, false)]), - }); - let raw = Rc::new(SummaryNode { - expr: SummaryExpr::KeepPreAsap(scan), - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }); - let make_agg = |child: Rc, kind, params| { - let family = FieldDataType::ExactAggregate(kind, params); - Rc::new(SummaryNode { - expr: SummaryExpr::SummaryAgg { - child, - family: family.clone(), - input: SummaryUpdate::column(ColumnRef::SampleValue), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - }, - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: family, - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }) - }; - let inner = make_agg(raw, ExactKind::Sum, ExactParams::Sum); - let outer = make_agg(Rc::clone(&inner), ExactKind::Sum, ExactParams::Sum); - let root = Rc::new(SummaryNode { - expr: SummaryExpr::ValueOperation { - child: outer, - operation: ValueOperation::FinalizeExactAccumulator, - timing: ExecutionTiming::QueryTime, - }, - schema: Schema::lifted( - vec![Field { - name: "value".into(), - dtype: FieldDataType::Plain(DataType::Float64), - nullable: false, - table: None, - }], - None, - ), - guarantee: None, - }); - - let compiled = compile_post_asap_dag_with_node_ids(&root).unwrap(); - assert_eq!(compiled.node_ids.node_id(&root), Some(PostAsapNodeId(3))); - assert!(Rc::ptr_eq( - compiled.node_ids.summary_node(PostAsapNodeId(1)).unwrap(), - &inner - )); - let dag = compiled.dag; - assert_eq!(dag.root, PostAsapNodeId(3)); - assert_eq!( - dag.nodes[1].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - assert_eq!( - dag.nodes[2].output_state, - ExecutionDataState::INGESTION_SUMMARY - ); - let dependency = dag - .edges - .iter() - .find(|e| e.producer == PostAsapNodeId(1) && e.consumer == PostAsapNodeId(2)) - .unwrap(); - assert_eq!(dependency.data_state, ExecutionDataState::INGESTION_SUMMARY); - assert_eq!(dependency.grouping, GroupingEdgeCompatibility::Identical); - assert_eq!( - dependency.window, - WindowEdgeCompatibility::RequiresAlignedPanePhaseOrExactWindowEdgeResidual - ); - assert!(matches!( - dependency.intermediate_schema.fields[0].dtype, - FieldDataType::ExactAggregate(ExactKind::Sum, _) - )); - let encoded = serde_json::to_string(&dag).expect("serialize post-ASAP DAG"); - let decoded: PostAsapDAG = - serde_json::from_str(&encoded).expect("deserialize post-ASAP DAG"); - assert_eq!(decoded, dag); - let document = PostAsapDAGDocument::new(decoded); - document.validate().unwrap(); - let mut invalid = serde_json::to_value(&document).unwrap(); - invalid["dag"]["nodes"][0]["operator"] = serde_json::json!("Binary"); - assert!(serde_json::from_value::(invalid).is_err()); - assert!(document.dag.nodes.iter().all(|node| { - let wire = serde_json::to_value(node).unwrap(); - wire.get("operator").is_none() && wire["payload"]["kind"].is_string() - })); - let mut old_version = document.clone(); - old_version.schema_version = 1; - assert_eq!( - old_version.validate(), - Err(PostAsapDAGValidationError::UnsupportedVersion(1)) - ); - let mut unknown = serde_json::to_value(&document).unwrap(); - unknown["unexpected"] = serde_json::json!(true); - assert!(serde_json::from_value::(unknown).is_err()); - assert!(matches!( - dag.nodes[2].payload, - PostAsapOperatorPayload::SummaryAgg { - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - reduction: Reduction::Reduce(_), - .. - } - )); - } - - #[test] - fn post_asap_node_ids_serialize_in_deterministic_binding_order() { - let mut bindings = BTreeMap::new(); - bindings.insert(PostAsapNodeId(10), "materialization-10"); - bindings.insert(PostAsapNodeId(2), "query-2"); - assert_eq!( - serde_json::to_string(&bindings).unwrap(), - r#"{"2":"query-2","10":"materialization-10"}"# - ); - } -} diff --git a/crates/types/src/post_asap/query_time/mod.rs b/crates/types/src/post_asap/query_time/mod.rs index 44e979117..511e72f84 100644 --- a/crates/types/src/post_asap/query_time/mod.rs +++ b/crates/types/src/post_asap/query_time/mod.rs @@ -1,6 +1,6 @@ //! Code meant to run at **query execution time**, not planning time — -//! folder-separated from `post_asap`'s other modules ([`super::expr`], -//! [`super::schema`], [`super::sketch`]) on purpose. +//! folder-separated from `post_asap`'s other modules ([`super::sketch`], +//! [`super::guarantee`]) on purpose. //! //! `asap-types`' crate doc states the invariant this workspace otherwise //! holds to without exception: "no execution logic lives in this diff --git a/crates/types/src/pre_asap/agg_intent.rs b/crates/types/src/pre_asap/agg_intent.rs index c60dd55e3..1e926bd44 100644 --- a/crates/types/src/pre_asap/agg_intent.rs +++ b/crates/types/src/pre_asap/agg_intent.rs @@ -10,28 +10,28 @@ //! heavy-hitter sketch when approximate — is a post-ASAP cost-aware decision, //! not encoded here. The semantic distinction that *is* made at lowering is //! intent vs operator: a heavy-hitter aggregate becomes `TopK`, whereas a -//! generic `ORDER BY value LIMIT k` stays as the `QueryExpr::Sort + Limit` +//! generic `ORDER BY value LIMIT k` stays as the `NonASAPOp::Sort + Limit` //! operator pair. use serde::{Deserialize, Serialize}; -use crate::pre_asap::query_expr::DataModel; +use crate::ir::operator_properties::DataModel; use crate::pre_asap::schema::{ColumnId, DataType, Field, FieldDataType}; use crate::types::AccuracyTarget; /// "What to compute" — the vocabulary the planner pivots on. /// -/// Grouping for `TopK` rides on the enclosing `QueryExpr::Aggregate.by` +/// Grouping for `TopK` rides on the enclosing `NonASAPOp::Aggregate`'s `reduction` /// (positional `ColumnId`s), like every other aggregate; the intent itself /// carries only `k` + the accuracy target. /// /// The single-column reducers (`Sum` / `Min` / `Max` / `Avg` / `StdDev` / /// `Variance` / `Quantile`) carry `col: Option` — the input /// column they reduce, generic over the column-reference state the same way -/// [`QueryExpr`](super::query_expr::QueryExpr) is: positional `ColumnId` once -/// bound (the default, and every existing use of the bare `AggIntent` name), -/// or an unresolved name-based `ColumnRef` for a front end constructing this -/// intent directly, before the [`SchemaResolver`](super::schema_resolver::SchemaResolver) has run. +/// the rest of the vocabulary is: positional `ColumnId` once bound (the +/// default, and every existing use of the bare `AggIntent` name), or an +/// unresolved name-based `ColumnRef` for a front end constructing this +/// intent directly, before name resolution (`asap_frontend_common`) has run. /// `None` is the PromQL convention "the time-series sample value"; SQL /// `SUM(bytes), AVG(latency)` sets distinct `Some(_)`s so a multi-aggregate /// node binds each reducer to the right column, and `plan::bind` knows which @@ -128,7 +128,7 @@ pub enum AggIntent { // ── Time-series streaming derivatives ──────────────────────────────── // Counter-reset adjustment; not equivalent to Sum/Count over a window. - // The temporal range lives on the enclosing `QueryExpr::TimeRange` node, + // The temporal range lives on the enclosing `NonASAPOp::TimeRange` node, // not in the intent — this keeps the intent vocabulary range-agnostic. Rate, /// PromQL `irate(v[w])` — reset-aware rate from the final two samples. @@ -234,7 +234,7 @@ pub enum AggIntent { /// A time / calendar accessor (issue #46) — `timestamp`, `minute`, `hour`, /// `day_of_week`, … over each sample's timestamp (or, for the no-arg forms, /// over the evaluation time). Label-preserving per-series value transform. - /// (`time()` is the evaluation time itself — a `QueryExpr::EvalTimestamp` leaf, + /// (`time()` is the evaluation time itself — a `ScalarExpr::EvalTimestamp` leaf, /// not this.) TimeFn(TimeFunc), @@ -378,9 +378,8 @@ pub enum MathFunc { } // `requires` / `is_per_series` / `output_column` never read `col`'s value — -// only its presence via a `{ .. }` pattern — so, unlike -// `QueryExpr::output_schema` (which genuinely cannot compile for an -// unresolved DAG — see its own doc), nothing stops these from being generic +// only its presence via a `{ .. }` pattern — so, unlike schema derivation +// (which needs bound positions), nothing stops these from being generic // over every `C`. And a front end constructing `AggIntent` // directly (issue #179) does need `is_per_series` pre-binding — it decides // the `PerEntity`/`Reduce` reduction shape right at construction time (see @@ -529,7 +528,7 @@ impl AggIntent { impl AggIntent { /// Output column name + type produced by this intent over `input`. - /// Used by `QueryExpr::Aggregate`'s schema-derivation rule. The PromQL + /// Used by `NonASAPOp::Aggregate`'s schema-derivation rule. The PromQL /// convention names the column after the intent kind so consumers can /// locate it without an alias lookup. pub fn output_column(&self, input: &Field) -> Field { diff --git a/crates/types/src/pre_asap/canonicalize.rs b/crates/types/src/pre_asap/canonicalize.rs deleted file mode 100644 index b9e6a653a..000000000 --- a/crates/types/src/pre_asap/canonicalize.rs +++ /dev/null @@ -1,782 +0,0 @@ -//! Shared post-lowering canonicalization of the resolved [`QueryExpr`]. -//! -//! Both language front ends funnel through [`resolve_root`](super::resolve::resolve_root), -//! which runs this pass over the resolved DAG. Its job is to erase -//! *structural* differences between semantically identical queries so a -//! post-ASAP binding rule matching on the intent algebra sees one canonical -//! spelling regardless of source language (issue #34). -//! -//! ## Heavy-hitter promotion -//! -//! An additive-ranked "order by the aggregate, take the top k" is a -//! heavy-hitter represented by [`AggIntent::TopK`]. Front ends may -//! emit it as an ordinary `Limit { Sort { … Aggregate } }`; this pass promotes -//! that shape to the canonical -//! -//! ```text -//! Aggregate { reduction: Reduce(), measures: [TopK{k}], -//! child: Aggregate { measures: [Count | Sum], … } } -//! ``` -//! -//! Count supplies unit weights and Sum supplies value weights. Because the -//! match is positional, aliases do not affect it. Other ranked expressions -//! retain Sort + Limit. - -use std::rc::Rc; - -use super::agg_intent::{topk, AggIntent}; -use super::expr_ir::{CompareOpKind, ScalarValue}; -use super::query_expr::{Predicate, QueryExpr, Reduction, SortKey, WindowFuncKind}; -use crate::types::AccuracyTarget; - -/// Rewrite `expr` into its canonical form (bottom-up). Idempotent: a DAG that -/// is already canonical is returned unchanged. -pub fn canonicalize(mut expr: QueryExpr) -> QueryExpr { - canon(&mut expr); - expr -} - -fn canon(expr: &mut QueryExpr) { - // A `Concat` asserting a caller-proven `discriminator_unique_key` (issue - // #228) had that key's `ColumnId`s resolved, in `resolve.rs`, against - // exactly the first branch's output schema *as it stood before this - // pass ran*. `try_promote_additive_top_ranking`/`try_rewrite_rownumber_topk` - // below can restructure that branch (anywhere within it — not only at - // its own top level, since the same recursive walk can rewrite a node - // nested under a pass-through wrapper too) into a shape with a - // different output schema, which would leave those `ColumnId`s - // pointing at the wrong column, or out of bounds, of the - // post-canonicalize schema. Snapshot the schema the discriminator key - // was actually resolved against, right here, before recursing into the - // children — this is the exact DAG state `resolve.rs` saw. - let discriminator_branch_schema_before = match expr { - QueryExpr::Concat { - children, - discriminator_unique_key: Some(_), - } => children.first().and_then(|c| c.output_schema().ok()), - _ => None, - }; - - // Bottom-up: canonicalize every child before matching at this node, so an - // inner heavy-hitter is promoted before an enclosing rewrite inspects it. - for child in children_mut(expr) { - canon(child); - } - - // If the first branch's output schema moved out from under the asserted - // key, the key can no longer be trusted — drop it (never re-derive it by - // guessing at name/position: the two rewrites above don't preserve - // column identity in a way that's safe to infer). A wrong `unique_keys` - // claim is a wrong query answer, not a missed optimization — see - // `ConcatDiscriminatorKey`'s soundness doc — so this errs conservatively: - // any difference at all (not just a column-count/type change) drops the - // key, including the schema becoming undecidable in either direction. - if let QueryExpr::Concat { - children, - discriminator_unique_key: key @ Some(_), - } = expr - { - let discriminator_branch_schema_after = - children.first().and_then(|c| c.output_schema().ok()); - if discriminator_branch_schema_before != discriminator_branch_schema_after { - *key = None; - } - } - - // Local rewrites chain: a `ROW_NUMBER()`-partitioned top-k rewrites to a - // `Limit{Sort}`, which the heavy-hitter rule may then promote to an - // `Aggregate([TopK])`. Each rule strictly simplifies the node, so applying - // them to a fixpoint terminates. - while let Some(rewritten) = - try_rewrite_rownumber_topk(expr).or_else(|| try_promote_additive_top_ranking(expr)) - { - *expr = rewritten; - } -} - -/// A `&mut QueryExpr` out of a child `Rc` — clone-on-write via -/// [`Rc::make_mut`]: free (no clone) while `r` is uniquely owned, which is -/// the overwhelmingly common case (a DAG `canonicalize` was just handed by -/// value); falls back to cloning just *this* node (its own fields — the -/// grandchildren stay shared `Rc`s, not deep-copied) only when some other -/// owner still holds the same `Rc`, e.g. a caller that kept its own clone -/// around (`once.clone()` in `is_idempotent` below — `QueryExpr::clone()` is -/// now a cheap `Rc`-bump, not a deep copy, so that clone shares structure -/// with `once` until a rewrite here needs to touch it). `Rc::get_mut` would -/// panic on exactly that case; `make_mut` degrades to a shallow copy instead -/// of requiring sole ownership as a precondition. Once a workload-level CSE -/// pass runs (issue #212, #222) and canonicalize sees an already-shared -/// sub-DAG from a *different* query, this is also the mechanism that keeps -/// canonicalizing one query from silently corrupting another's view of the -/// same shared node. -fn rc_mut(r: &mut Rc) -> &mut QueryExpr { - Rc::make_mut(r) -} - -/// Mutable references to the direct **operator** `QueryExpr` children of a -/// node — `canon`'s own top-down/bottom-up walk only ever visits the -/// relational skeleton, never descending into a scalar position (`Filter.pred`, -/// `ProjectItem.expr`, …): none of the three rewrite rules rewrite anything -/// inside a scalar sub-DAG, so there's nothing to gain by recursing into one, -/// and every scalar variant (issue #205) hits the catch-all below. -fn children_mut(expr: &mut QueryExpr) -> Vec<&mut QueryExpr> { - use QueryExpr::*; - match expr { - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220), not the relational skeleton — same "no children to recurse - // into" treatment as the scalar variants below. - Scan { .. } | EvalTimestamp | CurrentTimestamp | PromqlScalarBridge(_) => vec![], - PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => vec![rc_mut(c)], - PromqlRelabel { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } - | PromqlSeriesSample { child, .. } - | PromqlInfoEnrich { child, .. } - | Sort { child, .. } - | Limit { child, .. } => vec![rc_mut(child)], - Concat { children, .. } => children.iter_mut().collect(), - Join { left, right, .. } | SetOp { left, right, .. } => { - vec![rc_mut(left), rc_mut(right)] - } - BinaryOp { lhs, rhs, .. } => vec![rc_mut(lhs), rc_mut(rhs)], - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => vec![], - } -} - -/// Recognise an additive-ranked -/// `Limit { Sort { [Project] Aggregate([Count | Sum]) } }` and rewrite it to -/// the canonical heavy-hitter `Aggregate([TopK])` over the explicit inner -/// aggregate. Returns `None` when the shape does not match. -fn try_promote_additive_top_ranking(expr: &QueryExpr) -> Option { - // Limit k, no offset (an OFFSET means "not the top k"). - let QueryExpr::Limit { - n: k, - offset: 0, - child, - } = expr - else { - return None; - }; - // A single ordering key on a column. - let QueryExpr::Sort { - keys, - partition_by, - child: sort_child, - } = child.as_ref() - else { - return None; - }; - let [SortKey { - expr: QueryExpr::Column(sort_col), - ascending, - .. - }] = keys.as_slice() - else { - return None; - }; - - // The ordered relation is an `Aggregate`, optionally behind a passthrough - // projection (a bare-column SELECT list). Map the sort key through the - // projection to the aggregate's own output column. - let (agg_expr, ranked_col) = match sort_child.as_ref() { - QueryExpr::Project { cols, child, .. } => { - let QueryExpr::Column(underlying) = &cols.get(*sort_col)?.expr else { - return None; - }; - (child.as_ref(), *underlying) - } - other => (other, *sort_col), - }; - - // Exactly one aggregate, ranked by *its* output column — the measure sits at - // index `by.len()` (after the group keys). A `PerEntity` reduction has no - // `by` to rank a measure against — this shape can't be heavy-hitter - // promoted, so it's a non-match rather than an error. - let QueryExpr::Aggregate { - reduction, - measures, - filters, - child: aggregate_child, - .. - } = agg_expr - else { - return None; - }; - let Reduction::Reduce(by) = reduction else { - return None; - }; - let [ranked_agg] = measures.as_slice() else { - return None; - }; - // A heavy-hitter sketch ranks the raw update stream; a filtered measure - // only counts part of it, and no binding rule applies the filter (#466). - if filters.iter().any(Option::is_some) { - return None; - } - if ranked_col != by.len() { - return None; - } - // The heavy-hitter decision — descending, over a measure with a realised - // heavy-hitter sketch — is the shared rule both front ends' promotions - // consult (issue #38). So an ascending additive-ranked limit - // (`ORDER BY COUNT(*) ASC LIMIT k` = bottom-k) stays generic, exactly as - // PromQL `bottomk(k, count_over_time(…))` does. - if !topk::Ranking::from_aggregate(ranked_agg).is_supported(!ascending) { - return None; - } - // A direct Sum is a stream of additive observation weights. A Sum over a - // derived child such as Rate/Increase is different: a heavy-hitter sketch - // may propose candidate membership, but PromQL still requires exact - // reset-aware/extrapolated values to rerank those candidates. The current - // post-ASAP IR has no candidate-sidecar + exact-rerank node, so keep that - // shape as Sort + Limit instead of treating a sketch estimate as final. - if matches!(ranked_agg, AggIntent::Sum { .. }) - && matches!(aggregate_child.as_ref(), QueryExpr::Aggregate { .. }) - { - return None; - } - // Count ranks unit updates; a direct Sum ranks weighted updates. - let accuracy = match ranked_agg { - AggIntent::Count { accuracy } => accuracy.clone(), - AggIntent::Sum { .. } => AccuracyTarget::Exact, - _ => unreachable!("additive ranking gate admitted a non-additive measure"), - }; - - // Outer heavy-hitter `TopK`, grouped by the ranking's partition (empty for a - // global `ORDER BY … LIMIT k`; the `by` labels for a partitioned `topk by`), - // over the unchanged inner additive aggregate. - Some(QueryExpr::Aggregate { - reduction: Reduction::by(partition_by.to_vec()), - measures: vec![AggIntent::TopK { k: *k, accuracy }], - output_names: Vec::new(), - filters: Vec::new(), - having: None, - child: Rc::new(agg_expr.clone()), - }) -} - -/// Recognise the SQL partitioned-top-k idiom — `WHERE rn <= k` over a -/// `ROW_NUMBER() OVER (PARTITION BY p ORDER BY o)` — and rewrite it to the -/// generic partitioned top-k `Limit{k} { Sort{ o, partition_by: p } }` (issue -/// #24). The count-ranked case is then promoted to a heavy-hitter `TopK` by -/// [`try_promote_additive_top_ranking`], so a SQL `ROW_NUMBER` top-k and the PromQL -/// `topk by (…)` it mirrors converge on the same canonical shape. -fn try_rewrite_rownumber_topk(expr: &QueryExpr) -> Option { - // Filter { pred: `Column(rn) <= k` }. - let QueryExpr::Filter { pred, child } = expr else { - return None; - }; - let Predicate(pred_expr) = pred; - let QueryExpr::Compare { left, op, right } = pred_expr.as_ref() else { - return None; - }; - // `rn <= k` (top-k). `rn < k` would be off-by-one; require `<=`. - if *op != CompareOpKind::Le { - return None; - } - let (QueryExpr::Column(rn_col), QueryExpr::Literal(ScalarValue::Int64(k))) = - (left.as_ref(), right.as_ref()) - else { - return None; - }; - if *k < 0 { - return None; - } - - // Optionally strip a passthrough projection (the derived table's SELECT that - // re-exposes the aggregate columns + rn), mapping the rn column through it. - let (wf_expr, rn_in_wf) = match child.as_ref() { - QueryExpr::Project { cols, child, .. } => { - let QueryExpr::Column(underlying) = &cols.get(*rn_col)?.expr else { - return None; - }; - (child.as_ref(), *underlying) - } - other => (other, *rn_col), - }; - - // The filtered column must be a `ROW_NUMBER()` window output — the single - // column the SQLWindowFunc appends after its input, i.e. the last one. - let QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - partition_by, - order_by, - child: inner, - .. - } = wf_expr - else { - return None; - }; - if order_by.is_empty() { - return None; - } - let inner_cols = inner.output_schema().ok()?.fields.len(); - if rn_in_wf != inner_cols { - return None; // the predicate ranks some other column, not the row number - } - - // Generic partitioned top-k. The window's ORDER BY keys are relative to its - // input (`inner`), so they transfer directly to a `Sort` over `inner`. - Some(QueryExpr::Limit { - n: *k as usize, - offset: 0, - child: Rc::new(QueryExpr::Sort { - keys: order_by.clone(), - partition_by: partition_by.clone(), - child: Rc::new(inner.as_ref().clone()), - }), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::query_expr::{ - GroupKeys, ProjectItem, Source, WindowFrame, WindowFrameBound, WindowFrameOffset, - WindowFrameUnits, - }; - use crate::pre_asap::schema::{DataType, Field, Schema}; - use crate::types::AccuracyTarget; - - fn scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - /// `Aggregate{ by: [1], [Count] }` over the scan — output cols `[service, count]`. - fn count_by_service() -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - } - } - - fn desc(col: usize) -> Vec { - vec![SortKey { - expr: QueryExpr::Column(col), - ascending: false, - nulls_first: false, - }] - } - - fn limit(n: usize, offset: usize, child: QueryExpr) -> QueryExpr { - QueryExpr::Limit { - n, - offset, - child: Rc::new(child), - } - } - - fn sort(keys: Vec, child: QueryExpr) -> QueryExpr { - QueryExpr::Sort { - keys, - partition_by: GroupKeys::by(vec![]), - child: Rc::new(child), - } - } - - fn is_topk_over_count(qe: &QueryExpr) -> bool { - matches!(qe, - QueryExpr::Aggregate { measures, child, .. } - if matches!(measures.as_slice(), [AggIntent::TopK { k: 5, .. }]) - && matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Count { .. }]))) - } - - #[test] - fn promotes_count_ranked_limit_sort() { - // Limit 5 { Sort DESC by count-col (1) { Aggregate[Count] by [1] } }. - let q = limit(5, 0, sort(desc(1), count_by_service())); - assert!(is_topk_over_count(&canonicalize(q))); - } - - // A heavy-hitter sketch ranks every row; a count that only counts some - // rows (#466) is not that, so the generic Sort + Limit stays. - #[test] - fn does_not_promote_a_filtered_count_ranking() { - let mut filtered = count_by_service(); - let QueryExpr::Aggregate { filters, .. } = &mut filtered else { - unreachable!() - }; - *filters = vec![Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(2)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - })))]; - let q = limit(5, 0, sort(desc(1), filtered)); - let canonical = canonicalize(q.clone()); - assert!(!is_topk_over_count(&canonical)); - assert_eq!(canonical, q); - } - - #[test] - fn promotes_through_a_passthrough_projection() { - // …with a `SELECT service, count` projection between the Sort and the Agg. - let proj = QueryExpr::Project { - cols: vec![ - ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }, - ProjectItem { - alias: Some("c".into()), - expr: QueryExpr::Column(1), - }, - ], - qualifier: None, - child: Rc::new(count_by_service()), - }; - let q = limit(5, 0, sort(desc(1), proj)); - assert!(is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn is_idempotent() { - let q = limit(5, 0, sort(desc(1), count_by_service())); - let once = canonicalize(q); - let twice = canonicalize(once.clone()); - assert_eq!(once, twice, "canonicalize must be idempotent"); - } - - // ── Concat's discriminator_unique_key vs. canonicalize (issue #228 review) ── - // - // `resolve.rs` resolves `discriminator_unique_key`'s `ColumnId`s against - // the first branch's *pre-canonicalize* output schema. If canonicalize - // then restructures that branch (heavy-hitter promotion, the - // `ROW_NUMBER()` top-k rewrite), those `ColumnId`s can end up pointing at - // the wrong column — or out of bounds — of the new schema. The two tests - // below pin the fix: the key is dropped whenever the branch's schema - // actually changed, and survives untouched otherwise. Never guessed at. - - #[test] - fn concat_discriminator_key_survives_canonicalize_when_first_branch_is_unaffected() { - // A plain `Aggregate` first branch matches neither rewrite trigger, - // so its schema is identical before and after canonicalize. - let q = QueryExpr::concat_with_discriminator( - vec![count_by_service(), count_by_service()], - /* discriminator */ 0, - /* inner_key */ vec![1], - ); - let QueryExpr::Concat { - discriminator_unique_key, - .. - } = canonicalize(q) - else { - panic!("expected Concat"); - }; - assert!( - discriminator_unique_key.is_some(), - "an untouched first branch's discriminator key must survive canonicalize" - ); - } - - #[test] - fn concat_discriminator_key_is_dropped_when_first_branch_gets_rewritten() { - // The first branch is exactly the heavy-hitter promotion trigger — - // `Limit{Sort{Aggregate([Count])}}`, with an empty (global) - // `partition_by` — so canonicalize rewrites it in place to - // `Aggregate{TopK}`, whose own output is a single column, not the - // original two (`[service, count]`). A discriminator key resolved - // against the original 2-column shape (`discriminator` = `service` - // at index 0, `inner_key` = `count` at index 1) must not silently - // survive pointing at the new 1-column schema. - let promotable_branch = limit(5, 0, sort(desc(1), count_by_service())); - let q = QueryExpr::concat_with_discriminator( - vec![promotable_branch, count_by_service()], - /* discriminator */ 0, - /* inner_key */ vec![1], - ); - let QueryExpr::Concat { - children, - discriminator_unique_key, - } = canonicalize(q) - else { - panic!("expected Concat"); - }; - assert!( - is_topk_over_count(&children[0]), - "the first branch is still promoted normally" - ); - assert!( - discriminator_unique_key.is_none(), - "a stale discriminator key must be dropped, never silently kept wrong" - ); - } - - #[test] - fn does_not_promote_ascending_sort() { - // Ascending = bottom-k: the Top-K operator's ranking rule - // rejects it (needs descending), so it stays a generic Sort+Limit — the - // same call PromQL `bottomk` makes (issue #38). - let asc = vec![SortKey { - expr: QueryExpr::Column(1), - ascending: true, - nulls_first: false, - }]; - let q = limit(5, 0, sort(asc, count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn does_not_promote_with_offset() { - let q = limit(5, 2, sort(desc(1), count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn does_not_promote_ranking_by_a_group_key() { - // DESC by col 0 (the `service` group key), not the count → not a - // frequency heavy-hitter. - let q = limit(5, 0, sort(desc(0), count_by_service())); - assert!(!is_topk_over_count(&canonicalize(q))); - } - - #[test] - fn promotes_sum_ranked_limit_sort_as_weighted_heavy_hitter() { - let sum = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let q = limit(5, 0, sort(desc(1), sum)); - let out = canonicalize(q); - let QueryExpr::Aggregate { - measures, child, .. - } = out - else { - panic!("expected weighted TopK aggregate"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Sum { .. }])) - ); - } - - #[test] - fn keeps_sum_over_counter_reduction_as_exact_value_ranking() { - for counter in [AggIntent::Rate, AggIntent::Increase] { - let derived = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![counter], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let sum = QueryExpr::Aggregate { - reduction: Reduction::by(vec![1]), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(derived), - }; - let out = canonicalize(limit(5, 0, sort(desc(1), sum))); - assert!(matches!(out, QueryExpr::Limit { child, .. } - if matches!(child.as_ref(), QueryExpr::Sort { child, .. } - if matches!(child.as_ref(), QueryExpr::Aggregate { measures, child, .. } - if matches!(measures.as_slice(), [AggIntent::Sum { .. }]) - && matches!(child.as_ref(), QueryExpr::Aggregate { .. }))))); - } - } - - // ── ROW_NUMBER() partitioned top-k (issue #24) ────────────────────────── - - /// A scan with `[ts, service, region, value]`. - fn scan4() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("region", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - /// `Aggregate{ by: [1,2] (service, region), [agg] }` — output `[service, - /// region, ]` (3 cols), so a ROW_NUMBER over it appends `rn` at index 3. - fn grouped(agg: AggIntent) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(vec![1, 2]), - measures: vec![agg], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan4()), - } - } - - /// `ROW_NUMBER` ignores its frame clause, so the top-k rewrite doesn't care - /// what's in it; any concrete frame works as fixture data. - fn rownumber_frame() -> WindowFrame { - WindowFrame { - units: WindowFrameUnits::Rows, - start_bound: WindowFrameBound::Preceding(WindowFrameOffset::Scalar(ScalarValue::Null)), - end_bound: WindowFrameBound::Following(WindowFrameOffset::Scalar(ScalarValue::Null)), - } - } - - /// `Filter{ rn(3) <= 5 } { SQLWindowFunc{ RowNumber, PARTITION BY region(2), - /// ORDER BY col(2) DESC } { agg } }`. - fn rownumber_topk(agg: QueryExpr) -> QueryExpr { - let wf = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![2]), // region - order_by: vec![SortKey { - expr: QueryExpr::Column(2), // the aggregate output column - ascending: false, - nulls_first: true, - }], - frame: Some(rownumber_frame()), - output_name: "rn".into(), - child: Rc::new(agg), - }; - QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(3)), // rn = the appended window column - op: CompareOpKind::Le, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(5))), - })), - child: Rc::new(wf), - } - } - - #[test] - fn rownumber_count_topk_becomes_a_partitioned_heavy_hitter() { - // Count-ranked ROW_NUMBER top-k → outer TopK grouped by the partition - // (region, col 2) over the explicit inner Count. - let q = rownumber_topk(grouped(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - })); - let out = canonicalize(q); - let QueryExpr::Aggregate { - reduction, - measures, - child, - .. - } = &out - else { - panic!("expected outer Aggregate([TopK]), got {out:?}"); - }; - let Reduction::Reduce(by) = reduction else { - panic!("expected a Reduce grouping, got {reduction:?}"); - }; - assert!(matches!( - measures.as_slice(), - [AggIntent::TopK { k: 5, .. }] - )); - assert_eq!(**by, vec![2], "outer TopK partitioned by region"); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Count { .. }])) - ); - } - - #[test] - fn rownumber_avg_topk_becomes_a_partitioned_sort_limit() { - // Avg-ranked (not a frequency heavy-hitter) → generic partitioned - // top-k: Limit{5}{ Sort{ partition_by: [region] } }. - let q = rownumber_topk(grouped(AggIntent::Avg { col: None })); - let out = canonicalize(q); - let QueryExpr::Limit { n, child, .. } = &out else { - panic!("expected a Limit, got {out:?}"); - }; - assert_eq!(*n, 5); - let QueryExpr::Sort { - partition_by, - child, - .. - } = child.as_ref() - else { - panic!("expected a Sort under the Limit"); - }; - assert_eq!(**partition_by, vec![2], "partitioned by region"); - assert!( - matches!(child.as_ref(), QueryExpr::Aggregate { measures, .. } - if matches!(measures.as_slice(), [AggIntent::Avg { .. }])) - ); - } - - #[test] - fn filter_on_a_non_rownumber_column_is_left_alone() { - // `WHERE service_len <= 5` (col 0, not the rn window column) must not be - // mistaken for a top-k. - let wf = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![2]), - order_by: vec![SortKey { - expr: QueryExpr::Column(2), - ascending: false, - nulls_first: true, - }], - frame: Some(rownumber_frame()), - output_name: "rn".into(), - child: Rc::new(grouped(AggIntent::Count { - accuracy: AccuracyTarget::Exact, - })), - }; - let q = QueryExpr::Filter { - pred: Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(0)), // NOT the rn column (index 3) - op: CompareOpKind::Le, - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(5))), - })), - child: Rc::new(wf), - }; - assert!( - matches!(canonicalize(q), QueryExpr::Filter { .. }), - "left as a Filter" - ); - } -} diff --git a/crates/types/src/pre_asap/column_resolution.rs b/crates/types/src/pre_asap/column_resolution.rs index 350ea535b..30411f6a7 100644 --- a/crates/types/src/pre_asap/column_resolution.rs +++ b/crates/types/src/pre_asap/column_resolution.rs @@ -1,22 +1,13 @@ //! Schema-driven column resolution. //! -//! Front ends (issue #179) emit `ColumnRef` (name-based, optionally -//! table-qualified); the canonical DAG uses positional [`ColumnId`] resolved -//! against a per-node [`Schema`]. These helpers bridge the two — the -//! [`SchemaResolver`](super::schema_resolver) builds the schema, and [`resolve_column_refs`] -//! turns name-based refs (group keys, dedup columns) into positional ids, -//! qualifier-aware. - -use std::rc::Rc; +//! Front ends emit `ColumnRef` (name-based, optionally table-qualified); the +//! IR uses positional [`ColumnId`] resolved against a per-node [`Schema`]. +//! These helpers turn name-based refs into positional ids, qualifier-aware. +//! Front-end name resolution (`asap_frontend_common::resolve`) calls them. use thiserror::Error; -use super::agg_intent::AggIntent; use super::expr_ir::ColumnRef; -use super::query_expr::{ - aggregate_output_schema, GroupKeys, QueryExpr, QueryExprError, Reduction, ResolvedQueryExpr, - UnresolvedQueryExpr, -}; use super::schema::{ColumnId, DataType, FieldDataType, Schema}; /// Errors returned by the resolution helpers. @@ -129,102 +120,6 @@ pub fn resolve_group_keys_promql( .collect() } -/// Resolve a name-based scalar [`UnresolvedQueryExpr`] (one of `QueryExpr`'s scalar -/// variants, issue #205) into a positional [`ResolvedQueryExpr`] by resolving every -/// column reference against `schema`. Structural otherwise. `expr` must be -/// one of the scalar variants — an operator variant here is a construction -/// bug, not a shape this needs to handle silently. -pub fn resolve_expr( - expr: &UnresolvedQueryExpr, - schema: &Schema, -) -> Result { - let rc = |e: &UnresolvedQueryExpr| -> Result, ResolveError> { - Ok(Rc::new(resolve_expr(e, schema)?)) - }; - let each = |es: &[UnresolvedQueryExpr]| -> Result, ResolveError> { - es.iter().map(|e| resolve_expr(e, schema)).collect() - }; - Ok(match expr { - QueryExpr::Column(c) => QueryExpr::Column(resolve_column_ref(c, schema)?), - QueryExpr::Literal(s) => QueryExpr::Literal(s.clone()), - QueryExpr::EvalTimestamp => QueryExpr::EvalTimestamp, - QueryExpr::CurrentTimestamp => QueryExpr::CurrentTimestamp, - QueryExpr::Compare { left, op, right } => QueryExpr::Compare { - left: rc(left)?, - op: op.clone(), - right: rc(right)?, - }, - QueryExpr::BoolAnd(v) => QueryExpr::BoolAnd(each(v)?), - QueryExpr::BoolOr(v) => QueryExpr::BoolOr(each(v)?), - QueryExpr::Not(e) => QueryExpr::Not(rc(e)?), - QueryExpr::IsNull(e) => QueryExpr::IsNull(rc(e)?), - QueryExpr::IsNotNull(e) => QueryExpr::IsNotNull(rc(e)?), - QueryExpr::Cast { expr, to, try_cast } => QueryExpr::Cast { - expr: rc(expr)?, - to: to.clone(), - try_cast: *try_cast, - }, - QueryExpr::InList { - expr, - list, - negated, - } => QueryExpr::InList { - expr: rc(expr)?, - list: each(list)?, - negated: *negated, - }, - QueryExpr::FunctionCall { name, args } => QueryExpr::FunctionCall { - name: name.clone(), - args: each(args)?, - }, - QueryExpr::Arithmetic { op, left, right } => QueryExpr::Arithmetic { - op: op.clone(), - left: rc(left)?, - right: rc(right)?, - }, - QueryExpr::Case { - operand, - branches, - else_expr, - } => QueryExpr::Case { - operand: operand.as_deref().map(rc).transpose()?, - branches: branches - .iter() - .map(|(w, t)| Ok((resolve_expr(w, schema)?, resolve_expr(t, schema)?))) - .collect::, ResolveError>>()?, - else_expr: else_expr.as_deref().map(rc).transpose()?, - }, - other => unreachable!("resolve_expr called on a non-scalar QueryExpr variant: {other:?}"), - }) -} - -/// Output schema produced by an `Aggregate { by, measures }` over `input`. -/// Mirrors `QueryExpr::output_schema_in`'s `Aggregate` arm; out-of-range `by` -/// ids are silently dropped (callers needing the strict check resolve `by` -/// via [`resolve_column_refs`], which surfaces `NotFound`). -pub fn output_schema_for_aggregate( - input: &Schema, - by: &GroupKeys, - measures: &[AggIntent], - output_names: &[String], -) -> Result { - // Delegate to the single canonical derivation so HAVING resolution can never - // drift from `QueryExpr::output_schema_in` (issue #41). HAVING is SQL-only - // and cross-series (SQL has no `without`), but detect the child-independent - // per-entity case anyway (a lone `rate`/`increase`/`*_over_time` intent) so - // the two agree on every shared input — the range-window child marker the - // canonical arm also keys off is not visible here, and never co-occurs with - // HAVING. - let per_entity = - by.is_empty() && !by.is_without() && measures.len() == 1 && measures[0].is_per_series(); - let reduction = if per_entity { - Reduction::PerEntity - } else { - Reduction::Reduce(by.clone()) - }; - aggregate_output_schema(input, &reduction, measures, output_names) -} - #[cfg(test)] mod tests { use super::*; @@ -340,77 +235,4 @@ mod tests { Err(ResolveError::NotFound { .. }) )); } - - #[test] - fn aggregate_strips_time_and_keeps_unique_keys() { - let mut input = ts_value_schema(); - input - .fields - .push(Field::plain("host", DataType::Utf8, false)); - let out = output_schema_for_aggregate( - &input, - &GroupKeys::by(vec![2]), - &[AggIntent::Sum { col: None }], - &[], - ) - .expect("valid group-by column"); - assert_eq!(out.fields.len(), 2); // host, sum - assert_eq!(out.fields[0].name, "host"); - assert_eq!(out.fields[1].name, "sum"); - assert!(out.time_index.is_none()); - assert_eq!(out.unique_keys, vec![vec![0]]); - } - - #[test] - fn having_schema_agrees_with_canonical_for_a_per_series_reduction() { - // Issue #41: `output_schema_for_aggregate` (HAVING resolution) and the - // canonical `QueryExpr::output_schema_in` must produce identical schemas - // for the same aggregate. Before the dedup this diverged on a per-series - // reduction — the HAVING mirror lacked the per-series branch and would - // collapse `[ts, value]` to a single `rate` column. - use crate::pre_asap::query_expr::Source; - use std::time::Duration; - - let leaf_schema = Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - 0, - vec![], - ); - let scan = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: leaf_schema.clone(), - }; - // Aggregate{ reduction: PerEntity, [Rate], child: TimeRange{ Scan } } — - // a per-series reduction (label-preserving). - let agg = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan), - }), - }; - let canonical = agg.output_schema().expect("canonical schema"); - - // The HAVING-resolution derivation gets only the input schema (the - // TimeRange passes the leaf schema through). - let having_side = - output_schema_for_aggregate(&leaf_schema, &GroupKeys::none(), &[AggIntent::Rate], &[]) - .unwrap(); - - assert_eq!( - canonical, having_side, - "the two aggregate-schema derivations must agree (issue #41)" - ); - // Sanity: it really is the label-preserving per-series shape, not `[rate]`. - assert!(having_side.fields.iter().any(|c| c.name == "value")); - assert!(having_side.time_index.is_some()); - } } diff --git a/crates/types/src/pre_asap/cse.rs b/crates/types/src/pre_asap/cse.rs deleted file mode 100644 index 0a00475e6..000000000 --- a/crates/types/src/pre_asap/cse.rs +++ /dev/null @@ -1,1111 +0,0 @@ -//! Pre-ASAP structural common-subexpression elimination: bottom-up -//! hash-consing over an already-`resolve_root`'d [`QueryExpr`] DAG (issue -//! #212, #222, #223). -//! -//! CSE only runs on an already-bound, already-canonicalized DAG — -//! structural matching is meaningless before canonicalization has converged -//! semantically-equivalent queries onto one shape (`docs/develop_docs/pre-asap-ir.md` -//! design principle 3; `median(latency)` and `approx_percentile_cont(latency, -//! 0.5)` already lower to an identical `AggIntent::Quantile` today, per -//! `sql_lowering.rs`'s `median_is_the_same_intent_as_an_explicit_half_percentile` -//! test). [`share_common_sub_dags`] is the single entry point, run once per -//! workload batch (or once per query — see "Single-query CSE" below) *after* -//! `resolve_root`, *before* the pre-ASAP → post-ASAP replacement/search pass -//! (`asap_aware_mapping::replacement`). -//! -//! ## Algorithm: classic hash-consing / value-numbering -//! -//! Bottom-up: every child is interned before its parent, so a parent's -//! candidacy for sharing naturally incorporates whether its own children were -//! themselves shared — two parents whose children were independently -//! deduplicated down to the same `Rc`s are structurally identical iff their -//! own fields also match, without re-walking the sub-DAGs. -//! -//! Only the **relational skeleton** participates — the same set of "operator" -//! children [`canonicalize`](super::canonicalize)'s `children_mut` walks -//! (`Filter`/`Project`/`Aggregate`/`Concat`/`Join`/`BinaryOp`/…). A scalar -//! subexpression reachable only through a wrapper position (`Predicate`, -//! `ProjectItem.expr`, `Aggregate.having`, `SQLWindowFunc.args`, …) stays -//! embedded as opaque data on its owning operator node, compared by -//! `QueryExpr`'s derived `PartialEq` along with the rest of that node's -//! fields, rather than separately hash-consed — the same scope -//! `canonicalize.rs` settled on ("none of the rewrite rules touch a scalar -//! sub-DAG, so there's nothing to gain by recursing into one"). Widening this -//! to scalar positions is future work, not attempted here. -//! -//! ## Correctness: hash is a filter, `PartialEq` is the decision -//! -//! This is the one non-negotiable rule. A **false positive** here — two -//! sub-DAGs wrongly judged shareable — is a wrong query answer, not a missed -//! optimization: two different queries would read each other's data. -//! [`structural_hash`] (`DefaultHasher`/SipHash over a canonical -//! serialization, no collision-freedom guarantee) may only narrow the -//! candidate set within one bucket; [`InternTable::intern`]'s `PartialEq` -//! check on that bucket is what actually decides sharing, every time, no -//! exceptions for "the hash probably didn't collide." -//! -//! This also means the pass is safe by construction against the case #212 -//! flagged as a real historical bug (issue #115): `AggIntent::Quantile` -//! carries its input column and its `AccuracyTarget`, both `PartialEq` -//! fields, so `Quantile(x, 0.99, ε=0.01)` and `Quantile(x, 0.99, ε=0.001)` — -//! or `Quantile(x, ..)` vs `Quantile(y, ..)` — are never merged. This is -//! intentionally conservative: it only recognizes *exact* structural -//! matches, not "a stricter-accuracy summary could also answer a looser -//! request." That subsumption question already has a documented, -//! deliberately-unfilled home (`asap_aware_mapping::Matcher`) — -//! CSE here does not attempt it. -//! -//! ## Legality: gated by `Schema::unique_keys` -//! -//! Structural equality alone is necessary but not sufficient. Per -//! [`Schema::unique_keys`](super::schema::Schema::unique_keys)'s own doc: "a -//! producer's output can only be safely shared across consumers when its row -//! identity is provably stable across reads." A candidate node with no -//! provable unique key (`Schema::has_unique_key()` false, or `output_schema` -//! not even defined for that node, e.g. a `Concat`/`SetOp` branch whose union -//! drops `unique_keys`, or an ungrouped/global `Aggregate`, whose empty `by` -//! also reports no unique key today) is **never** hoisted, even when it is -//! structurally identical to something already interned — it is always -//! inserted fresh, matching the rule the (now-deleted) prior CSE attempt -//! already encoded and the doc comment on `Aggregate`'s `child` field -//! ("`unique_keys` feeds CSE's producer-sharing legality check"). -//! -//! ## Single-query CSE falls out for free -//! -//! A repeated sub-expression within *one* query (e.g. the same grouped -//! `Aggregate` referenced twice on two `BinaryOp` branches) is deduplicated -//! by the exact same bottom-up interning — a workload of size one still -//! interns bottom-up within that one DAG. No separate mechanism is needed; -//! see the `single_query_shares_its_own_repeated_sub_dag` test below. -//! -//! ## Landing plan (issue #223) -//! -//! This module is stage 1 of a 4-stage plan. Stage 2 -//! (`asap_aware_mapping::replacement::search_workload_with`, which runs -//! [`share_common_sub_dags`] itself before searching) is a real caller, -//! wired at the same time so this never becomes unwired dead code again -//! (the original `asap-plan::cse::dedupe_subtrees` was deleted in #192 for -//! exactly that). Stage 3 — [`dag_export`](crate::dag_export) computing its -//! per-node `hash` by calling this module's [`structural_hash`] directly, -//! instead of a parallel reimplementation — is also done, so -//! `tools/dag-viewer`'s "shared sub-DAG" highlighting now flags exactly the -//! candidate pairs this module's own `InternTable` would bucket together -//! (still only a hash match, not a guarantee of -//! `share_common_sub_dags`-actual sharing — see `dag_export`'s module doc). -//! Stage 4 (issue #237) is implemented in -//! `asap_aware_mapping::cost_model::CostModel::cse_share_decision`, called -//! from `asap_aware_mapping::replacement::CandidateLogicalASAPDAGs::cost_sorted` (via that -//! module's own `cse_preference`) — a real, Volcano/Cascades-style cost -//! comparison over what this module detects, not a fixed rule. See -//! `docs/design_docs/cost-model.md`. This module's own -//! unconditional "share whenever legal" behavior is unchanged: detection -//! stays cost-agnostic by construction (this crate cannot depend on -//! `asap-aware-mapping`'s `CostModel`), and the cost-aware decision is -//! applied downstream, after detection, over what this module finds. - -use std::collections::HashMap; -use std::hash::{Hash, Hasher}; -use std::rc::Rc; - -use super::query_expr::QueryExpr; - -/// Bottom-up hash-consing table: structurally-equal, sharing-legal -/// [`QueryExpr`] nodes collapse onto one `Rc`. -/// -/// `buckets` is keyed by [`structural_hash`] — a coarse candidate filter -/// only (see the module-level "Correctness" section). Every entry within one -/// bucket is a full node kept around for the `PartialEq` comparison that -/// actually decides a match; a hash collision between structurally different -/// nodes just means a (harmless) linear scan of a few extra candidates. -struct InternTable { - buckets: HashMap>>, - /// Memoizes [`structural_hash`] per already-hashed `Rc` pointer, shared - /// across every [`intern`](Self::intern) call for the table's whole - /// lifetime — see [`structural_hash`]'s own doc on why this matters: - /// without it, hashing an `N`-node bottom-up pass costs `O(N)` work - /// *per node* (every already-interned descendant gets re-walked), not - /// `O(1)` amortized. - hash_cache: HashCache, -} - -impl InternTable { - fn new() -> Self { - Self { - buckets: HashMap::new(), - hash_cache: HashMap::new(), - } - } - - /// Intern one already-children-rebuilt node: look it up by - /// [`structural_hash`], confirm with `PartialEq`, and — only when - /// sharing is legal (see "Legality" above) — return the existing `Rc` - /// instead of allocating a new one. - fn intern(&mut self, node: QueryExpr) -> Rc { - let hash = structural_hash(&node, &mut self.hash_cache); - // A node with no provable unique key is never *returned* as a match - // for something else — it may still go on to occupy a fresh slot in - // the bucket (harmless; it just never gets found by a later - // `PartialEq` scan that also requires `reusable`). - let reusable = node - .output_schema() - .is_ok_and(|schema| schema.has_unique_key()); - let bucket = self.buckets.entry(hash).or_default(); - if reusable { - if let Some(existing) = bucket.iter().find(|candidate| candidate.as_ref() == &node) { - return Rc::clone(existing); - } - } - let rc = Rc::new(node); - bucket.push(Rc::clone(&rc)); - rc - } -} - -/// [`structural_hash`]'s memoization cache: maps an already-hashed node's -/// `Rc` pointer to its computed hash. Not tied to any one `QueryExpr` — a -/// fresh, empty cache is correct to start with anywhere; what matters is -/// letting it *persist* across every node in one bottom-up pass (as -/// [`InternTable`] does via its own `hash_cache` field), rather than -/// starting a new one per call. -/// -/// `pub` (not `pub(crate)`) so `asap_aware_mapping`'s workload-search MEMO -/// engine (`replacement::is_duplicate_rewrite`) can reuse this exact -/// candidate-narrowing filter for its own dedup, instead of maintaining a -/// parallel reimplementation — the same "one real hash, reused everywhere -/// it's needed" rationale [`structural_hash`]'s own doc gives for -/// [`dag_export`](crate::dag_export)'s `pub(crate)` reuse. -pub type HashCache = HashMap<*const QueryExpr, u64>; - -/// Coarse structural hash used only to bucket [`InternTable::intern`]'s -/// candidate search — never the actual sharing decision (`PartialEq` is). -/// -/// `QueryExpr` carries `f64`s (`Literal(ScalarValue::Float64)`, `AggIntent::Quantile.q`, …), so it -/// cannot derive `std::hash::Hash`. Serializing to a canonical JSON string -/// and hashing that sidesteps the `f64` problem — but only for `node`'s own -/// tag and non-child fields, *not* its children's full values: each -/// `Rc`-backed child's contribution is its own [`structural_hash`], looked -/// up in `cache` if already computed there (memoized by `Rc` pointer -/// identity) rather than recursed into again. -/// -/// This is the DAG-aware fix a naive "just serialize the whole sub-DAG" -/// hash would get wrong: after [`share_common_sub_dags`] (or even before -/// it — a front end can emit internal `Rc` sharing directly, e.g. a -/// repeated subexpression within one query), `node` generally has internal -/// sharing. A full-sub-DAG serialization re-serializes — re-walks — -/// any descendant `node` already shares internally once per parent that -/// references it; called once per node in a bottom-up pass (as -/// [`InternTable::intern`] and [`dag_export`](crate::dag_export) both do), -/// that costs `O(sub-DAG size)` *per node* instead of `O(1)` amortized — -/// quadratic-or-worse for a deep chain, compounding further with any real -/// internal sharing. Memoizing each child's hash by pointer identity in -/// `cache` (persisted across the whole pass by the caller, not reset per -/// node) makes each node's own contribution `O(1)` beyond its children's -/// already-known hashes, giving `O(N)` total for `N` nodes — matching -/// [`dag_node_count`]'s own shared-node counting fix (issue #212/#223/#237's stage -/// 4) in spirit, applied to hashing instead of counting. -/// -/// `pub` (not private) so [`dag_export`](crate::dag_export) can call -/// this exact function for its exported nodes' `hash` field instead of -/// maintaining its own parallel reimplementation — issue #223 stage 3. That -/// makes `tools/dag-viewer`'s "shared sub-DAG" highlighting reflect this -/// module's real hashing, not a lookalike computed a different way; see the -/// module doc's "Landing plan" section. A NaN/infinite `f64` makes JSON -/// serialization fail; falling back to a fixed hash just puts every such -/// node in one (larger, still `PartialEq`-disambiguated) bucket. Made `pub` -/// (rather than staying `pub(crate)`) for one more reuse across the crate -/// boundary: `asap_aware_mapping`'s workload-search MEMO engine -/// (`replacement::is_duplicate_rewrite`) needs the identical -/// candidate-narrowing filter this module's own [`InternTable::intern`] -/// already uses, so it doesn't have to reinvent (and risk drifting from) it. -/// -/// Exhaustive over every `QueryExpr` variant, matching [`rebuild_children`] -/// in which fields count as an operator child (must stay in sync — a new -/// variant fails to compile in both places until both are extended). -pub fn structural_hash(node: &QueryExpr, cache: &mut HashCache) -> u64 { - use QueryExpr::*; - - fn child_hash(child: &Rc, cache: &mut HashCache) -> u64 { - let ptr = Rc::as_ptr(child); - if let Some(&h) = cache.get(&ptr) { - return h; - } - let h = structural_hash(child, cache); - cache.insert(ptr, h); - h - } - - /// Hash `own_fields` (this node's own tag and non-child scalar - /// fields — anything JSON-serializable and small, i.e. never a - /// `QueryExpr` sub-DAG) via the same canonical-JSON-string trick the - /// whole-sub-DAG version used, just applied to `O(1)` fields instead - /// of `O(sub-DAG size)`. - fn hash_own_fields(hasher: &mut impl Hasher, own_fields: &impl serde::Serialize) { - serde_json::to_string(own_fields) - .unwrap_or_default() - .hash(hasher); - } - - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - match node { - Scan { - source, - predicates, - schema, - } => hash_own_fields(&mut hasher, &("Scan", source, predicates, schema)), - PromqlVectorFromScalar(c) => { - "PromqlVectorFromScalar".hash(&mut hasher); - child_hash(c, cache).hash(&mut hasher); - } - PromqlScalarFromVector(c) => { - "PromqlScalarFromVector".hash(&mut hasher); - child_hash(c, cache).hash(&mut hasher); - } - PromqlRelabel { dst, value, child } => { - hash_own_fields(&mut hasher, &("PromqlRelabel", dst, value)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlInfoEnrich { selector, child } => { - hash_own_fields(&mut hasher, &("PromqlInfoEnrich", selector)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlSeriesSample { by, kind, child } => { - hash_own_fields(&mut hasher, &("PromqlSeriesSample", by, kind)); - child_hash(child, cache).hash(&mut hasher); - } - Filter { pred, child } => { - hash_own_fields(&mut hasher, &("Filter", pred)); - child_hash(child, cache).hash(&mut hasher); - } - Project { - cols, - qualifier, - child, - } => { - hash_own_fields(&mut hasher, &("Project", cols, qualifier)); - child_hash(child, cache).hash(&mut hasher); - } - Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - hash_own_fields( - &mut hasher, - &( - "Aggregate", - reduction, - measures, - output_names, - filters, - having, - ), - ); - child_hash(child, cache).hash(&mut hasher); - } - Dedup { cols, child } => { - hash_own_fields(&mut hasher, &("Dedup", cols)); - child_hash(child, cache).hash(&mut hasher); - } - Concat { - children, - discriminator_unique_key, - } => { - hash_own_fields(&mut hasher, &("Concat", discriminator_unique_key)); - for c in children { - // Stored by value, not `Rc` — see `rebuild_children`'s - // `intern_owned` use for this variant — so there's no - // pointer to memoize on here; recurse directly. Any - // `Rc`-typed descendant beneath `c` still gets memoized - // once this call reaches it. - structural_hash(c, cache).hash(&mut hasher); - } - } - Join { - kind, - pred, - left, - right, - } => { - hash_own_fields(&mut hasher, &("Join", kind, pred)); - child_hash(left, cache).hash(&mut hasher); - child_hash(right, cache).hash(&mut hasher); - } - SetOp { - kind, - all, - left, - right, - } => { - hash_own_fields(&mut hasher, &("SetOp", kind, all)); - child_hash(left, cache).hash(&mut hasher); - child_hash(right, cache).hash(&mut hasher); - } - Sort { - keys, - partition_by, - child, - } => { - hash_own_fields(&mut hasher, &("Sort", keys, partition_by)); - child_hash(child, cache).hash(&mut hasher); - } - Limit { n, offset, child } => { - hash_own_fields(&mut hasher, &("Limit", n, offset)); - child_hash(child, cache).hash(&mut hasher); - } - PromqlSubquery { - range, - resolution, - child, - } => { - hash_own_fields(&mut hasher, &("PromqlSubquery", range, resolution)); - child_hash(child, cache).hash(&mut hasher); - } - TimeRange { range, child } => { - hash_own_fields(&mut hasher, &("TimeRange", range)); - child_hash(child, cache).hash(&mut hasher); - } - TimeShift { shift, child } => { - hash_own_fields(&mut hasher, &("TimeShift", shift)); - child_hash(child, cache).hash(&mut hasher); - } - SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => { - hash_own_fields( - &mut hasher, - &( - "SQLWindowFunc", - func, - args, - partition_by, - order_by, - frame, - output_name, - ), - ); - child_hash(child, cache).hash(&mut hasher); - } - BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - hash_own_fields(&mut hasher, &("BinaryOp", op, vector_match)); - child_hash(lhs, cache).hash(&mut hasher); - child_hash(rhs, cache).hash(&mut hasher); - } - // `EvalTimestamp`, `PromqlScalarBridge`, and the scalar variants - // (issue #205) are all leaves for this traversal's purposes — none - // has an operator child to look up in `cache` — so hashing the - // whole node via `serde_json` in one shot is already `O(node - // size)`, not `O(sub-DAG size)`: exactly the same cost the - // per-variant `hash_own_fields` calls above pay, just without - // needing to spell out each field individually. Matches - // `rebuild_children`'s and `dag_node_count`'s identical scope - // decision for these variants ("never descended into"). - EvalTimestamp - | CurrentTimestamp - | PromqlScalarBridge(_) - | Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => hash_own_fields(&mut hasher, node), - } - hasher.finish() -} - -/// Count of *unique* nodes reachable from `root`, deduplicated by `Rc` -/// pointer identity (`Rc::as_ptr`) — the real size of the DAG rooted at -/// `root`, not a per-path walk count. -/// -/// After [`share_common_sub_dags`] runs (or even before it, for a DAG a -/// front end already built with internal `Rc` sharing — e.g. re-running -/// CSE, or a single-query repeated subexpression), `root` is generally a -/// **DAG** with internal sharing — that is this whole module's premise. Anything that -/// walks `root` as if every reference were a fresh sub-DAG (a naive -/// recursive walk with no identity tracking, or a naive full -/// `serde_json` serialization — `Rc`'s `Serialize` impl serializes the -/// pointee's *value* at every occurrence, it does not dedupe by identity) -/// re-visits/re-counts an already-shared descendant once per parent that -/// references it, over-counting relative to the actual work of holding it -/// in memory or recomputing it once. This function is the DAG-correct -/// alternative: each unique node is counted exactly once, regardless of -/// how many places within `root` reference it. -/// -/// `pub` so cost-aware callers outside this crate (e.g. -/// `asap_aware_mapping::CostModel::cse_recompute_cost`'s default) have a -/// DAG-correct structural-size proxy available, instead of reaching for -/// something per-path like a raw serialization length. -/// -/// Same operator-child traversal scope as [`share_common_sub_dags`] itself -/// (see the module doc's "Algorithm" section, and this module's private -/// `rebuild_children`) — a scalar subexpression embedded in a wrapper -/// position (`Predicate`, `ProjectItem.expr`, `Aggregate.having`, …) is not -/// separately visited, matching this module's own stated scope; it's -/// counted as part of its owning operator node, the same node -/// `rebuild_children` treats as a single opaque leaf for interning -/// purposes. -pub fn dag_node_count(root: &QueryExpr) -> usize { - let mut seen: std::collections::HashSet<*const QueryExpr> = std::collections::HashSet::new(); - count_unique(root, &mut seen) -} - -/// One node's own contribution (`1`) plus each *not-yet-seen* operator -/// child's contribution — exhaustive over every `QueryExpr` variant, -/// enumerating the same fields [`rebuild_children`] does (kept as a -/// separate, read-only traversal rather than threaded through -/// `rebuild_children` itself, since that function consumes and rebuilds -/// its input while this one only ever reads it). -fn count_unique(node: &QueryExpr, seen: &mut std::collections::HashSet<*const QueryExpr>) -> usize { - use QueryExpr::*; - - /// Visit one `Rc`-held child: counts (and recurses into) it only the - /// first time its pointer is seen, `0` on every later occurrence — - /// this is the actual dedup step. - fn visit( - child: &Rc, - seen: &mut std::collections::HashSet<*const QueryExpr>, - ) -> usize { - if seen.insert(Rc::as_ptr(child)) { - count_unique(child, seen) - } else { - 0 - } - } - - 1 + match node { - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220), never descended into — same treatment `rebuild_children` - // gives it (see that function's comment on this same variant). - Scan { .. } | PromqlScalarBridge(_) | EvalTimestamp | CurrentTimestamp => 0, - PromqlVectorFromScalar(c) | PromqlScalarFromVector(c) => visit(c, seen), - PromqlRelabel { child, .. } - | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | Sort { child, .. } - | Limit { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } => visit(child, seen), - // `Concat`'s branches are stored by value (`Vec`, not - // `Rc` — see `rebuild_children`'s `intern_owned` use for - // this variant), so a branch has no `Rc` identity of its own to - // dedup on at this position; still recurse into each in case an - // `Rc`-shared descendant appears further down. - Concat { children, .. } => children.iter().map(|c| count_unique(c, seen)).sum(), - Join { left, right, .. } | SetOp { left, right, .. } => { - visit(left, seen) + visit(right, seen) - } - BinaryOp { lhs, rhs, .. } => visit(lhs, seen) + visit(rhs, seen), - // Scalar variants (issue #205) — never descended into, matching - // `rebuild_children`'s own scope exactly (see its trailing match - // arm and this module's "Algorithm" section). - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => 0, - } -} - -/// Recurse into `child`, then intern the result. `Rc::try_unwrap` recovers -/// the owned node without cloning in the overwhelmingly common case — a -/// DAG freshly built by a front end / `resolve_root`, not yet shared by any -/// prior CSE pass, where every `Rc` is uniquely owned. Falls back to cloning -/// this node's own fields (its children stay `Rc`s, not deep-copied) only -/// when `child` is already shared — e.g. re-running CSE over a DAG that -/// went through a previous `share_common_sub_dags` pass; a structural -/// duplicate collapses right back onto `child` itself via `PartialEq`, an -/// already-optimal no-op. -fn intern_child(table: &mut InternTable, child: Rc) -> Rc { - match Rc::try_unwrap(child) { - Ok(owned) => intern_bottom_up(table, owned), - Err(shared) => intern_bottom_up(table, (*shared).clone()), - } -} - -/// Like [`intern_child`], for a `Concat` branch — stored by value -/// (`Vec`, not `Rc`), so this position itself can never -/// alias another parent. Interning it anyway still lets any `Rc`-typed -/// descendant of the branch participate in sharing, and registers the -/// branch's own hash/value in the table for a *different* `Concat` elsewhere -/// with a structurally identical branch (which — being in its own `Vec` -/// slot too — still can't literally share the `Rc`, but this keeps the -/// interning behavior uniform and the table's bucket contents consistent). -fn intern_owned(table: &mut InternTable, expr: QueryExpr) -> QueryExpr { - let rc = intern_bottom_up(table, expr); - Rc::try_unwrap(rc).unwrap_or_else(|shared| (*shared).clone()) -} - -/// Bottom-up: rebuild `expr`'s children (recursively interning each), then -/// intern the rebuilt node itself. -fn intern_bottom_up(table: &mut InternTable, expr: QueryExpr) -> Rc { - let rebuilt = rebuild_children(table, expr); - table.intern(rebuilt) -} - -/// Rebuild `expr` with each **operator** child (see the module doc on scope) -/// replaced by its interned `Rc`. Exhaustive over every `QueryExpr` variant, -/// matching `canonicalize.rs`'s `children_mut` exactly in which fields count -/// as an operator child — new variants fail to compile here until this match -/// is extended. -fn rebuild_children(table: &mut InternTable, expr: QueryExpr) -> QueryExpr { - use QueryExpr::*; - match expr { - Scan { .. } | EvalTimestamp | CurrentTimestamp => expr, - PromqlVectorFromScalar(c) => PromqlVectorFromScalar(intern_child(table, c)), - PromqlScalarFromVector(c) => PromqlScalarFromVector(intern_child(table, c)), - PromqlRelabel { dst, value, child } => PromqlRelabel { - dst, - value, - child: intern_child(table, child), - }, - PromqlInfoEnrich { selector, child } => PromqlInfoEnrich { - selector, - child: intern_child(table, child), - }, - PromqlSeriesSample { by, kind, child } => PromqlSeriesSample { - by, - kind, - child: intern_child(table, child), - }, - Filter { pred, child } => Filter { - pred, - child: intern_child(table, child), - }, - Project { - cols, - qualifier, - child, - } => Project { - cols, - qualifier, - child: intern_child(table, child), - }, - Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => Aggregate { - reduction, - measures, - output_names, - filters, - having, - child: intern_child(table, child), - }, - Dedup { cols, child } => Dedup { - cols, - child: intern_child(table, child), - }, - Concat { - children, - discriminator_unique_key, - } => Concat { - children: children - .into_iter() - .map(|c| intern_owned(table, c)) - .collect(), - discriminator_unique_key, - }, - Join { - kind, - pred, - left, - right, - } => Join { - kind, - pred, - left: intern_child(table, left), - right: intern_child(table, right), - }, - SetOp { - kind, - all, - left, - right, - } => SetOp { - kind, - all, - left: intern_child(table, left), - right: intern_child(table, right), - }, - Sort { - keys, - partition_by, - child, - } => Sort { - keys, - partition_by, - child: intern_child(table, child), - }, - Limit { n, offset, child } => Limit { - n, - offset, - child: intern_child(table, child), - }, - PromqlSubquery { - range, - resolution, - child, - } => PromqlSubquery { - range, - resolution, - child: intern_child(table, child), - }, - TimeRange { range, child } => TimeRange { - range, - child: intern_child(table, child), - }, - TimeShift { shift, child } => TimeShift { - shift, - child: intern_child(table, child), - }, - SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child: intern_child(table, child), - }, - BinaryOp { - op, - lhs, - rhs, - vector_match, - } => BinaryOp { - op, - lhs: intern_child(table, lhs), - rhs: intern_child(table, rhs), - vector_match, - }, - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220) — same "never descended into" treatment as the scalar - // variants below; the whole bridge node is still interned as a unit - // by the `table.intern(rebuilt)` call in `intern_bottom_up`. - PromqlScalarBridge(_) => expr, - // Scalar variants (issue #205) — never descended into; see the - // module doc's "Algorithm" section on scope. Left byte-for-byte - // unchanged: predicate / project-list / sort-key / window-arg - // expressions stay embedded as opaque leaf data, compared by the - // enclosing operator node's derived `PartialEq`. - Column(_) - | Literal(_) - | Compare { .. } - | BoolAnd(_) - | BoolOr(_) - | Not(_) - | IsNull(_) - | IsNotNull(_) - | Cast { .. } - | InList { .. } - | FunctionCall { .. } - | Arithmetic { .. } - | Case { .. } => expr, - } -} - -/// Share structurally-identical, sharing-legal sub-DAGs across a workload's -/// query roots (or within one query, for `roots.len() == 1` — see the -/// module doc's "Single-query CSE" section). Every root's *value* is -/// unchanged (`PartialEq`-equal to its input) — only its internal `Rc` -/// structure may now alias another root's, or another part of its own DAG. -/// -/// `roots` must already be bound + canonicalized (post-`resolve_root`). -/// `Id` is caller-chosen — a `QueryWorkload` entry's own key, an index, a -/// query name, whatever identifies one root through the pipeline; this -/// module has no opinion on its shape. -pub fn share_common_sub_dags(roots: Vec<(Id, QueryExpr)>) -> Vec<(Id, Rc)> { - let mut table = InternTable::new(); - roots - .into_iter() - .map(|(id, expr)| (id, intern_bottom_up(&mut table, expr))) - .collect() -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::agg_intent::AggIntent; - use crate::pre_asap::expr_ir::{CompareOpKind, ScalarValue}; - use crate::pre_asap::query_expr::{BinaryOpKind, GroupKeys, Predicate, Reduction, Source}; - use crate::pre_asap::schema::{DataType, Field, Schema}; - use crate::types::AccuracyTarget; - - /// `[ts, service, value, latency]`. - fn scan() -> QueryExpr { - QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("service", DataType::Utf8, false), - Field::plain("value", DataType::Float64, false), - Field::plain("latency", DataType::Float64, false), - ], - 0, - vec![], - ), - } - } - - fn quantile_agg(by: Vec, col: Option, q: f64) -> QueryExpr { - QueryExpr::Aggregate { - reduction: Reduction::by(by), - measures: vec![AggIntent::Quantile { - col, - q, - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - } - } - - #[test] - fn distinct_column_quantiles_do_not_merge() { - // Grouped (unique_keys present) so the legality gate isn't what's - // blocking the merge — only the differing `col` is. - let a = quantile_agg(vec![1], Some(2), 0.5); - let b = quantile_agg(vec![1], Some(3), 0.5); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - !Rc::ptr_eq(ra, rb), - "distinct-column Quantiles must not be shared" - ); - assert_ne!(ra, rb); - } - - // Two aggregates that differ only in one measure's `FILTER` predicate - // compute different values, so structural sharing must keep them apart. - #[test] - fn filtered_and_unfiltered_aggregates_do_not_merge() { - let a = quantile_agg(vec![1], Some(2), 0.5); - let mut b = quantile_agg(vec![1], Some(2), 0.5); - let QueryExpr::Aggregate { filters, .. } = &mut b else { - unreachable!() - }; - *filters = vec![Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(3)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - })))]; - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(!Rc::ptr_eq(ra, rb), "a filtered measure must not be shared"); - assert_ne!(ra, rb); - } - - #[test] - fn no_unique_keys_means_no_merge_even_when_structurally_identical() { - // Ungrouped (global) aggregate: `by` is empty, so - // `aggregate_output_schema` reports no unique key today — not - // hoistable even though `a` and `b` are structurally identical. - let a = quantile_agg(vec![], Some(2), 0.9); - let b = quantile_agg(vec![], Some(2), 0.9); - assert_eq!(a, b, "fixture sanity: the two DAGs are structurally equal"); - assert!( - !a.output_schema().unwrap().has_unique_key(), - "fixture sanity: an ungrouped aggregate has no provable unique key" - ); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - !Rc::ptr_eq(ra, rb), - "no unique key ⇒ never hoisted, even for an identical structural match" - ); - } - - #[test] - fn median_and_explicit_half_percentile_merge() { - // Two front-end spellings ("median" and "approx_percentile_cont(., - // 0.5)") already lower to the identical `AggIntent::Quantile { q: - // 0.5, .. }` today (see `sql_lowering.rs`'s - // `median_is_the_same_intent_as_an_explicit_half_percentile`) — here - // built directly (grouped, so a unique key is provable) as two - // independently-constructed but structurally identical DAGs, the - // way two different call sites in a workload would produce them. - let median = quantile_agg(vec![1], Some(2), 0.5); - let approx_percentile_cont_half = quantile_agg(vec![1], Some(2), 0.5); - let shared = share_common_sub_dags(vec![ - ("median", median), - ("percentile", approx_percentile_cont_half), - ]); - let [(_, m), (_, p)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - Rc::ptr_eq(m, p), - "median and an explicit 0.5 percentile must merge onto one Rc" - ); - } - - #[test] - fn single_query_shares_its_own_repeated_sub_dag() { - // One query root referencing the same grouped Aggregate on both - // BinaryOp branches — built as two separately-allocated but - // structurally identical sub-DAGs (`.clone()` into two distinct - // `Rc::new` calls), the shape a front end emitting a repeated - // sub-expression would actually produce (no sharing yet). A - // workload of size 1 still interns bottom-up within this one DAG — - // no separate single-query mechanism needed. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), - vector_match: None, - }; - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - let QueryExpr::BinaryOp { lhs, rhs, .. } = root.as_ref() else { - panic!("expected BinaryOp root, got {root:?}"); - }; - assert!( - Rc::ptr_eq(lhs, rhs), - "the two structurally identical branches must collapse onto one Rc" - ); - } - - // ── structural_hash (DAG-aware memoization) ───────────────────────── - - #[test] - fn structural_hash_is_stable_across_cache_states() { - // The hash of a given *value* must not depend on whether its cache - // started warm or cold — memoization changes how much work is - // redone, never what a node's hash actually is. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let mut cold = HashMap::new(); - let mut warm = HashMap::new(); - // Prime `warm` with an unrelated node first, so it's non-empty but - // holds nothing relevant to `agg`. - structural_hash(&scan(), &mut warm); - assert_eq!( - structural_hash(&agg, &mut cold), - structural_hash(&agg, &mut warm), - "hash must be independent of unrelated cache state" - ); - } - - #[test] - fn structural_hash_of_an_internally_shared_dag_matches_the_unshared_equivalent() { - // The same BinaryOp-with-shared-branches shape as - // `dag_node_count_deduplicates_an_internally_shared_sub_dag` below: - // hashing it (however the memoization internally short-circuits the - // second branch) must produce the exact same value as hashing a - // structurally-identical DAG built with *no* sharing at all — the - // whole point of memoization is not changing the answer, only the - // work needed to reach it. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared_root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg.clone()), - vector_match: None, - }; - let unshared_root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), // a second, independently-allocated Rc with an equal value - vector_match: None, - }; - let mut cache = HashMap::new(); - assert_eq!( - structural_hash(&shared_root, &mut cache), - structural_hash(&unshared_root, &mut HashMap::new()), - ); - } - - #[test] - fn structural_hash_memoizes_a_shared_descendant_exactly_once() { - // Direct proof the cache is actually doing its job: hashing a - // BinaryOp whose two branches are the *same* Rc (2 underlying - // nodes: Scan + Aggregate) should populate the cache with exactly - // 2 entries — the shared branch's nodes, cached once each when - // first reached — not a fresh entry (or a fresh, redundant - // recursive walk) for the second occurrence. - let agg = quantile_agg(vec![1], Some(2), 0.5); - let shared = Rc::new(agg); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::clone(&shared), - rhs: Rc::clone(&shared), - vector_match: None, - }; - let mut cache = HashMap::new(); - structural_hash(&root, &mut cache); - assert_eq!( - cache.len(), - 2, - "expected exactly one cache entry per unique node in the shared \ - branch (Aggregate + its Scan child), got {} entries: {:?}", - cache.len(), - cache - ); - } - - // ── dag_node_count ─────────────────────────────────────────────────── - - #[test] - fn dag_node_count_is_the_naive_count_when_nothing_is_shared() { - // scan() alone: 1 node. - assert_eq!(dag_node_count(&scan()), 1); - // quantile_agg's own child is a fresh, unshared scan(): 2 nodes. - assert_eq!(dag_node_count(&quantile_agg(vec![1], Some(2), 0.5)), 2); - } - - #[test] - fn dag_node_count_deduplicates_an_internally_shared_sub_dag() { - // Same shape as `single_query_shares_its_own_repeated_sub_dag`: a - // BinaryOp whose two branches are the *same* Rc after - // `share_common_sub_dags` (2 nodes: Scan + Aggregate) — the root - // itself makes 3 unique nodes total (BinaryOp, Aggregate, Scan), - // not 5 (which a per-path walk / naive serialization, counting the - // shared branch's 2 nodes twice, would report). - let agg = quantile_agg(vec![1], Some(2), 0.5); - let root = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(crate::pre_asap::expr_ir::CompareOpKind::Eq), - lhs: Rc::new(agg.clone()), - rhs: Rc::new(agg), - vector_match: None, - }; - let shared = share_common_sub_dags(vec![("q", root)]); - let [(_, root)] = shared.as_slice() else { - panic!("expected 1 root"); - }; - assert_eq!( - dag_node_count(root), - 3, - "the shared branch's 2 nodes must be counted once, not once per \ - occurrence — got {} for {root:?}", - dag_node_count(root) - ); - } - - #[test] - fn dag_node_count_deduplicates_across_two_workload_roots() { - // Two workload roots sharing one Aggregate after - // `share_common_sub_dags` (the `duplicate_workload_queries_...` - // shape from `crates/integration-tests/tests/cse.rs`, built - // directly here): each root's own `dag_node_count` must report the - // shared sub-DAG's real size once, not double-count anything — - // there's nothing *to* double-count from a single root's own count - // in this case (no root references the shared node twice), so this - // pins the simpler, more common case that a per-candidate cost - // proxy (`CseCandidate::sub-DAG` in `asap-aware-mapping`) actually - // exercises: counting one occurrence's own reachable DAG size. - let a = quantile_agg(vec![1], Some(2), 0.5); - let b = quantile_agg(vec![1], Some(2), 0.5); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(Rc::ptr_eq(ra, rb), "fixture sanity: the two roots merged"); - assert_eq!(dag_node_count(ra), 2); - assert_eq!(dag_node_count(rb), 2); - } - - #[test] - fn dedup_gates_sharing_the_same_as_aggregate() { - // `Dedup { cols }` adds `cols` as a unique key — so two identical - // `Dedup` sub-DAGs over a keyed column *do* merge, exercising the - // legality gate on a non-`Aggregate` node. - let dedup = |cols: Vec| QueryExpr::Dedup { - cols, - child: Rc::new(scan()), - }; - let a = dedup(vec![1]); - let b = dedup(vec![1]); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!( - Rc::ptr_eq(ra, rb), - "Dedup on the same cols has a provable unique key and should merge" - ); - } - - #[test] - fn group_keys_gate_still_prevented_when_partition_by_without_used() { - // Sanity on the module's advertised precedent: a `without(...)` - // grouping stays open (no unique key) even though `by` is - // non-empty-shaped structurally, so two identical `without` groups - // do not merge under the same gate that blocks the ungrouped case. - let without_agg = || QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan()), - }; - let a = without_agg(); - let b = without_agg(); - assert!(!a.output_schema().unwrap().has_unique_key()); - let shared = share_common_sub_dags(vec![("a", a), ("b", b)]); - let [(_, ra), (_, rb)] = shared.as_slice() else { - panic!("expected 2 roots"); - }; - assert!(!Rc::ptr_eq(ra, rb)); - } -} diff --git a/crates/types/src/pre_asap/expr_ir.rs b/crates/types/src/pre_asap/expr_ir.rs index 21ffe21bf..fe373e58f 100644 --- a/crates/types/src/pre_asap/expr_ir.rs +++ b/crates/types/src/pre_asap/expr_ir.rs @@ -1,29 +1,20 @@ -//! Column-reference and scalar-operator vocabulary shared by the whole -//! canonical [`QueryExpr`](super::query_expr::QueryExpr) DAG. -//! -//! Issue #205: the scalar expression shapes (`Column`/`Literal`/`Compare`/…) -//! used to live in a separate, self-recursive `Expr` DAG here, reachable -//! from `QueryExpr` only through wrapper fields (`Predicate`, `ProjectItem`, -//! `SortKey`). They're variants of `QueryExpr` itself now — one recursive -//! DAG, not two type families joined by wrappers — generic over the same -//! column-reference state `C` the rest of `QueryExpr` already carries -//! (issue #179): [`ColumnRef`] (name-based, front-end-emitted) or -//! [`ColumnId`](super::schema::ColumnId) (positional, once bound). -//! -//! What's left here is the vocabulary those scalar variants are built from — -//! [`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`] — the **union** of what the two +//! Column-reference and scalar-operator vocabulary shared by the IR's scalar +//! expressions ([`crate::ir::ScalarExpr`]) and the front ends' unresolved +//! form: [`ColumnRef`] (name-based, front-end-emitted; positional +//! [`ColumnId`](super::schema::ColumnId) once bound), and [`ScalarValue`], +//! [`CompareOpKind`], [`ArithmeticOpKind`] — the **union** of what the two //! front ends need: PromQL contributes `Regex` / `NotRegex` (`=~` / `!~`); SQL //! contributes arithmetic, `CASE`, `IN`, `CAST`, `IS [NOT] NULL`, scalar //! function calls, and the `LIKE` / `ILIKE` comparison family. use serde::{Deserialize, Serialize}; -/// A name-based column reference — the front-end-emitted, unresolved state of -/// [`QueryExpr::Column`](super::query_expr::QueryExpr::Column) (`C = -/// ColumnRef`); the [`SchemaResolver`](super::schema_resolver::SchemaResolver) resolves it to a -/// positional [`ColumnId`](super::schema::ColumnId). This is a logical reference, -/// not schema metadata or a runtime data array. `SampleValue` names the implicit -/// PromQL sample column; `Wildcard` represents an all-columns/rows request. +/// A name-based column reference — the front-end-emitted, unresolved form of +/// [`ScalarExpr::Column`](crate::ir::ScalarExpr::Column); front-end name +/// resolution turns it into a positional [`ColumnId`](super::schema::ColumnId). +/// This is a logical reference, not schema metadata or a runtime data array. +/// `SampleValue` names the implicit PromQL sample column; `Wildcard` represents +/// an all-columns/rows request. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub enum ColumnRef { Named(String), diff --git a/crates/types/src/pre_asap/mod.rs b/crates/types/src/pre_asap/mod.rs index 1a017ec27..c309c7b30 100644 --- a/crates/types/src/pre_asap/mod.rs +++ b/crates/types/src/pre_asap/mod.rs @@ -1,66 +1,34 @@ -//! The canonical pre-ASAP intent algebra IR. +//! Shared vocabulary of the operator IR. The operators themselves live in +//! [`crate::ir`]; this module holds the field types they are built from. //! -//! - [`query_expr`] — the canonical, language- and deployment-independent -//! intent algebra: one recursive [`QueryExpr`] DAG (relational operators -//! *and* scalar expression shapes both, since issue #205) + [`AggIntent`], -//! generic over the column-reference state (positional [`ColumnId`] once -//! bound, name-based [`ColumnRef`] before). -//! - [`agg_intent`] — the aggregation-intent vocabulary. -//! - [`expr_ir`] — the [`ColumnRef`] column-reference type and the scalar -//! operator/literal vocabulary ([`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`]) -//! [`QueryExpr`]'s scalar variants are built from. +//! Operator parameters and schema derivation live in [`crate::ir`]. +//! - [`agg_intent`] — the aggregation-intent vocabulary ([`AggIntent`]). +//! - [`expr_ir`] — [`ColumnRef`] and the scalar literal / operator kinds +//! ([`ScalarValue`], [`CompareOpKind`], [`ArithmeticOpKind`]). //! - [`schema`] — the per-edge [`Schema`] every node carries. -//! - [`schema_resolver`] / [`column_resolution`] — name resolution: turn a `ColumnRef` -//! into a positional `ColumnId` against an in-scope [`Schema`]. -//! - [`resolve`] — binds a whole front-end-emitted [`UnresolvedQueryExpr`] DAG to -//! canonical [`ResolvedQueryExpr`] (issue #179): both front ends -//! (`asap-frontend-promql`, `asap-frontend-sql`) construct `UnresolvedQueryExpr` -//! directly during their own `interpret` step and call -//! [`resolve_root`] on the result — there is no separate per-language -//! relational DAG or converter anymore. -//! - [`canonicalize`] — post-lowering structural normalization of [`QueryExpr`] -//! (issue #34), run by [`resolve_root`]. -//! - [`cse`] — workload-level structural common-subexpression elimination -//! over an already-`resolve_root`'d DAG (issue #212, #222, #223), run -//! *after* `resolve_root` / `canonicalize` and *before* implementation -//! (`asap_aware_mapping::replacement`). -//! -//! Formerly the separate `asap-l2` crate; folded in here since -//! `schema_resolver`/`column_resolution`/`canonicalize`/`resolve` have no -//! front-end-specific logic — they operate directly on this crate's own -//! `QueryExpr`. +//! - [`column_resolution`] — turn a name-based `ColumnRef` into a positional +//! `ColumnId` against a [`Schema`] (used by front-end name resolution). +//! - [`scalar_type_rules`] — shared scalar type and nullability rules. pub mod agg_intent; -pub mod canonicalize; pub mod column_resolution; -pub mod cse; pub mod expr_ir; -pub mod query_expr; -pub mod resolve; pub mod scalar_type_rules; pub mod schema; -pub mod schema_resolver; +pub use crate::ir::operator_properties::{ + AtModifier, BinaryOpKind, ColState, ConcatDiscriminatorKey, DataModel, GroupKeys, GroupSide, + InfoMatcher, JoinKind, PromQLVectorSetOpKind, Reduction, RelationalSetOpKind, SampleKind, + Source, TimeShift, VectorGrouping, VectorMatch, VectorMatchKind, WindowFrame, WindowFrameBound, + WindowFrameOffset, WindowFrameUnits, WindowFuncKind, +}; pub use agg_intent::{ agg_accuracy, agg_is_exact, agg_is_mergeable, default_cardinality, default_quantile, AggIntent, MathFunc, TimeFunc, }; -pub use canonicalize::canonicalize; -pub use column_resolution::{ - output_schema_for_aggregate, resolve_column_ref, resolve_column_refs, resolve_expr, - ResolveError, -}; -pub use cse::share_common_sub_dags; +pub use column_resolution::{resolve_column_ref, resolve_column_refs, ResolveError}; pub use expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -pub use query_expr::{ - aggregate_output_schema, any_measure_filtered, AtModifier, BinaryOpKind, ColState, DataModel, - GroupKeys, GroupSide, InfoMatcher, JoinKind, Predicate, ProjectItem, PromQLVectorSetOpKind, - QueryExpr, QueryExprError, Reduction, RelationalSetOpKind, ResolvedQueryExpr, SampleKind, - SortKey, Source, TimeShift, UnresolvedQueryExpr, VectorGrouping, VectorMatch, VectorMatchKind, - WindowFrame, WindowFrameBound, WindowFrameOffset, WindowFrameUnits, WindowFuncKind, -}; -pub use resolve::{resolve_root, ResolveDAGError}; pub use schema::{ColumnId, DataType, Field, FieldDataType, Schema}; -pub use schema_resolver::{SchemaCatalog, SchemaResolver, UsageDerivedCatalog}; +pub use crate::ir::aggregate_schema::aggregate_output_schema; pub use crate::ir::SchemaDerivationError; diff --git a/crates/types/src/pre_asap/query_expr.rs b/crates/types/src/pre_asap/query_expr.rs deleted file mode 100644 index e389f84a2..000000000 --- a/crates/types/src/pre_asap/query_expr.rs +++ /dev/null @@ -1,2774 +0,0 @@ -//! The canonical pre-ASAP intent algebra IR. -//! -//! Language- and deployment-independent. `Rc`-owned DAG — a child field is -//! `Rc>` rather than `Box>` so a structurally -//! identical sub-expression can be shared (the same `Rc`) across more than -//! one parent, within one query or across a `QueryWorkload` batch, instead of -//! being duplicated. Nothing in this module produces that sharing on its -//! own — construction still allocates a fresh `Rc` per node, the same shape -//! as the old `Box` DAG — a separate CSE pass is what turns two -//! independently constructed, structurally-equal sub-DAGs into two -//! references to one `Rc` (issue #212, #222). Field identity is -//! **positional** (`Aggregate.reduction: Reduction`, wrapping `GroupKeys` -//! for the grouped case), resolved by the [`SchemaResolver`](super::schema_resolver) against -//! the self-contained [`Schema`] carried on each `Scan`. - -use std::rc::Rc; -use std::time::Duration; - -use serde::{Deserialize, Serialize}; -use thiserror::Error; - -use super::agg_intent::AggIntent; -use super::expr_ir::{ArithmeticOpKind, ColumnRef, CompareOpKind, ScalarValue}; -use super::schema::{ColumnId, DataType, Field, FieldDataType, Schema}; - -/// The column-reference resolution state a [`QueryExpr`] DAG carries — -/// [`ColumnId`] (the default, and what the bare `QueryExpr` name has always -/// meant) once the [`SchemaResolver`](super::schema_resolver::SchemaResolver) has resolved every -/// reference positionally, or the front-end-emitted, name-based [`ColumnRef`] -/// before binding. The only place the two states differ in *shape* rather -/// than just in which type fills `C` is [`QueryExpr::Scan`]'s `schema` field: -/// a bound DAG's binding schema is always known (the SchemaResolver is total, so -/// [`ScanSchema`](Self::ScanSchema) `= Schema`); an unresolved front-end -/// `Scan` knows its schema only when the front end already has it without -/// binding — a SQL leaf, catalog-backed (`Some`) — `None` (PromQL) defers to -/// the SchemaResolver, so `ScanSchema = Option`. -pub trait ColState: - Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de> -{ - /// What [`QueryExpr::Scan`]'s `schema` field holds for a DAG in this state. - type ScanSchema: Clone + std::fmt::Debug + PartialEq + Serialize + for<'de> Deserialize<'de>; -} - -impl ColState for ColumnId { - type ScanSchema = Schema; -} - -impl ColState for ColumnRef { - type ScanSchema = Option; -} - -/// Errors from schema derivation over a canonical DAG. -#[derive(Debug, Error)] -pub enum QueryExprError { - #[error("invalid scalar function signature: {0}")] - InvalidScalarSignature(String), - #[error("by-column id {0} out of range (input has {1} columns)")] - InvalidGroupByColumn(ColumnId, usize), - #[error("Concat requires at least one child")] - EmptyConcat, - /// [`QueryExpr::output_schema`] called on (or reached, while recursing, a - /// child that is) one of the scalar variants (issue #205) — those have no - /// independent row schema of their own; a scalar expression's *type* only - /// makes sense against the schema it's embedded in (see `infer_expr_type`, - /// used by `Project`'s own `output_schema` arm instead). - #[error("a scalar expression has no row schema of its own")] - ScalarHasNoRowSchema, - #[error("invalid per-series sample column: {0}")] - InvalidSampleColumn(String), -} - -// ── Leaf / supporting types ─────────────────────────────────────────────────── - -/// Positional grouping keys, shared by every "operate per group" operator: -/// `Aggregate.by` (reduce per group), `Sort.partition_by` (rank per group — -/// including generic `topk`/`bottomk`), and `SQLWindowFunc.partition_by` (window -/// per group). One spelling so grouping has a single home to evolve. Empty -/// (and `by`) = no grouping (a global operation). -/// -/// Heavy-hitter `AggIntent::TopK` carries its grouping here too, via the -/// enclosing `Aggregate.by` (issue #13) — so reduce, rank, and window groupings -/// all share this one type. -/// -/// ## `by` vs `without` (issue #39) -/// -/// The stored [`keys`](Self::keys) are **kept** labels for `by(...)` and -/// **excluded** labels for `without(...)`. PromQL's `without(labels)` groups by -/// every label *except* those listed; the complement can't be enumerated at -/// lowering time under an open (usage-derived) schema, so it is deferred to the -/// runtime — the excluded positions are stored, the kept set stays open. Only -/// `Aggregate` ever produces the `without` form; `Sort` / `SQLWindowFunc` / -/// `PromqlSeriesSample` groupings are always `by`. -/// -/// Serialises as a bare array for the (overwhelmingly common) `by` case — -/// wire-compatible with the `Vec` this field held before — and as -/// `{"without": [...]}` for the exclusion case. -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct GroupKeys { - keys: Vec, - without: bool, -} - -// Not `#[derive(Default)]`: derive would add a `C: Default` bound, but an -// empty key set needs nothing from `C` — `ColumnRef` has no meaningful -// default anyway. -impl Default for GroupKeys { - fn default() -> Self { - Self { - keys: Vec::new(), - without: false, - } - } -} - -impl GroupKeys { - /// An empty key set — a global (ungrouped) operation. - pub fn none() -> Self { - Self::default() - } - /// `by(keys)` — group by exactly these columns. - pub fn by(keys: Vec) -> Self { - Self { - keys, - without: false, - } - } - /// `without(keys)` — group by every label *except* these (issue #39). The - /// kept set is runtime-resolved; only the excluded positions are stored. - pub fn without(keys: Vec) -> Self { - Self { - keys, - without: true, - } - } - /// Whether this is a `without(...)` exclusion grouping. - pub fn is_without(&self) -> bool { - self.without - } - /// The named keys — kept labels for `by`, excluded labels for `without`. - pub fn keys(&self) -> &[C] { - &self.keys - } -} - -impl std::ops::Deref for GroupKeys { - type Target = [C]; - fn deref(&self) -> &Self::Target { - &self.keys - } -} - -impl From> for GroupKeys { - fn from(keys: Vec) -> Self { - Self::by(keys) - } -} - -impl FromIterator for GroupKeys { - fn from_iter>(iter: I) -> Self { - Self::by(iter.into_iter().collect()) - } -} - -impl<'a, C> IntoIterator for &'a GroupKeys { - type Item = &'a C; - type IntoIter = std::slice::Iter<'a, C>; - fn into_iter(self) -> Self::IntoIter { - self.keys.iter() - } -} - -/// Compare directly against a `Vec` so call sites and tests can keep -/// writing `keys == vec![..]` / `assert_eq!(keys, &vec![..])`. A `without` -/// grouping never equals a bare `by` list. -impl PartialEq> for GroupKeys { - fn eq(&self, other: &Vec) -> bool { - !self.without && &self.keys == other - } -} - -/// (De)serialise as a bare array for `by`, or `{"without": [...]}` for the -/// exclusion form — keeping the `by` wire format identical to the old newtype. -/// Borrowed for `Serialize` (no `C: Clone` needed to write one out), owned for -/// `Deserialize` (there's nothing to borrow from). -#[derive(Serialize)] -#[serde(untagged)] -enum GroupKeysReprRef<'a, C> { - By(&'a [C]), - Without { without: &'a [C] }, -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum GroupKeysRepr { - By(Vec), - Without { without: Vec }, -} - -impl Serialize for GroupKeys { - fn serialize(&self, serializer: S) -> Result { - if self.without { - GroupKeysReprRef::Without { - without: self.keys.as_slice(), - } - .serialize(serializer) - } else { - GroupKeysReprRef::By(self.keys.as_slice()).serialize(serializer) - } - } -} - -impl<'de, C: Deserialize<'de>> Deserialize<'de> for GroupKeys { - fn deserialize>(deserializer: D) -> Result { - Ok(match GroupKeysRepr::deserialize(deserializer)? { - GroupKeysRepr::By(keys) => Self::by(keys), - GroupKeysRepr::Without { without } => Self::without(without), - }) - } -} - -/// Which data model a `Source` / `AggIntent` operates over. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum DataModel { - TimeSeries, - Tabular, - Any, -} - -/// The leaf data source of a `Scan`. The schema itself rides on the -/// `Scan.schema` field (SchemaResolver-built); `Source` carries only the leaf's -/// identity. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum Source { - /// Time-series leaf — PromQL / DC lifecycle. Produces `(ts, value, *labels)`. - TimeSeries { metric: String }, - /// Tabular leaf — asap-fusion / future OLAP. Columns ride on `Scan.schema`. - Table { table_ref: String }, -} - -impl Source { - pub fn data_model(&self) -> DataModel { - match self { - Source::TimeSeries { .. } => DataModel::TimeSeries, - Source::Table { .. } => DataModel::Tabular, - } - } -} - -/// Operator on the query-level `BinaryOp` node. Reuses the scalar IR's -/// [`ArithmeticOpKind`] / [`CompareOpKind`] so every arithmetic/comparison -/// operator has exactly one representation (and one `Display`) across the IR. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum BinaryOpKind { - /// Arithmetic — `Add/Sub/Mul/Div/Mod` (shared with `QueryExpr::Arithmetic`). - Arithmetic(ArithmeticOpKind), - /// Comparison — `Eq/Ne/Lt/Le/Gt/Ge` + `Like/ILike/Regex` family (shared - /// with `QueryExpr::Compare`). PromQL keeps the matched series whose - /// comparison holds. - Compare(CompareOpKind), - /// PromQL comparison with the `bool` modifier: every matched series - /// yields 1 or 0 and loses its metric name. A separate variant, not a - /// flag, because only comparisons take `bool`. - CompareBool(CompareOpKind), - /// PromQL vector-set operation. - Set(PromQLVectorSetOpKind), -} - -impl std::fmt::Display for BinaryOpKind { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - BinaryOpKind::Arithmetic(op) => write!(f, "{op}"), - BinaryOpKind::Compare(op) => write!(f, "{op}"), - BinaryOpKind::CompareBool(op) => write!(f, "{op} bool"), - BinaryOpKind::Set(PromQLVectorSetOpKind::And) => f.write_str("AND"), - BinaryOpKind::Set(PromQLVectorSetOpKind::Or) => f.write_str("OR"), - BinaryOpKind::Set(PromQLVectorSetOpKind::Unless) => f.write_str("unless"), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum JoinKind { - Inner, - Left, - Right, - Full, - Cross, - /// Left semi-join — each left row that has **at least one** match, once. - /// `WHERE c IN (SELECT …)` / `WHERE EXISTS (…)` (issue #111). - /// - /// Output schema is the **left's alone**; the right side is a filter, not a - /// source of columns. The join predicate still resolves against the - /// concatenated `left ++ right` schema — its scope is deliberately wider - /// than the node's output. - Semi, - /// Left anti-join — each left row with **no** match. `WHERE NOT EXISTS (…)`. - /// Same schema rule as [`JoinKind::Semi`]. - /// - /// Note this is *not* `NOT IN (SELECT …)`: under SQL's three-valued logic a - /// NULL on the right makes `NOT IN` yield no rows at all, where an anti-join - /// yields every left row. The SQL front end rejects `NOT IN (subquery)` - /// rather than lower it here. - Anti, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum RelationalSetOpKind { - Union, - Intersect, - Except, -} - -/// PromQL vector-set operator used by [`BinaryOpKind::Set`]. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum PromQLVectorSetOpKind { - And, - Or, - Unless, -} - -/// SQL analytic window function (`fn(...) OVER (…)`). Distinct from a streaming -/// time `Window`: this is an analytic frame over already-materialised rows. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFuncKind { - RowNumber, - Rank, - DenseRank, - Lag, - Lead, - /// ClickHouse `lagInFrame`/`leadInFrame`: unlike [`Lag`](Self::Lag)/[`Lead`](Self::Lead), - /// these respect the window frame bounds (NULL/default past the frame edge) - /// rather than reaching arbitrarily far back/forward. Kept as distinct - /// variants so the frame clause is never silently discarded by conflating - /// them with `Lag`/`Lead` (#267). `WindowFuncKind` still has no frame - /// representation, so today these lower and behave exactly like - /// `Lag`/`Lead` — the tag is correct, the frame-respecting behavior isn't - /// implemented yet. See #231 for modeling window frames properly. - LagInFrame, - LeadInFrame, - FirstValue, - LastValue, - /// `NTH_VALUE(expr, n)` — `n` is resolved from the (literal) 2nd argument. - NthValue(Option), - Sum, - Avg, - Count, - Min, - Max, -} - -/// A window's frame-spec (`ROWS`/`RANGE BETWEEN … AND …`) — which rows around -/// the current one an analytic window function reads. `GROUPS` is rejected at -/// lowering time (issue #268): every SQL corpus in this repo uses only `ROWS`, -/// and nothing downstream interprets frame semantics yet, so it isn't worth -/// modelling untested. -/// -/// Meaningless (but harmless) on the rank-only and navigation functions -/// (`ROW_NUMBER`/`RANK`/`DENSE_RANK`/`LAG`/`LEAD`), which ignore the frame per -/// SQL semantics — DataFusion still attaches one, stored here verbatim. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct WindowFrame { - pub units: WindowFrameUnits, - pub start_bound: WindowFrameBound, - pub end_bound: WindowFrameBound, -} - -/// A finite window-frame displacement. Intervals are normalized to Arrow's -/// month/day/nanosecond representation so SQL `RANGE INTERVAL ...` bounds -/// survive lowering without leaking DataFusion types into the canonical IR. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameOffset { - Scalar(ScalarValue), - Interval { - months: i32, - days: i32, - nanoseconds: i64, - }, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameUnits { - /// Boundaries count physical rows: `ROWS BETWEEN 2 PRECEDING AND CURRENT ROW`. - Rows, - /// Boundaries count by value-distance on the (single) `ORDER BY` column: - /// `RANGE BETWEEN INTERVAL '1' HOUR PRECEDING AND CURRENT ROW`. - Range, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WindowFrameBound { - /// `UNBOUNDED PRECEDING` is - /// `Preceding(WindowFrameOffset::Scalar(ScalarValue::Null))`. - Preceding(WindowFrameOffset), - CurrentRow, - /// `UNBOUNDED FOLLOWING` is - /// `Following(WindowFrameOffset::Scalar(ScalarValue::Null))`. - Following(WindowFrameOffset), -} - -/// A symbolic label matcher on the **info metric** side of an -/// [`QueryExpr::PromqlInfoEnrich`] (issue #84). Unlike a `Scan` predicate it is not -/// resolved positionally — it references the info metric's labels (`__name__` -/// picks the metric, the rest constrain data labels), which aren't in the input -/// vector's schema; the post-ASAP realization pass applies it against the info metric. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct InfoMatcher { - pub label: String, - /// One of `Eq` / `Ne` / `Regex` / `NotRegex` (PromQL `=`/`!=`/`=~`/`!~`). - pub op: CompareOpKind, - pub value: String, -} - -/// Series-sampling selection mode (PromQL `limitk` / `limit_ratio`, issue #86). -/// A [`QueryExpr::PromqlSeriesSample`] keeps a *subset of whole series*, unchanged — it does -/// not rank or reduce, so it is distinct from `TopK` and from `Sort → Limit`. -#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum SampleKind { - /// `limitk(k, v)` — up to `k` series per group. Which series survive is - /// deterministic across evaluations but otherwise unspecified (no ordering). - LimitK(usize), - /// `limit_ratio(r, v)` — a deterministic `r`-fraction of series per group. - /// `r ∈ [-1, 1]`; a negative `r` selects the complementary fraction. - LimitRatio(f64), -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct SortKey { - pub expr: QueryExpr, - pub ascending: bool, - pub nulls_first: bool, -} - -/// PromQL vector-match modifier (`on`/`ignoring` + `group_left`/`group_right`). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorMatch { - pub kind: VectorMatchKind, - pub labels: Vec, - pub grouping: Option, -} - -/// PromQL `@` modifier — pins a selector's evaluation time to an anchor instead -/// of the query evaluation time (issue #40). -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum AtModifier { - /// `@ start()` — the query range's start instant. - Start, - /// `@ end()` — the query range's end instant. - End, - /// `@ ` — an absolute instant, milliseconds since the Unix epoch (may be - /// negative). PromQL writes the timestamp in seconds; the front end scales it. - Timestamp(i64), -} - -/// PromQL per-selector **time-shift** modifiers — `offset` and `@` (issue #40). -/// Neither changes a selector's *schema*; both move *when* it is evaluated, so -/// the shift is a pass-through wrapper ([`QueryExpr::TimeShift`]) over the -/// selector rather than a new leaf shape. The runtime resolves the anchor and -/// applies the offset. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)] -pub struct TimeShift { - /// `offset ` as signed milliseconds — a positive value shifts the - /// lookback *back* in time (`offset 5m`), a negative value shifts it - /// *forward* (`offset -5m`). `0` = no offset. - pub offset_ms: i64, - /// `@` anchor; `None` = evaluate at the query time. - pub at: Option, -} - -impl TimeShift { - /// Whether this shift is the identity (no `offset`, no `@`) — the state of - /// every selector that carries neither modifier. - pub fn is_identity(&self) -> bool { - self.offset_ms == 0 && self.at.is_none() - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum VectorMatchKind { - On, - Ignoring, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct VectorGrouping { - pub side: GroupSide, - pub labels: Vec, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum GroupSide { - Left, - Right, -} - -/// A row-level filter predicate (WHERE clause / PromQL label matcher). -/// Boxed: `Predicate` sits directly (not behind a `Vec`) in -/// `Filter.pred`/`Join.pred`/`Aggregate.having`, and `QueryExpr` is -/// self-recursive without further indirection once the scalar variants are -/// part of it — the box is what makes the recursive type's size finite there. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct Predicate(pub Rc>); - -/// Whether any entry of an `Aggregate.filters` vector is set — the shape -/// no binding rule accepts yet (issue #466): a filtered measure stays -/// `KeepPreAsap`, and heavy-hitter promotion skips it. -pub fn any_measure_filtered(filters: &[Option>]) -> bool { - filters.iter().any(Option::is_some) -} - -/// One item in a SELECT projection list. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct ProjectItem { - pub alias: Option, - pub expr: QueryExpr, -} - -// ── Intent algebra IR ──────────────────────────────────────────────────────── - -/// What kind of computation an `Aggregate` node performs — orthogonal to -/// *which* columns it groups by (that's still [`GroupKeys`], inside -/// `Reduce`). Explicit, decided once by whichever pass constructs the node -/// (structural, at front-end lowering time), rather than inferred downstream from -/// whether a grouping-key list happens to be empty or from a neighboring -/// node's shape. See design proposal #165. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub enum Reduction { - /// Collapses input rows via `by` — `by`/`without` semantics are exactly - /// [`GroupKeys`]'s. May still collapse every row into one (an empty, - /// non-`without` `by`) — that's a genuine reduction with zero grouping - /// columns, not "no grouping concept." - Reduce(GroupKeys), - /// No grouping concept at all: preserves one output row per input - /// entity (e.g. a per-series windowed computation with no `by(...)` - /// clause to begin with, because there's no aggregation operator here - /// for such a clause to attach to). Never merges across entities, and - /// never collapses an entity's own row structure (e.g. a time axis) — - /// unlike `Reduce(GroupKeys::without(vec![]))` ("group by every - /// label"), which is still a genuine reduction and does collapse it. - PerEntity, -} - -impl Reduction { - /// Shorthand for the common case — group by these (possibly empty) - /// keys, kept rather than excluded. - pub fn by(keys: Vec) -> Self { - Self::Reduce(GroupKeys::by(keys)) - } - - /// The grouping keys, if this is a genuine reduction — `None` for - /// `PerEntity`, which has no grouping-keys concept to report. - pub fn group_keys(&self) -> Option<&GroupKeys> { - match self { - Self::Reduce(by) => Some(by), - Self::PerEntity => None, - } - } - - /// The grouping keys, panicking if this is `PerEntity` — for call sites - /// (tests, mostly) that already know, from the shape they built or are - /// asserting on, that this must be a genuine reduction. Prefer - /// [`group_keys`](Self::group_keys) wherever the caller can't assume that. - pub fn expect_reduce(&self) -> &GroupKeys { - match self { - Self::Reduce(by) => by, - Self::PerEntity => panic!("expected Reduction::Reduce, got PerEntity"), - } - } -} - -/// A caller-proven compound unique key for a [`QueryExpr::Concat`] (issue -/// #228) — built only via [`QueryExpr::concat_with_discriminator`] / -/// [`ConcatDiscriminatorKey::new`], never by naming `discriminator` directly -/// in a struct literal (both fields are private): from *other Rust code*, -/// the only way to end up with one of these is to hand over a specific -/// column as the discriminator, by name, at the call site. -/// -/// Caveat: this is a Rust-API-level guarantee, not a data-level one. The -/// derived `Deserialize` impl below builds a `ConcatDiscriminatorKey` -/// directly from field values, bypassing `new()`. Deserialization is therefore -/// equivalent to a caller supplying the assertion directly; it does not prove -/// either fact below. An external boundary accepting `QueryExpr` data must -/// reject this field or validate both obligations before treating it as -/// uniqueness evidence. -/// -/// # Soundness -/// -/// `Concat`'s default (see its own doc) is to drop `unique_keys` -/// unconditionally, because a key unique **within** one branch is not unique -/// **across** the concatenation unless the branches' value sets for that key -/// are provably disjoint — nothing about matching schemas or matching -/// per-branch keys establishes that on its own. Two different branches can -/// trivially emit the same `inner_key` value (e.g. two PromQL -/// `histogram_quantiles` branches keyed on `(host, le)` can both produce a -/// `(host, le)` pair for different φ). -/// -/// Prepending `discriminator` restores a compound key only when two facts -/// hold: `inner_key` uniquely identifies rows **within every branch**, and -/// `discriminator`'s value is **guaranteed to differ between branches** — a -/// literal the producer just tagged the branch with (PromQL φ riding along via -/// [`QueryExpr::PromqlRelabel`], a Postgres-style synthetic `GROUPING()` id -/// for `ROLLUP`/`CUBE`, …), never something inferred structurally from the -/// branches' own data — then `discriminator` alone partitions rows into -/// disjoint sets independent of what the branches actually contain, so -/// `(discriminator, inner_key)` is sound even when otherwise-identical -/// `inner_key` values occur in different branches. Neither fact is verified -/// here; both are part of the caller-proven claim. -/// -/// This is a **caller-proven claim, not something `Concat` can verify**: -/// nothing stops a caller from asserting a discriminator that in fact -/// repeats across branches, in which case the resulting `unique_keys` claim -/// is simply wrong — `output_schema` trusts it without checking. The -/// obligation is on the constructor call site, exactly as it is on -/// [`QueryExpr::Dedup`]'s `cols` or any other unverified `unique_keys` -/// producer in this module. -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub struct ConcatDiscriminatorKey { - discriminator: C, - inner_key: Vec, -} - -impl ConcatDiscriminatorKey { - /// The only constructor — `discriminator` must be named explicitly by - /// the caller. See the type's doc for the soundness obligation this - /// puts on that caller. - pub fn new(discriminator: C, inner_key: Vec) -> Self { - Self { - discriminator, - inner_key, - } - } - - pub fn discriminator(&self) -> &C { - &self.discriminator - } - - pub fn inner_key(&self) -> &[C] { - &self.inner_key - } -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(bound(serialize = "C: ColState", deserialize = "C: ColState"))] -pub enum QueryExpr { - /// Outermost leaf. `schema` is the **binding schema** — the resolved column - /// set every positional `ColumnId` in the DAG indexes into, *not* a full - /// description of the runtime row — once bound (`schema: Schema`, always - /// present: the [`SchemaResolver`](super::schema_resolver) is total). Before binding, a - /// front-end-emitted `Scan` (`C = ColumnRef`) knows it only when the front - /// end already has it without binding — a catalog-backed SQL leaf — `None` - /// (PromQL) defers to the SchemaResolver; see [`ColState::ScanSchema`]. Complete - /// when catalog-backed (SQL); for schemaless PromQL the bound schema is - /// usage-derived (the `(ts, value)` floor + the labels the query - /// references), since a metric's label set is open and known only at - /// runtime. That distinction is carried explicitly by - /// [`Schema::closed`](super::schema::Schema::closed) (SQL leaf → `true`, - /// PromQL leaf → `false`). `predicates` are leaf-level row filters (PromQL - /// label matchers, pushed-down `WHERE` conjuncts). - Scan { - source: Source, - #[serde(default)] - predicates: Vec>, - schema: C::ScanSchema, - }, - /// A scalar sub-expression sitting in an **operator-DAG position** — a - /// [`BinaryOp`](Self::BinaryOp) operand for ` op ` - /// thresholds / unit conversions (#35), a - /// [`PromqlVectorFromScalar`](Self::PromqlVectorFromScalar) child, or a - /// whole query's root (a bare PromQL scalar query, e.g. `5`). - /// - /// Formerly its own leaf variant, `PromqlScalar(f64)`. Issue #220: that - /// variant held exactly the same value [`Literal`](Self::Literal) does - /// (every PromQL scalar is `f64`), duplicating it for no reason but - /// *which DAG position* it was allowed to appear in. This wrapper - /// carries that position instead of the value — the inner node is an - /// ordinary scalar sub-language expression (in practice always - /// `Literal(ScalarValue::Float64(_))`, since a front end only ever - /// constructs this fully constant-folded — see - /// [`promql_scalar`](Self::promql_scalar)) — and is what `output_schema`, - /// `canonicalize`, and `resolve` now key off to tell "this operand has - /// its own row schema" from "this is a nested scalar leaf with none," - /// in place of the old `PromqlScalar` vs. `Literal` variant tag. - PromqlScalarBridge(Rc>), - - /// The query **evaluation timestamp** as Unix seconds, exposed by PromQL - /// `time()`. This is not inherently the current wall-clock time: its value - /// is the instant or range-step at which the expression is evaluated. It - /// is also the implicit input of no-argument calendar functions. Issue #46. - EvalTimestamp, - - /// The SQL statement evaluation time (`NOW()` / `CURRENT_TIMESTAMP`) as - /// a SQL [`DataType::Timestamp`]. Kept distinct from [`EvalTimestamp`], - /// whose PromQL `time()` contract is Unix seconds as `Float64`. - CurrentTimestamp, - - /// PromQL `vector(s)` — the scalar→instant-vector bridge. Promotes a - /// scalar-typed child to a single label-less series carrying the scalar's - /// value at every step. Lets a scalar participate where a vector is required - /// (`up or vector(0)` dead-man's-switch). Issue #48. - PromqlVectorFromScalar(Rc>), - - /// PromQL `scalar(v)` — the instant-vector→scalar bridge. Collapses a - /// single-element vector to its value (NaN at runtime if the input is not - /// exactly one series). Lets a vector feed a scalar position (`vector` / - /// aggregation `k` args, thresholds). Issue #48. - PromqlScalarFromVector(Rc>), - - /// ρ — a per-series **label rewrite** (PromQL `label_replace` / - /// `label_join`). Every input row passes through unchanged except for the - /// destination label `dst`, whose new value is computed by `value` — a - /// scalar expression over the child's (source) label columns: - /// `label_replace` → a `label_replace(src, regex, replacement)` function - /// call (regex capture-expansion), `label_join` → a `label_join(sep, srcs…)` - /// concatenation. Sample values and the time axis are untouched. Issue #50. - PromqlRelabel { - /// The label written by this rewrite (PromQL `dst_label`). - dst: String, - value: Rc>, - child: Rc>, - }, - - /// PromQL `info(v, [selector])` — left-join **label enrichment** (#84). Each - /// series in `child` is enriched with labels from the matching info metric(s) - /// (`target_info` by default; `selector`'s `__name__` matchers pick the - /// metric(s), the rest constrain the data labels), joined on their shared - /// identifying labels. Those join keys are the info metric's identifying - /// labels — runtime/metadata-resolved, since an open PromQL schema can't - /// enumerate them — so they are NOT carried here; the post-ASAP realization pass - /// resolves them from the info metric's schema. The output keeps - /// `child`'s (open) schema: the - /// grafted labels appear at runtime. - PromqlInfoEnrich { - #[serde(default)] - selector: Vec, - child: Rc>, - }, - - /// Series-sampling **selection** — PromQL `limitk` / `limit_ratio` (#86). - /// Keeps a subset of whole series per `by` group (empty = global), passing - /// each surviving series through unchanged. Not a ranking (`TopK`) and not a - /// reduction: the output schema equals the child's. - PromqlSeriesSample { - #[serde(default)] - by: GroupKeys, - kind: SampleKind, - child: Rc>, - }, - - /// σ — row-level filter. Output schema = child schema. - Filter { - pred: Predicate, - child: Rc>, - }, - /// π — column projection. - Project { - cols: Vec>, - /// Re-qualifies every output column with this table alias (a derived - /// table / inline view). `None` for an ordinary SELECT list. - #[serde(default)] - qualifier: Option, - child: Rc>, - }, - - /// γ + α — GROUP BY (positional) + aggregate intents. - Aggregate { - reduction: Reduction, - measures: Vec>, - /// Output column names parallel to `measures`. A non-empty entry overrides - /// the synthetic intent-keyed name — SQL threads DataFusion's generated - /// name (e.g. `"sum(metrics.bytes)"`) here so an enclosing `Project` - /// resolves the aggregate output by the name it references. An empty - /// entry (or empty vec) falls back to `AggIntent::output_column`'s name - /// (PromQL's convention). - #[serde(default)] - output_names: Vec, - /// Per-measure row predicates, parallel to `measures` — SQL - /// `FILTER (WHERE …)` semantics (issue #466): only rows where - /// `filters[i]` is `TRUE` update `measures[i]`; groups are still - /// formed from every row. Positional against `child`'s output - /// schema, like `Filter.pred` — not against this node's output like - /// `having`. `None` (or an entry past the end of a shorter vec) is - /// an unfiltered measure, so an empty vec is the pre-#466 shape. - #[serde(default)] - filters: Vec>>, - #[serde(default)] - having: Option>, - child: Rc>, - }, - - /// δ — SQL `DISTINCT` / row deduplication. Positional like every other - /// column reference here; empty = dedup on all columns (`SELECT DISTINCT *`). - Dedup { - cols: Vec, - child: Rc>, - }, - /// ⊕ — exact, n-ary `UNION ALL` of independent branches. Rows are - /// concatenated, never deduplicated; SQL's `UNION`/`INTERSECT`/`EXCEPT` are - /// [`QueryExpr::SetOp`], not this. - /// - /// Used for the branches of one query that a single `Aggregate` cannot - /// express — PromQL `histogram_quantiles` (one branch per φ, issue #109) and - /// SQL `ROLLUP`/`CUBE`/`GROUPING SETS` (one branch per grouping level, issue - /// #118) — as well as for sharded / fan-in plans. - /// - /// **The branches must be union-compatible; nothing here enforces it.** The - /// output schema is the *first* child's, so branches that disagree on a - /// column name or type leave the merged schema silently misdescribing every - /// branch but one. A producer that cannot guarantee compatibility must - /// project the branches into a common shape first. - /// - /// A row may appear in several branches, so no branch's unique key survives - /// the union — `unique_keys` is dropped, as in `SetOp`. **Unless** the - /// constructor asserted `discriminator_unique_key` (issue #228, - /// [`QueryExpr::concat_with_discriminator`]): a caller-proven claim that - /// one column's value is guaranteed distinct per branch, which makes - /// `(discriminator, inner_key)` a sound compound unique key regardless of - /// whether `inner_key` alone repeats across branches. `None` — every - /// ordinary construction path, including the plain struct literal and - /// [`QueryExpr::concat`] — reproduces the old, unconditional-drop - /// behavior exactly; see [`ConcatDiscriminatorKey`]'s doc for the - /// soundness argument and the obligation this puts on whoever asserts it. - /// - /// Empty children is an error ([`QueryExprError::EmptyConcat`]), not an - /// empty relation: there would be no schema to derive. - Concat { - children: Vec>, - /// See the field-level doc above and [`ConcatDiscriminatorKey`]. - #[serde(default)] - discriminator_unique_key: Option>, - }, - - /// Logical join. Post-ASAP binding picks the physical alternative. - Join { - kind: JoinKind, - pred: Predicate, - left: Rc>, - right: Rc>, - }, - SetOp { - kind: RelationalSetOpKind, - all: bool, - left: Rc>, - right: Rc>, - }, - - /// Generic order-by for non-heavy-hitter cases. - /// - /// `partition_by` makes the ordering **per-group**: a non-empty set means - /// "rank within each `partition_by` group" — the semantics behind PromQL - /// `topk by (host) (…)` / SQL `… OVER (PARTITION BY host ORDER BY …)`. It is - /// row-preserving (schema pass-through) and is where the grouping of a - /// generic (non-heavy-hitter) ranking lives, so there is no separate - /// `Partition` node (issue #12: reducing GROUP BY → `Aggregate.by`, per-group - /// ranking → here, parallel sharding → a deployment's own physical - /// stage). Empty = a global order-by. - Sort { - keys: Vec>, - #[serde(default)] - partition_by: GroupKeys, - child: Rc>, - }, - Limit { - n: usize, - offset: usize, - child: Rc>, - }, - - /// PromQL sub-query (`[range:resolution]`). Logical pass-through. - PromqlSubquery { - range: Duration, - #[serde(default)] - resolution: Option, - child: Rc>, - }, - - /// Temporal range selection — "look back `range` of history for this - /// computation." Used for all range-vector functions: `rate`, `increase`, - /// `*_over_time`. The range is distinct from a row-level `Filter`. - /// - /// Structural marker: an `Aggregate` whose direct child is a `TimeRange` - /// is a *per-series* reduction (label-preserving); one whose child is a - /// plain `Scan` or another `Aggregate` is a *cross-series* reduction. - TimeRange { - range: Duration, - child: Rc>, - }, - - /// PromQL `offset` / `@` **time shift** on a selector (issue #40). A - /// pass-through wrapper: it moves *when* `child` is evaluated (the runtime - /// resolves the `@` anchor and applies the offset) but leaves its schema - /// unchanged. Wraps the shifted selector directly — `m offset 1h` → - /// `TimeShift { Scan }`; a ranged selector `m[5m] offset 1h` → - /// `TimeRange { 5m, TimeShift { Scan } }` (the range is taken at the shifted - /// time). A shifted subquery wraps the `PromqlSubquery`, moving its step - /// grid. Never carries the identity shift (the converter emits a bare - /// selector when neither modifier is present). - TimeShift { - shift: TimeShift, - child: Rc>, - }, - - /// SQL analytic window function: `func(args) OVER (PARTITION BY … ORDER BY … - /// ROWS/RANGE BETWEEN …)`. Output schema = child schema + one column named - /// `output_name` (the name the enclosing `Project` references). - SQLWindowFunc { - func: WindowFuncKind, - /// Operand expressions (`LAG(value)` → `[Column(value_id)]`); empty for - /// the rank-only functions (`ROW_NUMBER`/`RANK`/`DENSE_RANK`). - args: Vec>, - partition_by: GroupKeys, - order_by: Vec>, - /// `None` is accepted only for backward compatibility with serialized - /// pre-#268 IR, where the engine's implicit frame was not retained. - /// Newly lowered SQL always carries `Some` with DataFusion's resolved - /// concrete default or explicit frame. - #[serde(default)] - frame: Option, - /// The output column's name — DataFusion's window-expr field name, so a - /// `Project` above resolves it (cf. `Aggregate.output_names`). - output_name: String, - child: Rc>, - }, - - /// Arithmetic / comparison / boolean composition (PromQL binary ops). - BinaryOp { - op: BinaryOpKind, - lhs: Rc>, - rhs: Rc>, - #[serde(default)] - vector_match: Option, - }, - - // ── Scalar expression shapes (issue #205) ─────────────────────────── - // - // Formerly a separate, self-recursive `Expr` DAG, reachable from the - // operator variants above only through wrapper fields (`Predicate`, - // `ProjectItem`, `SortKey`). They're variants of this same DAG now — a - // scalar sub-expression is only ever reachable through one of those same - // wrapper positions (`Filter.pred`, `ProjectItem.expr`, `Aggregate.having`, - // `PromqlRelabel.value`, `SQLWindowFunc.args`, …), which is a *convention* this - // type no longer enforces at compile time the way the old, closed - // `Expr` variant set did — nothing stops constructing, say, a `Scan` - // where a `Compare`'s `left` operand belongs. `output_schema` and every - // scalar-position consumer (`resolve`, `canonicalize`, `infer_expr_type`) - // reject a non-scalar variant found there instead (a `QueryExprError` or - // an `unreachable!`, depending on the call site) — the accepted - // replacement, since the alternative (a marker-trait/sub-enum bound - // restricting which variants are constructible in a scalar position) adds - // real type-level machinery for a distinction every constructor already - // has to get right structurally anyway (a `Filter` is never built with an - // operator sub-DAG as its `pred`). - /// A column reference — unresolved [`ColumnRef`] (front-end-emitted, `C = - /// ColumnRef`) or positional [`ColumnId`] (once bound, `C = ColumnId`). - Column(C), - /// A constant literal value. - Literal(ScalarValue), - /// `left op right` — binary comparison. - Compare { - left: Rc>, - op: CompareOpKind, - right: Rc>, - }, - /// Flat conjunction (logical AND). An empty list is vacuously true. - BoolAnd(Vec>), - /// Flat disjunction (logical OR). An empty list is vacuously false. - BoolOr(Vec>), - /// Logical NOT. - Not(Rc>), - /// `expr IS NULL`. - IsNull(Rc>), - /// `expr IS NOT NULL`. - IsNotNull(Rc>), - /// `CAST(expr AS to)`; `try_cast` for SQL `TRY_CAST` (NULL on failure). - Cast { - expr: Rc>, - to: DataType, - try_cast: bool, - }, - /// `expr [NOT] IN (v1, v2, …)`. - InList { - expr: Rc>, - list: Vec>, - negated: bool, - }, - /// Scalar function call, e.g. `LOWER(col)`, `ABS(x)`. - FunctionCall { - name: String, - args: Vec>, - }, - /// Binary arithmetic: `left op right`. - Arithmetic { - op: ArithmeticOpKind, - left: Rc>, - right: Rc>, - }, - /// SQL `CASE` (both searched and simple forms). `operand` present for the - /// simple form (`CASE expr WHEN …`), absent for searched. - Case { - operand: Option>>, - branches: Vec<(QueryExpr, QueryExpr)>, - else_expr: Option>>, - }, -} - -impl QueryExpr { - /// Construct the [`PromqlScalarBridge`](Self::PromqlScalarBridge) leaf - /// for a bare PromQL numeric literal / folded constant scalar (issue - /// #220) — `Literal(ScalarValue::Float64(v))` at an operator-DAG - /// position. The one constructor every front end / test that used to - /// write `QueryExpr::PromqlScalar(v)` should use instead. - pub fn promql_scalar(v: f64) -> Self { - QueryExpr::PromqlScalarBridge(Rc::new(QueryExpr::Literal(ScalarValue::Float64(v)))) - } - - /// Build an ordinary [`Concat`](Self::Concat) — the ordinary/default - /// construction path every call site should prefer over the bare struct - /// literal: `output_schema` drops `unique_keys` unconditionally, exactly - /// as before issue #228. Use - /// [`concat_with_discriminator`](Self::concat_with_discriminator) instead - /// when the caller can prove branch disjointness via a discriminator - /// column. - pub fn concat(children: Vec>) -> Self { - QueryExpr::Concat { - children, - discriminator_unique_key: None, - } - } - - /// Build a [`Concat`](Self::Concat) whose output schema carries the - /// caller-proven compound unique key `(discriminator, inner_key)` (issue - /// #228). See [`ConcatDiscriminatorKey`]'s doc for the soundness - /// argument and the obligation this puts on the caller — - /// `output_schema` trusts this claim without verifying it: nothing here - /// checks that `inner_key` is unique within every branch or that - /// `discriminator`'s value is distinct between branches. - pub fn concat_with_discriminator( - children: Vec>, - discriminator: C, - inner_key: Vec, - ) -> Self { - QueryExpr::Concat { - children, - discriminator_unique_key: Some(ConcatDiscriminatorKey::new(discriminator, inner_key)), - } - } - - /// The value of a [`PromqlScalarBridge`](Self::PromqlScalarBridge) leaf - /// wrapping a plain `Literal(ScalarValue::Float64(_))` — every one a - /// front end constructs today (see [`promql_scalar`](Self::promql_scalar)). - /// `None` for any other shape, including a `PromqlScalarBridge` wrapping - /// something else (not constructed today, but not precluded by the type). - pub fn as_promql_scalar(&self) -> Option { - match self { - QueryExpr::PromqlScalarBridge(inner) => match inner.as_ref() { - QueryExpr::Literal(ScalarValue::Float64(v)) => Some(*v), - _ => None, - }, - _ => None, - } - } - - /// If this expression is a `BoolAnd`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn conjuncts(&self) -> &[QueryExpr] { - match self { - QueryExpr::BoolAnd(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// If this expression is a `BoolOr`, return its elements; otherwise a - /// single-element slice containing `self`. - pub fn disjuncts(&self) -> &[QueryExpr] { - match self { - QueryExpr::BoolOr(v) => v.as_slice(), - _ => std::slice::from_ref(self), - } - } - - /// Recursively collect every column reference in a **scalar** sub-DAG — - /// used by the [`SchemaResolver`](super::schema_resolver::SchemaResolver) to seed usage-derived - /// leaf schemas, and available to post-ASAP binding for column-lineage / - /// selectivity. - /// `self` must be one of the scalar variants (see the module doc on - /// [`QueryExpr`]'s scalar shapes) — every caller already only reaches - /// this through a scalar-typed position (`Predicate`, `ProjectItem.expr`, - /// …), so an operator variant here indicates a construction bug, not a - /// shape this needs to handle silently. - pub fn columns_referenced(&self) -> Vec<&C> { - match self { - QueryExpr::Column(c) => vec![c], - QueryExpr::Literal(_) => vec![], - QueryExpr::EvalTimestamp => vec![], - QueryExpr::CurrentTimestamp => vec![], - QueryExpr::Compare { left, right, .. } | QueryExpr::Arithmetic { left, right, .. } => { - let mut v = left.columns_referenced(); - v.extend(right.columns_referenced()); - v - } - QueryExpr::BoolAnd(parts) | QueryExpr::BoolOr(parts) => { - parts.iter().flat_map(|e| e.columns_referenced()).collect() - } - QueryExpr::Not(e) | QueryExpr::IsNull(e) | QueryExpr::IsNotNull(e) => { - e.columns_referenced() - } - QueryExpr::Cast { expr, .. } => expr.columns_referenced(), - QueryExpr::InList { expr, list, .. } => { - let mut v = expr.columns_referenced(); - v.extend(list.iter().flat_map(|e| e.columns_referenced())); - v - } - QueryExpr::FunctionCall { args, .. } => { - args.iter().flat_map(|e| e.columns_referenced()).collect() - } - QueryExpr::Case { - operand, - branches, - else_expr, - } => { - let mut v = vec![]; - if let Some(op) = operand { - v.extend(op.columns_referenced()); - } - for (when, then) in branches { - v.extend(when.columns_referenced()); - v.extend(then.columns_referenced()); - } - if let Some(e) = else_expr { - v.extend(e.columns_referenced()); - } - v - } - other => unreachable!( - "columns_referenced called on a non-scalar QueryExpr variant: {other:?}" - ), - } - } -} - -/// The canonical, positional, resolved DAG — what the bare `QueryExpr` name -/// has always meant (the default `C = ColumnId`). Every existing consumer -/// keeps using `QueryExpr` unparameterized; this alias exists only to name -/// the resolved state explicitly at a use site that also wants to name -/// [`UnresolvedQueryExpr`] nearby. -pub type ResolvedQueryExpr = QueryExpr; - -/// The front-end-emitted, name-based, unresolved DAG — -/// `QueryExpr`: front ends construct this directly during their -/// own `interpret` step (issue #179), and the [`SchemaResolver`](super::schema_resolver) -/// resolves it into [`ResolvedQueryExpr`]. -pub type UnresolvedQueryExpr = QueryExpr; - -// `output_schema` needs a fully bound DAG — it reads `Scan.schema` as a plain -// `Schema` and resolves every scalar `Expr::Column` positionally — so it lives -// only on the resolved instantiation, not `impl QueryExpr`. -// Same reasoning as `AggIntent`'s `output_column`/`requires`/`is_per_series` -// (#205): a schema-shaped property that is only meaningful post-binding. -impl QueryExpr { - /// Infer a scalar expression against its input relation using the same - /// canonical rules as projection schema derivation. - pub fn scalar_type(&self, input: &Schema) -> Result<(DataType, bool), QueryExprError> { - infer_expr_type(self, input) - } - - /// Output schema of the root of a canonical DAG. - pub fn output_schema(&self) -> Result { - match self { - QueryExpr::Scan { schema, .. } => Ok(schema.clone()), - - QueryExpr::Aggregate { - reduction, - measures, - output_names, - child, - .. - } => { - let in_schema = child.output_schema()?; - aggregate_output_schema(&in_schema, reduction, measures, output_names) - } - - QueryExpr::Filter { child, .. } - | QueryExpr::Sort { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - // Series sampling keeps a subset of whole series unchanged, so the - // output schema (and row-uniqueness) is exactly the child's (#86). - | QueryExpr::PromqlSeriesSample { child, .. } - // Info enrichment adds runtime info labels — the statically-known - // schema is the child's (open), so it passes through (#84). - | QueryExpr::PromqlInfoEnrich { child, .. } - | QueryExpr::TimeRange { child, .. } - // A time shift (`offset`/`@`) moves *when* the child is evaluated, - // never its columns — schema passes through (#40). - | QueryExpr::TimeShift { child, .. } => child.output_schema(), - - // ρ — relabel preserves every input column and writes one label - // `dst` (Utf8): overwritten in place if it already exists, else - // appended (nullable — a `label_replace` regex non-match leaves it - // unset). The schema stays open (other labels remain runtime-only). - // A rewrite can collapse two label sets into one, so row-uniqueness - // is no longer provable — drop unique_keys. - QueryExpr::PromqlRelabel { dst, child, .. } => { - let mut out = child.output_schema()?; - if let Some(existing) = out.fields.iter_mut().find(|c| c.name == *dst) { - existing.dtype = FieldDataType::Plain(DataType::Utf8); - existing.nullable = true; - } else { - out.fields.push(Field::plain(dst.clone(), DataType::Utf8, true)); - } - out.unique_keys.clear(); - Ok(out) - } - - // π — one output column per projection item. Each item's type is - // inferred from its expression against the child schema; the name - // is the explicit alias or a derived default. A child unique key - // survives exactly when every one of its columns is passed through - // as a bare `Field` item (possibly reordered or aliased). Derived - // expressions cannot carry key identity. `time_index` is re-found - // by name. - QueryExpr::Project { cols, qualifier, child } => { - let in_schema = child.output_schema()?; - let columns: Vec = cols - .iter() - .enumerate() - .map(|(i, item)| { - let (dtype, nullable) = infer_expr_type(&item.expr, &in_schema)?; - let name = item - .alias - .clone() - .unwrap_or_else(|| default_proj_name(&item.expr, i, &in_schema)); - let c = Field::plain(name, dtype, nullable); - // A derived table re-qualifies its output columns with - // its alias, so `t.col` (and a join over two derived - // tables) resolves to the right relation. - Ok(match qualifier { - Some(q) => c.with_table(q), - None => c, - }) - }) - .collect::, QueryExprError>>()?; - let time_index = columns.iter().position(|c| c.name == "ts"); - let unique_keys = in_schema - .unique_keys - .iter() - .filter_map(|key| { - key.iter() - .map(|input_col| { - cols.iter().position(|item| { - matches!(&item.expr, QueryExpr::Column(col) if col == input_col) - }) - }) - .collect::>>() - }) - .collect(); - Ok(Schema { - fields: columns, - time_index, - unique_keys, - // Projection enumerates exactly its items → closed. - closed: true, - }) - } - - QueryExpr::Dedup { cols, child } => { - let mut out = child.output_schema()?; - // Deduplicating on `cols` makes them a unique key of the result. - if !cols.is_empty() { - out.add_unique_key(cols.clone()); - } - Ok(out) - } - - // ⊕ — the branches are union-compatible by construction, so the - // output shape is the first child's. A row can appear in more than - // one branch, so no key of one branch is a key of the union: drop - // unique_keys, exactly as `SetOp` does — unless the constructor - // asserted `discriminator_unique_key` (issue #228), in which case - // `(discriminator, inner_key)` becomes the sole unique key. That - // assertion is trusted verbatim here, never checked: see - // `ConcatDiscriminatorKey`'s doc for the soundness argument and - // whose obligation it is. - QueryExpr::Concat { - children, - discriminator_unique_key, - } => { - let mut s = children - .first() - .ok_or(QueryExprError::EmptyConcat) - .and_then(|c| c.output_schema())?; - s.unique_keys.clear(); - if let Some(key) = discriminator_unique_key { - let mut compound = vec![*key.discriminator()]; - compound.extend(key.inner_key().iter().copied()); - s.add_unique_key(compound); - } - Ok(s) - } - // Set operations are union-compatible: both sides share the left's - // column shape, so the output schema is the left's. (Row identity - // is not preserved across a UNION, so unique_keys are dropped.) - QueryExpr::SetOp { left, .. } => { - let mut s = left.output_schema()?; - s.unique_keys.clear(); - Ok(s) - } - // ⋈ — output is the concatenation of both inputs' columns. Outer - // joins make the non-preserved side nullable. Post-join row - // identity isn't provable in general, so unique_keys reset. - QueryExpr::Join { - kind, left, right, .. - } => { - let l = left.output_schema()?; - let r = right.output_schema()?; - // Semi / anti joins filter the left side; the right contributes - // no columns, so the output is the left's schema unchanged. Row - // identity *is* preserved (each left row appears at most once), - // but a left row can be dropped, so unique_keys still reset. - if matches!(kind, JoinKind::Semi | JoinKind::Anti) { - return Ok(Schema { - unique_keys: Vec::new(), - ..l - }); - } - let (left_null, right_null) = match kind { - JoinKind::Left => (false, true), - JoinKind::Right => (true, false), - JoinKind::Full => (true, true), - JoinKind::Inner | JoinKind::Cross => (false, false), - JoinKind::Semi | JoinKind::Anti => unreachable!("handled above"), - }; - let l_len = l.fields.len(); - let mut columns = Vec::with_capacity(l_len + r.fields.len()); - columns.extend(l.fields.iter().cloned().map(|mut c| { - c.nullable |= left_null; - c - })); - columns.extend(r.fields.iter().cloned().map(|mut c| { - c.nullable |= right_null; - c - })); - let time_index = l.time_index.or(r.time_index.map(|i| i + l_len)); - Ok(Schema { - fields: columns, - time_index, - unique_keys: Vec::new(), - // The concatenation is complete only if both sides are. - closed: l.closed && r.closed, - }) - } - // ψ-analytic — child schema + one appended window-output column. - QueryExpr::SQLWindowFunc { - func, - args, - output_name, - child, - .. - } => { - let mut out = child.output_schema()?; - // First operand's (dtype, nullable) from the child schema, owned - // so the borrow ends before we append. - let arg = args.first().and_then(|a| match a { - QueryExpr::Column(id) => out.fields.get(*id), - _ => None, - }); - let arg_dtype = || { - arg.and_then(|c| c.plain_dtype().cloned()) - .unwrap_or(DataType::Float64) - }; - let (dtype, nullable) = match func { - WindowFuncKind::RowNumber - | WindowFuncKind::Rank - | WindowFuncKind::DenseRank - | WindowFuncKind::Count => (DataType::Int64, false), - WindowFuncKind::Sum | WindowFuncKind::Avg => (DataType::Float64, true), - // Navigation funcs: arg type, nullable (boundary rows are NULL). - WindowFuncKind::Lag - | WindowFuncKind::Lead - | WindowFuncKind::LagInFrame - | WindowFuncKind::LeadInFrame - | WindowFuncKind::FirstValue - | WindowFuncKind::LastValue - | WindowFuncKind::NthValue(_) => (arg_dtype(), true), - WindowFuncKind::Min | WindowFuncKind::Max => { - (arg_dtype(), arg.is_none_or(|c| c.nullable)) - } - }; - out.fields - .push(Field::plain(output_name.clone(), dtype, nullable)); - Ok(out) - } - - // A scalar bridge has no series — model it as a single `value` - // column so it can sit as a `BinaryOp` operand. Both scalar - // leaves — a bridged scalar sub-expression and the eval time — - // are a single `value` column with no labels. Every - // `PromqlScalarBridge` constructed today wraps a plain - // `Literal(Float64)` (issue #220), so the schema doesn't need to - // inspect the inner node. - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp => Ok(Schema { - fields: vec![Field::plain("value", DataType::Float64, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - QueryExpr::CurrentTimestamp => Ok(Schema { - fields: vec![Field::plain("value", DataType::Timestamp, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - // `vector(s)` yields a label-less instant vector: the (ts, value) - // floor and nothing else. `closed` — its full label set (empty) is - // known statically (#48). - QueryExpr::PromqlVectorFromScalar(_) => Ok(Schema { - fields: vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ], - time_index: Some(0), - unique_keys: Vec::new(), - closed: true, - }), - - // `scalar(v)` collapses to a single `value`, no time index — the same - // scalar shape as a constant or `time()` (#48). - QueryExpr::PromqlScalarFromVector(_) => Ok(Schema { - fields: vec![Field::plain("value", DataType::Float64, false)], - time_index: None, - unique_keys: Vec::new(), - closed: true, - }), - - // The output shape of ` op ` (or ` op - // `) is the vector side's — a scalar operand (a constant or - // `time()`) contributes only its value, no labels. Prefer the - // non-scalar side. - QueryExpr::BinaryOp { lhs, rhs, op, vector_match } => { - fn scalar(expression: &QueryExpr) -> bool { - match expression { - QueryExpr::PromqlScalarBridge(_) | QueryExpr::EvalTimestamp | QueryExpr::PromqlScalarFromVector(_) => true, - QueryExpr::BinaryOp { lhs, rhs, .. } => scalar(lhs) && scalar(rhs), - _ => false, - } - } - let left = lhs.output_schema()?; - let right = rhs.output_schema()?; - if scalar(lhs) { return Ok(right); } - let mut output = left; - let grouping = vector_match.as_ref().and_then(|m| m.grouping.as_ref()); - let right_rows = matches!(op, BinaryOpKind::Set(PromQLVectorSetOpKind::Or)) - || matches!(grouping, Some(g) if g.side == GroupSide::Right); - let mut additions = Vec::new(); - if right_rows { - additions.extend(right.fields.iter().filter(|c| c.dtype == DataType::Utf8).cloned()); - } - if let Some(grouping) = grouping { - additions.extend(grouping.labels.iter().map(|name| Field::plain(name.clone(), DataType::Utf8, true))); - } - for column in additions { - if !output.fields.iter().any(|c| c.name == column.name) { - output.fields.push(column); - } - } - Ok(output) - }, - - // The scalar variants (issue #205) — see `QueryExprError::ScalarHasNoRowSchema`. - QueryExpr::Column(_) - | QueryExpr::Literal(_) - | QueryExpr::Compare { .. } - | QueryExpr::BoolAnd(_) - | QueryExpr::BoolOr(_) - | QueryExpr::Not(_) - | QueryExpr::IsNull(_) - | QueryExpr::IsNotNull(_) - | QueryExpr::Cast { .. } - | QueryExpr::InList { .. } - | QueryExpr::FunctionCall { .. } - | QueryExpr::Arithmetic { .. } - | QueryExpr::Case { .. } => Err(QueryExprError::ScalarHasNoRowSchema), - } - } -} - -/// Output schema of a *per-series* window/range reduction (`rate`/`increase`, -/// or an `*_over_time` reducer under a time `Window`). Such a reduction emits -/// one value per series, so every label column of `input` is preserved and only -/// the sample value is replaced — kept named `value` so the PromQL sample-value -/// convention (and any outer `SampleValue` reference) still resolves it by name. -fn per_series_reduction_schema(input: &Schema, agg: &AggIntent) -> Result { - let vi = if let Some(index) = agg.input_cols().first() { - *index - } else { - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, input) - .map_err(|error| QueryExprError::InvalidSampleColumn(error.to_string()))? - }; - if !matches!( - input.fields.get(vi).map(|column| &column.dtype), - Some(FieldDataType::Plain(DataType::Float64 | DataType::Int64)) - ) { - return Err(QueryExprError::InvalidSampleColumn(format!( - "column {vi} is not numeric" - ))); - } - let mut columns = input.fields.clone(); - { - let mut out = agg.output_column(&columns[vi]); - out.name = "value".into(); - // A per-series range reduction produces a PromQL sample value, which is - // always `float64` — override the reducer's own output dtype so - // `count_over_time` (whose `Count` intent types `Int64`) matches every - // other range reducer instead of leaking an `Int64` value column (#69). - out.dtype = FieldDataType::Plain(DataType::Float64); - columns[vi] = out; - } - Ok(Schema { - fields: columns, - time_index: input.time_index, - unique_keys: input.unique_keys.clone(), - // Per-series reduction is label-preserving: it inherits its input's - // completeness (an open scan stays open; a closed one stays closed). - closed: input.closed, - }) -} - -/// The output schema of an `Aggregate { reduction, measures }` over `in_schema` — -/// the **single** canonical derivation shared by -/// [`QueryExpr::output_schema`]'s `Aggregate` arm and the converter's -/// HAVING-resolution path (`column_resolution::output_schema_for_aggregate`), -/// so the two can never drift (issue #41). -/// -/// `Reduction::PerEntity` selects the label-preserving -/// [`per_series_reduction_schema`] (`rate`/`increase`/`*_over_time`) instead -/// of the cross-series `by ++ measures` shape. Which one applies is read directly -/// off `reduction` — decided once, at construction, by whoever built the -/// `Aggregate` node (issue #165) — not re-derived here from `by`/child shape. -pub fn aggregate_output_schema( - in_schema: &Schema, - reduction: &Reduction, - measures: &[AggIntent], - output_names: &[String], -) -> Result { - let by = match reduction { - Reduction::PerEntity => { - debug_assert_eq!( - measures.len(), - 1, - "a per-entity reduction is single-aggregate" - ); - return per_series_reduction_schema(in_schema, &measures[0]); - } - Reduction::Reduce(by) => by, - }; - - // `without(excluded)` groups by every label *except* those listed: the kept - // labels are the input's label columns minus the excluded positions (and the - // ts / sample-value columns), and the schema stays **open** because the full - // runtime label set isn't known. The `by(...)` path instead enumerates its - // kept columns and freezes to closed (issue #39). - if by.is_without() { - return without_output_schema(in_schema, by.keys(), measures, output_names); - } - - let mut out_cols: Vec = Vec::with_capacity(by.len() + measures.len()); - for &id in by.keys() { - let c = in_schema - .fields - .get(id) - .ok_or(QueryExprError::InvalidGroupByColumn( - id, - in_schema.fields.len(), - ))?; - out_cols.push(c.clone()); - } - let value_col_idx = - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, in_schema) - .ok() - .or_else(|| (0..in_schema.fields.len()).find(|i| !by.contains(i))); - let probe = value_col_idx - .and_then(|i| in_schema.fields.get(i)) - .cloned() - .unwrap_or_else(|| Field::plain("value", DataType::Float64, false)); - // Each reducer types off its own input column (`SUM(bytes)` vs `AVG(latency)` - // in one node); `None` falls back to the sample-value probe (PromQL's - // single-column convention). A non-empty `output_names[i]` overrides the - // synthetic output column name. - for (i, intent) in measures.iter().enumerate() { - // `count_values("l", v)` emits TWO columns: the synthesized `Utf8` label - // `l` (the stringified sample value it groups by) and the per-value - // count. If `l` collides with a group-by key of the same name, PromQL's - // synthesized label takes precedence — emit a single column, never a - // duplicate. - if let AggIntent::CountValues { label } = intent { - if !out_cols.iter().any(|c| c.name == *label) { - out_cols.push(Field::plain(label.clone(), DataType::Utf8, false)); - } - let mut cnt = intent.output_column(&probe); - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - cnt.name = name.clone(); - } - out_cols.push(cnt); - continue; - } - // Only the output *type* is read from here, so the leading column is - // enough for the multi-column intents: `Cardinality` and `PearsonCorr` - // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); - let mut out = intent.output_column(in_col); - // A global extremum emits NULL for an empty input, even if its input - // column is non-nullable. Grouped extrema only emit existing groups. - if by.is_empty() && matches!(intent, AggIntent::Min { .. } | AggIntent::Max { .. }) { - out.nullable = true; - } - if let Some((arg, _)) = intent - .arg_selector_columns(in_schema) - .map_err(QueryExprError::InvalidScalarSignature)? - { - out.dtype = in_schema.fields[arg].dtype.clone(); - out.nullable = in_schema.fields[arg].nullable; - } - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - out.name = name.clone(); - } - out_cols.push(out); - } - // `count_values` groups by (by-keys ∪ the synthesized value label), so the - // by-keys alone are not a unique key — be conservative and claim none. - let has_count_values = measures - .iter() - .any(|a| matches!(a, AggIntent::CountValues { .. })); - let unique_keys = if by.is_empty() || has_count_values { - Vec::new() - } else { - vec![(0..by.len()).collect()] - }; - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys, - // A cross-series aggregate enumerates exactly `by ++ measures`, so its output - // is closed even over an open input — this is where an open schema - // freezes to closed. - closed: true, - }) -} - -/// Output schema of a `without(excluded)` aggregate: the kept labels (every -/// input label column except the `excluded` positions, the time axis, and the -/// sample-value column) followed by the aggregate output column(s). Unlike the -/// `by` path this stays **open** — the excluded set is enumerable but the kept -/// set is not (the runtime carries labels the usage-derived schema never saw), -/// so the schema can't freeze to closed and claims no unique key (issue #39). -fn without_output_schema( - in_schema: &Schema, - excluded: &[ColumnId], - measures: &[AggIntent], - output_names: &[String], -) -> Result { - for &id in excluded { - if id >= in_schema.fields.len() { - return Err(QueryExprError::InvalidGroupByColumn( - id, - in_schema.fields.len(), - )); - } - } - // A nested aggregate renames the sample value (`sum by (le) (…)` → `sum`); - // it is still the value, not a kept label. - let value = - super::column_resolution::resolve_column_ref(&ColumnRef::SampleValue, in_schema).ok(); - let mut out_cols: Vec = Vec::new(); - for (i, col) in in_schema.fields.iter().enumerate() { - let is_time = in_schema.time_index == Some(i); - if !is_time && value != Some(i) && !excluded.contains(&i) { - out_cols.push(col.clone()); - } - } - let probe = value - .and_then(|i| in_schema.fields.get(i)) - .cloned() - .unwrap_or_else(|| Field::plain("value", DataType::Float64, false)); - for (i, intent) in measures.iter().enumerate() { - // Only the output *type* is read from here, so the leading column is - // enough for the multi-column intents: `Cardinality` and `PearsonCorr` - // both have a fixed output type that ignores it. - let in_col = intent - .input_cols() - .first() - .and_then(|id| in_schema.fields.get(*id)) - .unwrap_or(&probe); - let mut out = intent.output_column(in_col); - if let Some((arg, _)) = intent - .arg_selector_columns(in_schema) - .map_err(QueryExprError::InvalidScalarSignature)? - { - out.dtype = in_schema.fields[arg].dtype.clone(); - out.nullable = in_schema.fields[arg].nullable; - } - if let Some(name) = output_names.get(i).filter(|s| !s.is_empty()) { - out.name = name.clone(); - } - out_cols.push(out); - } - Ok(Schema { - fields: out_cols, - time_index: None, - unique_keys: Vec::new(), - // The kept label set is runtime-only, so — unlike `by` — this does not - // freeze the open schema to closed. - closed: false, - }) -} - -/// Infer the `(DataType, nullable)` a scalar [`QueryExpr`] produces against an -/// input [`Schema`]. Used by `Project` schema derivation. Approximate here: -/// unknown columns and bare `FunctionCall`s fall back to a permissive default -/// (post-ASAP binding refines with a real function/type registry). `expr` -/// must be one of the scalar variants (issue #205) — an operator variant here -/// is a construction bug, not a shape this needs to handle silently. -fn infer_expr_type( - expr: &QueryExpr, - schema: &Schema, -) -> Result<(DataType, bool), QueryExprError> { - Ok(match expr { - QueryExpr::CurrentTimestamp => (DataType::Timestamp, false), - QueryExpr::Column(id) => match schema.fields.get(*id) { - Some(c) => match c.plain_dtype() { - Some(dtype) => (dtype.clone(), c.nullable), - // Summary state is not a scalar value: it has to be read - // out (estimated / finalized) before an expression can use it. - None => { - return Err(QueryExprError::InvalidScalarSignature(format!( - "column `{}` carries summary state and cannot be read as a value", - c.name - ))) - } - }, - None => (DataType::Float64, true), - }, - QueryExpr::Literal(s) => match s { - ScalarValue::Int64(_) => (DataType::Int64, false), - ScalarValue::Float64(_) => (DataType::Float64, false), - ScalarValue::Utf8(_) => (DataType::Utf8, false), - ScalarValue::Boolean(_) => (DataType::Bool, false), - ScalarValue::Null => (DataType::Null, true), - ScalarValue::Interval { .. } => (DataType::Interval, false), - }, - // Boolean-valued expressions (SQL three-valued logic → nullable). - QueryExpr::Compare { .. } - | QueryExpr::BoolAnd(_) - | QueryExpr::BoolOr(_) - | QueryExpr::Not(_) - | QueryExpr::IsNull(_) - | QueryExpr::IsNotNull(_) - | QueryExpr::InList { .. } => (DataType::Bool, true), - QueryExpr::Arithmetic { op, left, right } => { - let (lt, ln) = infer_expr_type(left, schema)?; - let (rt, rn) = infer_expr_type(right, schema)?; - // Temporal subtraction yields a fixed duration with a unit, not a - // calendar interval or a floating-point number. Until the IR can - // preserve that unit, fail instead of publishing a numeric schema. - if matches!(op, ArithmeticOpKind::Sub) - && matches!(lt, DataType::Date | DataType::Timestamp) - && matches!(rt, DataType::Date | DataType::Timestamp) - { - return Err(QueryExprError::InvalidScalarSignature( - "temporal subtraction produces an unsupported duration type".into(), - )); - } - - // Operand order is not checked: the orders that are not valid SQL - // (`Interval - Timestamp`) are rejected by the planner upstream, so - // a pair rule stays as small as the numeric one it sits beside. - let dtype = match (<, &rt) { - // SQL unary minus lowers to -1 * expression, including intervals. - (DataType::Int64, DataType::Interval) | (DataType::Interval, DataType::Int64) - if matches!(op, ArithmeticOpKind::Mul) => - { - DataType::Interval - } - (DataType::Timestamp, DataType::Interval) - | (DataType::Interval, DataType::Timestamp) => DataType::Timestamp, - (DataType::Date, DataType::Interval) | (DataType::Interval, DataType::Date) => { - DataType::Date - } - (DataType::Interval, DataType::Interval) => DataType::Interval, - (DataType::Int64, DataType::Int64) => DataType::Int64, - _ => DataType::Float64, - }; - (dtype, ln || rn) - } - QueryExpr::Cast { to, try_cast, expr } => { - let (_, nullable) = infer_expr_type(expr, schema)?; - (to.clone(), *try_cast || nullable) - } - QueryExpr::FunctionCall { name, args } => { - if name == "asap_element_access" { - super::scalar_type_rules::element_access_type(args, schema) - .map_err(QueryExprError::InvalidScalarSignature)? - } else if name == "asap_struct_field" { - super::scalar_type_rules::struct_field_type(args, schema) - .map_err(QueryExprError::InvalidScalarSignature)? - } else if let Some(function) = - super::scalar_type_rules::MapScalarFunction::from_name(name) - { - let arguments = args - .iter() - .map(|arg| infer_expr_type(arg, schema)) - .collect::, _>>()?; - function - .output_type(&arguments) - .map_err(QueryExprError::InvalidScalarSignature)? - } else { - // Legacy unknown functions retain their existing policy. - (DataType::Float64, true) - } - } - QueryExpr::Case { - branches, - else_expr, - .. - } => { - if let Some((_, then)) = branches.first() { - (infer_expr_type(then, schema)?.0, true) - } else if let Some(other) = else_expr { - infer_expr_type(other, schema)? - } else { - (DataType::Null, true) - } - } - other => { - unreachable!("infer_expr_type called on a non-scalar QueryExpr variant: {other:?}") - } - }) -} - -/// Default output-column name for a projection item with no explicit alias: -/// a bare column keeps its (schema) name; anything else gets `col_{i}`. -fn default_proj_name(expr: &QueryExpr, idx: usize, schema: &Schema) -> String { - match expr { - QueryExpr::Column(id) => schema - .fields - .get(*id) - .map(|c| c.name.clone()) - .unwrap_or_else(|| format!("col_{idx}")), - _ => format!("col_{idx}"), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::expr_ir::{ArithmeticOpKind, CompareOpKind}; - use crate::types::AccuracyTarget; - - fn col(name: &str, dtype: DataType, nullable: bool) -> Field { - Field::plain(name, dtype, nullable) - } - - /// Shifting an instant by a duration stays an instant, and shifting a date - /// stays a date — neither falls through to the numeric default, which is - /// what `l_shipdate + INTERVAL '30' DAY` would otherwise be typed as. - #[test] - fn interval_arithmetic_keeps_the_temporal_type() { - let schema = Schema::new(vec![ - col("ts", DataType::Timestamp, false), - col("d", DataType::Date, false), - ]); - let thirty_days = || { - Rc::new(QueryExpr::Literal(ScalarValue::Interval { - months: 0, - days: 30, - nanos: 0, - })) - }; - let shift = |column, op| QueryExpr::Arithmetic { - op, - left: Rc::new(QueryExpr::Column(column)), - right: thirty_days(), - }; - - assert_eq!( - shift(0, ArithmeticOpKind::Add) - .scalar_type(&schema) - .unwrap() - .0, - DataType::Timestamp - ); - assert_eq!( - shift(1, ArithmeticOpKind::Sub) - .scalar_type(&schema) - .unwrap() - .0, - DataType::Date - ); - assert_eq!( - QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: thirty_days(), - right: thirty_days(), - } - .scalar_type(&schema) - .unwrap() - .0, - DataType::Interval - ); - } - - fn scan( - columns: Vec, - time_index: Option, - uk: Vec>, - ) -> QueryExpr { - QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema { - fields: columns, - time_index, - unique_keys: uk, - closed: true, - }, - } - } - - #[test] - fn project_preserves_unique_keys_that_are_passed_through() { - let input = Rc::new(scan( - vec![ - col("tenant", DataType::Utf8, false), - col("region", DataType::Utf8, false), - col("value", DataType::Int64, false), - ], - None, - vec![vec![0, 1]], - )); - let projected = QueryExpr::Project { - cols: vec![ - ProjectItem { - alias: Some("r".into()), - expr: QueryExpr::Column(1), - }, - ProjectItem { - alias: Some("t".into()), - expr: QueryExpr::Column(0), - }, - ProjectItem { - alias: None, - expr: QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Rc::new(QueryExpr::Column(2)), - right: Rc::new(QueryExpr::Literal(ScalarValue::Int64(1))), - }, - }, - ], - qualifier: None, - child: input, - }; - - assert_eq!( - projected.output_schema().unwrap().unique_keys, - vec![vec![1, 0]] - ); - } - - #[test] - fn project_drops_a_unique_key_when_a_key_column_is_omitted() { - let input = Rc::new(scan( - vec![ - col("tenant", DataType::Utf8, false), - col("region", DataType::Utf8, false), - ], - None, - vec![vec![0, 1]], - )); - let projected = QueryExpr::Project { - cols: vec![ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }], - qualifier: None, - child: input, - }; - - assert!(projected.output_schema().unwrap().unique_keys.is_empty()); - } - - #[test] - fn legacy_window_json_without_frame_deserializes_as_unspecified() { - let window = QueryExpr::SQLWindowFunc { - func: WindowFuncKind::RowNumber, - args: vec![], - partition_by: GroupKeys::by(vec![]), - order_by: vec![], - frame: Some(WindowFrame { - units: WindowFrameUnits::Range, - start_bound: WindowFrameBound::Preceding(WindowFrameOffset::Scalar( - ScalarValue::Null, - )), - end_bound: WindowFrameBound::CurrentRow, - }), - output_name: "row_number".into(), - child: Rc::new(scan(vec![col("v", DataType::Int64, false)], None, vec![])), - }; - let mut json = serde_json::to_value(window).unwrap(); - json.get_mut("SQLWindowFunc") - .and_then(serde_json::Value::as_object_mut) - .unwrap() - .remove("frame"); - - let decoded: QueryExpr = serde_json::from_value(json).unwrap(); - assert!(matches!( - decoded, - QueryExpr::SQLWindowFunc { frame: None, .. } - )); - } - - /// A row can appear in more than one branch, so no branch's unique key is a - /// key of the union. `Concat` took the first child's schema verbatim, which - /// let a `Dedup`'s key leak out and claim a uniqueness the merged rows do - /// not have — `unique_keys` feeds CSE's producer-sharing legality check. - #[test] - fn merge_drops_the_branches_unique_keys() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![], - )), - }; - assert_eq!( - branch().output_schema().unwrap().unique_keys, - vec![vec![0]], - "a Dedup branch does have a unique key on its own" - ); - - let merged = QueryExpr::concat(vec![branch(), branch()]); - let schema = merged.output_schema().unwrap(); - assert!( - schema.unique_keys.is_empty(), - "the union of two deduplicated branches is not deduplicated" - ); - // The column shape is still the first branch's. - assert_eq!(schema.fields.len(), 2); - } - - /// Same rule as `SetOp`, which already dropped them. - #[test] - fn merge_and_setop_agree_on_unique_keys() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - let merged = QueryExpr::concat(vec![branch(), branch()]); - let setop = QueryExpr::SetOp { - kind: RelationalSetOpKind::Union, - all: true, - left: Rc::new(branch()), - right: Rc::new(branch()), - }; - assert_eq!( - merged.output_schema().unwrap().unique_keys, - setop.output_schema().unwrap().unique_keys, - ); - } - - #[test] - fn an_empty_merge_has_no_schema() { - assert!(matches!( - QueryExpr::concat(vec![]).output_schema(), - Err(QueryExprError::EmptyConcat) - )); - } - - /// Issue #228: a `Concat` built via `concat_with_discriminator` gets a - /// sound compound `(discriminator, inner_key)` unique key, even though - /// each branch's own `inner_key` alone repeats across branches (exactly - /// the shape `merge_drops_the_branches_unique_keys` shows is unsafe - /// *without* a discriminator). - #[test] - fn discriminator_override_produces_a_compound_unique_key() { - // Two branches, each individually deduplicated on column 0 (`k`) — - // but, per `merge_drops_the_branches_unique_keys`, that alone proves - // nothing about the union. Field 1 (`branch_id`) stands in for a - // discriminator the constructor has separately proven distinct per - // branch (PromQL φ, a synthetic `GROUPING()` id, ...) — this - // schema-level test only checks the shape `output_schema` derives - // from asserting one, not how a real caller proves distinctness. - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan( - vec![ - col("k", DataType::Utf8, false), - col("branch_id", DataType::Int64, false), - ], - None, - vec![], - )), - }; - let merged = QueryExpr::concat_with_discriminator( - vec![branch(), branch()], - /* discriminator */ 1, - /* inner_key */ vec![0], - ); - let schema = merged.output_schema().unwrap(); - assert_eq!( - schema.unique_keys, - vec![vec![1, 0]], - "(discriminator, inner_key) is the sole asserted unique key" - ); - assert_eq!( - schema.fields.len(), - 2, - "column shape is still the first branch's" - ); - } - - #[test] - fn discriminator_assertion_rejects_unknown_wire_fields() { - let json = r#"{"discriminator":1,"inner_key":[0],"unverified":true}"#; - assert!(serde_json::from_str::(json).is_err()); - } - - /// The override is opt-in: building a `Concat` without asserting a - /// discriminator — via the plain struct literal, exactly like every call - /// site before issue #228 — still drops `unique_keys` by default, - /// unchanged. - #[test] - fn ordinary_concat_struct_literal_still_drops_unique_keys_by_default() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - let merged = QueryExpr::Concat { - children: vec![branch(), branch()], - discriminator_unique_key: None, - }; - assert!(merged.output_schema().unwrap().unique_keys.is_empty()); - } - - /// Misuse check (issue #228): there is no way to end up with a - /// discriminator-backed unique key without a call site literally naming - /// a column as the discriminator. Neither the ordinary `concat` - /// constructor nor a bare struct literal with `discriminator_unique_key: - /// None` can be coaxed into fabricating one — the only path that - /// produces `Some` is `concat_with_discriminator` / - /// `ConcatDiscriminatorKey::new`, both of which require `discriminator` - /// as an explicit, named argument. - #[test] - fn no_way_to_fabricate_a_unique_key_without_naming_a_discriminator() { - let branch = || QueryExpr::Dedup { - cols: vec![0], - child: Rc::new(scan(vec![col("k", DataType::Utf8, false)], None, vec![])), - }; - // The ordinary builder. - assert_eq!( - QueryExpr::concat(vec![branch(), branch()]) - .output_schema() - .unwrap() - .unique_keys, - Vec::>::new() - ); - // The bare struct literal, explicitly opting out. - assert_eq!( - QueryExpr::Concat { - children: vec![branch(), branch()], - discriminator_unique_key: None, - } - .output_schema() - .unwrap() - .unique_keys, - Vec::>::new() - ); - } - - #[test] - fn project_retypes_and_renames_per_item() { - let child = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("host", DataType::Utf8, false), - col("value", DataType::Float64, false), - ], - Some(0), - vec![vec![0, 1]], - ); - let q = QueryExpr::Project { - qualifier: None, - cols: vec![ - // bare column passthrough keeps its (schema) name + type: host=col 1 - ProjectItem { - alias: None, - expr: QueryExpr::Column(1), - }, - // arithmetic over value (col 2) → Float64 - ProjectItem { - alias: Some("dbl".into()), - expr: QueryExpr::Arithmetic { - op: ArithmeticOpKind::Add, - left: Rc::new(QueryExpr::Column(2)), - right: Rc::new(QueryExpr::Column(2)), - }, - }, - // comparison → Bool (nullable under 3-valued logic) - ProjectItem { - alias: Some("flag".into()), - expr: QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(2)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(0.0))), - }, - }, - ], - child: Rc::new(child), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields.len(), 3); - assert_eq!(s.fields[0], col("host", DataType::Utf8, false)); - assert_eq!(s.fields[1], col("dbl", DataType::Float64, false)); - assert_eq!(s.fields[2], col("flag", DataType::Bool, true)); - // projection drops the time axis + unique keys (ts not retained) - assert!(s.time_index.is_none()); - assert!(s.unique_keys.is_empty()); - } - - #[test] - fn group_keys_by_vs_without_semantics() { - let by = GroupKeys::by(vec![1, 2]); - let without = GroupKeys::without(vec![1, 2]); - assert!(!by.is_without()); - assert!(without.is_without()); - // Deref / iteration expose the stored keys regardless of mode. - assert_eq!(by.len(), 2); - assert_eq!(without.keys(), &[1, 2]); - // A `by` compares equal to its bare vec; a `without` never does. - assert_eq!(by, vec![1, 2]); - assert_ne!(without, vec![1, 2]); - assert_ne!(by, without); - } - - #[test] - fn group_keys_serde_by_is_bare_array_without_is_tagged() { - // `by` keeps the pre-#39 bare-array wire format; `without` uses an object. - let by = serde_json::to_string(&GroupKeys::by(vec![2, 3])).unwrap(); - assert_eq!(by, "[2,3]"); - let without = serde_json::to_string(&GroupKeys::without(vec![2])).unwrap(); - assert_eq!(without, r#"{"without":[2]}"#); - // Round-trip both. - for g in [GroupKeys::by(vec![2, 3]), GroupKeys::without(vec![2])] { - let json = serde_json::to_string(&g).unwrap(); - let back: GroupKeys = serde_json::from_str(&json).unwrap(); - assert_eq!(back, g); - } - } - - #[test] - fn without_aggregate_keeps_open_schema_minus_excluded() { - // `sum without (instance) (m)` over `[ts, value, instance, job]`: the - // kept labels are the input labels minus the excluded `instance` (and ts - // / value), followed by the `sum` column, and the schema stays OPEN - // (issue #39). `job` survives; `instance` is dropped. - let scan_node = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("instance", DataType::Utf8, true), - col("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - }; - let agg = QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![2])), // exclude `instance` - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(scan_node), - }; - let s = agg.output_schema().unwrap(); - let names: Vec<_> = s.fields.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["job", "sum"], "kept `job`, dropped `instance`"); - assert!(!s.closed, "a `without` result stays open"); - assert!(s.time_index.is_none()); - assert!(s.unique_keys.is_empty(), "kept set unknown → no unique key"); - } - - // A nested aggregate's renamed sample value is not a kept label. - #[test] - fn without_aggregate_drops_a_renamed_sample_value() { - // `sum without (inst) (sum by (inst, job) (m))` over `[inst, job, sum]`. - let inner = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: Schema::new(vec![ - col("inst", DataType::Utf8, true), - col("job", DataType::Utf8, true), - col("sum", DataType::Float64, false), - ]), - }; - let agg = QueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::without(vec![0])), - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(inner), - }; - let s = agg.output_schema().unwrap(); - let names: Vec<_> = s.fields.iter().map(|c| c.name.as_str()).collect(); - assert_eq!(names, vec!["job", "sum"]); - } - - #[test] - fn time_shift_is_schema_pass_through() { - // `offset`/`@` move *when* a selector is evaluated, never its columns — - // a `TimeShift` output schema equals its child's (issue #40). - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let shifted = QueryExpr::TimeShift { - shift: TimeShift { - offset_ms: 3_600_000, - at: Some(AtModifier::Timestamp(1_609_746_000_000)), - }, - child: Rc::new(scan_node.clone()), - }; - assert_eq!( - shifted.output_schema().unwrap(), - scan_node.output_schema().unwrap(), - ); - } - - #[test] - fn time_shift_identity_and_serde() { - let offset_only = TimeShift { - offset_ms: 1, - at: None, - }; - let at_only = TimeShift { - offset_ms: 0, - at: Some(AtModifier::End), - }; - assert!(TimeShift::default().is_identity()); - assert!(!offset_only.is_identity()); - assert!(!at_only.is_identity()); - // Round-trip the shift + anchor. - let s = TimeShift { - offset_ms: -300_000, - at: Some(AtModifier::Timestamp(60_000)), - }; - let back: TimeShift = serde_json::from_str(&serde_json::to_string(&s).unwrap()).unwrap(); - assert_eq!(back, s); - } - - // Nested temporal aggregation must replace the sample, never the grouping label. - #[test] - fn temporal_reduction_of_grouped_sum_preserves_job() { - let input = Schema::new(vec![ - col("job", DataType::Utf8, true), - col("sum", DataType::Float64, false), - ]); - for aggregate in [ - AggIntent::Avg { col: None }, - AggIntent::Avg { col: Some(1) }, - AggIntent::Rate, - ] { - let output = - aggregate_output_schema(&input, &Reduction::PerEntity, &[aggregate], &[]).unwrap(); - assert_eq!(output.fields[0], input.fields[0]); - assert_eq!(output.fields[1].name, "value"); - assert_eq!(output.fields[1].dtype, DataType::Float64); - } - } - - #[test] - fn per_series_rate_preserves_labels() { - // A per-series range reduction (`rate`) is label-preserving: it produces - // one value per series, so every label survives and only the sample - // value is replaced (kept named `value`). The TimeRange child is the - // structural marker; the outer Aggregate carries the Rate intent. - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let rate = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan_node), - }), - }; - let s = rate.output_schema().unwrap(); - assert_eq!( - s.fields.iter().map(|c| c.name.as_str()).collect::>(), - vec!["ts", "value", "job"], - "rate preserves all labels; only the sample value is replaced" - ); - assert_eq!(s.time_index, Some(0)); - assert!(s.column_id("job").is_some(), "label survives the reduction"); - } - - #[test] - fn over_time_reduction_preserves_labels() { - // `*_over_time` lowers to `Aggregate { by:[], [reducer], TimeRange { Scan } }`: - // a per-series time-range reduction. The TimeRange child confers per-series - // semantics on otherwise cross-series intents like `Avg`, so an outer - // `sum by(job)(avg_over_time(...))` resolves its key positionally. - let scan_node = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - Some(0), - vec![], - ); - let avg_over_time = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Avg { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(QueryExpr::TimeRange { - range: Duration::from_secs(300), - child: Rc::new(scan_node), - }), - }; - let s = avg_over_time.output_schema().unwrap(); - assert_eq!( - s.fields.iter().map(|c| c.name.as_str()).collect::>(), - vec!["ts", "value", "job"], - "TimeRange-child marks per-series: labels preserved, value renamed" - ); - assert!( - s.column_id("job").is_some(), - "outer Aggregate.by can resolve it" - ); - } - - #[test] - fn completeness_open_leaf_freezes_to_closed_at_cross_series_aggregate() { - // A schemaless (PromQL-style) leaf is *open*; it stays open through a - // per-series reduction (`rate`), then is **frozen to closed** by a - // cross-series aggregate (which enumerates exactly its output columns). - let open_leaf = QueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - // `with_time_index` defaults to `closed: false` (open). - schema: Schema::with_time_index( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - col("job", DataType::Utf8, true), - ], - 0, - vec![], - ), - }; - assert!( - !open_leaf.output_schema().unwrap().closed, - "schemaless leaf is open" - ); - - let rate = QueryExpr::Aggregate { - reduction: Reduction::PerEntity, - measures: vec![AggIntent::Rate], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(open_leaf), - }; - assert!( - !rate.output_schema().unwrap().closed, - "per-series rate is label-preserving → stays open" - ); - - let sum_by_job = QueryExpr::Aggregate { - reduction: Reduction::by(vec![2]), // `job` - measures: vec![AggIntent::Sum { col: None }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(rate), - }; - assert!( - sum_by_job.output_schema().unwrap().closed, - "cross-series aggregate enumerates `by ++ measures` → frozen to closed" - ); - } - - #[test] - fn project_keeps_time_index_when_ts_passed_through() { - let child = scan( - vec![ - col("ts", DataType::Timestamp, false), - col("value", DataType::Float64, false), - ], - Some(0), - vec![], - ); - let q = QueryExpr::Project { - qualifier: None, - cols: vec![ - // value=col 1, ts=col 0 - ProjectItem { - alias: None, - expr: QueryExpr::Column(1), - }, - ProjectItem { - alias: None, - expr: QueryExpr::Column(0), - }, - ], - child: Rc::new(child), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields[0].name, "value"); - assert_eq!(s.fields[1].name, "ts"); - assert_eq!(s.time_index, Some(1)); - } - - fn join(kind: JoinKind) -> QueryExpr { - let left = scan(vec![col("a", DataType::Int64, false)], None, vec![vec![0]]); - let right = scan(vec![col("b", DataType::Utf8, false)], None, vec![]); - QueryExpr::Join { - kind, - pred: Predicate(Rc::new(QueryExpr::Literal(ScalarValue::Boolean(true)))), - left: Rc::new(left), - right: Rc::new(right), - } - } - - #[test] - fn inner_join_concatenates_both_sides() { - let s = join(JoinKind::Inner).output_schema().unwrap(); - assert_eq!(s.fields.len(), 2); - assert_eq!(s.fields[0], col("a", DataType::Int64, false)); - assert_eq!(s.fields[1], col("b", DataType::Utf8, false)); - // post-join row identity not provable → no unique keys - assert!(s.unique_keys.is_empty()); - } - - #[test] - fn left_join_makes_right_side_nullable() { - let s = join(JoinKind::Left).output_schema().unwrap(); - assert!(!s.fields[0].nullable, "preserved left side stays non-null"); - assert!(s.fields[1].nullable, "right side nullable under LEFT JOIN"); - } - - #[test] - fn full_join_makes_both_sides_nullable() { - let s = join(JoinKind::Full).output_schema().unwrap(); - assert!(s.fields[0].nullable); - assert!(s.fields[1].nullable); - } - - #[test] - fn setop_takes_left_shape_and_drops_unique_keys() { - let left = scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![vec![0]], - ); - let right = scan( - vec![ - col("k", DataType::Utf8, false), - col("v", DataType::Int64, false), - ], - None, - vec![vec![0]], - ); - let q = QueryExpr::SetOp { - kind: RelationalSetOpKind::Union, - all: false, - left: Rc::new(left), - right: Rc::new(right), - }; - let s = q.output_schema().unwrap(); - assert_eq!(s.fields.len(), 2); - assert_eq!(s.fields[0].name, "k"); - assert!( - s.unique_keys.is_empty(), - "UNION does not preserve row identity" - ); - } - - // ── PromqlScalarBridge / Literal dedup (issue #220) ───────────────────── - - /// `QueryExpr::promql_scalar(v)` — what every front end now constructs in - /// place of the old `PromqlScalar(v)` leaf — wraps exactly - /// `Literal(ScalarValue::Float64(v))`: the same value a SQL-emitted typed - /// float literal in a scalar-sub-language position would carry, just at a - /// different DAG position. `as_promql_scalar` is the round-trip inverse. - #[test] - fn promql_scalar_bridges_a_literal_float_at_an_operator_position() { - let bridge = QueryExpr::::promql_scalar(2.5); - assert_eq!( - bridge, - QueryExpr::PromqlScalarBridge(Rc::new(QueryExpr::Literal(ScalarValue::Float64(2.5)))) - ); - assert_eq!(bridge.as_promql_scalar(), Some(2.5)); - - // The same value a SQL `Compare`/`Arithmetic` operand would carry, in - // its native (unwrapped, no row schema) scalar-sub-language position — - // no longer a different variant, just not bridged to this DAG - // position. - let sql_literal = QueryExpr::::Literal(ScalarValue::Float64(2.5)); - assert_eq!(bridge.as_promql_scalar(), Some(2.5)); - assert_ne!( - bridge, sql_literal, - "bridge and bare literal are distinct nodes" - ); - // Not every shape is a scalar bridge: neither a bare `Literal` nor an - // operator node reports a value. - assert_eq!(sql_literal.as_promql_scalar(), None); - assert_eq!(scan(vec![], None, vec![]).as_promql_scalar(), None); - } - - /// Pins the DAG-position distinction issue #220 asks for: the very same - /// `Literal(ScalarValue::Float64(_))` value has a row schema when it sits - /// at the operator-DAG position (wrapped in `PromqlScalarBridge` — a - /// `BinaryOp` operand, `PromqlVectorFromScalar` child, or a query root), - /// and has none when it sits bare, in a scalar-sub-language position - /// (`Compare`/`Arithmetic`/… operand) — no longer decided by which of two - /// duplicate variants was used, only by whether the wrapper is present. - #[test] - fn row_schema_rides_on_the_bridge_wrapper_not_the_literal_variant() { - let bridged = QueryExpr::::promql_scalar(42.0); - let schema = bridged.output_schema().expect("bridge has a row schema"); - assert_eq!(schema.fields.len(), 1); - assert_eq!(schema.fields[0].name, "value"); - assert_eq!(schema.fields[0].dtype, DataType::Float64); - assert!(schema.time_index.is_none()); - - // The identical value, unwrapped (the scalar-sub-language position a - // `Compare`/`Arithmetic` operand would occupy) has no row schema of - // its own — it's a construction bug to call `output_schema` on it - // directly, caught as `ScalarHasNoRowSchema` rather than panicking. - let bare = QueryExpr::::Literal(ScalarValue::Float64(42.0)); - assert!(matches!( - bare.output_schema(), - Err(QueryExprError::ScalarHasNoRowSchema) - )); - } - - /// `BinaryOp`'s schema derivation follows the non-scalar (vector) side - /// when the other operand is a `PromqlScalarBridge`, and a `VectorMatch` - /// modifier survives unchanged alongside it — the relational binary-op - /// path (issue #220's Instance 2, left as follow-up) is untouched by the - /// Instance-1 `PromqlScalar` → `PromqlScalarBridge` collapse. - // `filters` (#466) round-trips, and an `Aggregate` serialized before the - // field existed still deserializes as unfiltered. - #[test] - fn aggregate_filters_serde_round_trip_and_default() { - let child = Rc::new(scan( - vec![ - col("service", DataType::Utf8, false), - col("latency", DataType::Float64, false), - ], - None, - vec![], - )); - let filtered = QueryExpr::Aggregate { - reduction: Reduction::by(vec![0]), - measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum { col: Some(1) }, - ], - output_names: vec![], - filters: vec![ - Some(Predicate(Rc::new(QueryExpr::Compare { - left: Rc::new(QueryExpr::Column(1)), - op: CompareOpKind::Gt, - right: Rc::new(QueryExpr::Literal(ScalarValue::Float64(1.0))), - }))), - None, - ], - having: None, - child: Rc::clone(&child), - }; - let json = serde_json::to_value(&filtered).unwrap(); - assert_eq!( - serde_json::from_value::(json.clone()).unwrap(), - filtered - ); - - let mut legacy = json; - legacy["Aggregate"] - .as_object_mut() - .unwrap() - .remove("filters") - .expect("fixture sanity: filters was serialized"); - let decoded: QueryExpr = serde_json::from_value(legacy).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &decoded else { - unreachable!() - }; - assert!(filters.is_empty()); - } - - #[test] - fn binary_op_schema_follows_the_vector_side_over_a_scalar_bridge_with_vector_match_intact() { - let vector = scan( - vec![ - col("host", DataType::Utf8, false), - col("value", DataType::Float64, false), - ], - None, - vec![], - ); - let vm = VectorMatch { - kind: VectorMatchKind::On, - labels: vec!["host".into()], - grouping: None, - }; - let op = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOpKind::Gt), - lhs: Rc::new(vector.clone()), - rhs: Rc::new(QueryExpr::promql_scalar(1.0)), - vector_match: Some(vm.clone()), - }; - assert_eq!(op.output_schema().unwrap(), vector.output_schema().unwrap()); - let QueryExpr::BinaryOp { vector_match, .. } = &op else { - unreachable!() - }; - assert_eq!(vector_match.as_ref(), Some(&vm)); - } -} diff --git a/crates/types/src/pre_asap/resolve.rs b/crates/types/src/pre_asap/resolve.rs deleted file mode 100644 index b4a5c87a6..000000000 --- a/crates/types/src/pre_asap/resolve.rs +++ /dev/null @@ -1,857 +0,0 @@ -//! Resolve a front-end-emitted, unresolved [`UnresolvedQueryExpr`] (`QueryExpr`) -//! into the canonical, positional [`ResolvedQueryExpr`] (`QueryExpr`). -//! -//! Both front ends (`asap-frontend-promql`, `asap-frontend-sql`) construct -//! canonical `QueryExpr` shapes directly during their own `interpret` step -//! (issue #179) — heavy-hitter `topk` recognition, the window-over-aggregate -//! fold, the `PerEntity`/`Reduce` reduction choice, and every other -//! *structural* decision happen right there, since a front end already knows -//! the answer at parse time. What's left for [`resolve_root`] is exactly the -//! "mechanical, schema-dependent substitution" #179 describes: a single -//! generic, shape-preserving walk — every [`UnresolvedQueryExpr`] variant maps to the -//! identical [`ResolvedQueryExpr`] variant — that resolves every [`ColumnRef`] to -//! the [`SchemaResolver`](super::schema_resolver::SchemaResolver)-computed positional [`ColumnId`]. -//! -//! ## Why positional `ColumnId`, not just carrying names all the way through (issue #216) -//! -//! A mature query engine can legitimately choose either design — DataFusion's -//! own logical plan (what `asap-frontend-sql` walks to build its `QueryExpr`) -//! and Calcite both keep names, with an optional table qualifier, all the way -//! through logical optimization, only going positional once they lower to a -//! physical plan. Resolving once, immediately after each front end's own -//! `interpret` step, is the better trade for *this* codebase's shape — one -//! front-end-facing DAG feeding several independent downstream passes -//! (`canonicalize`, the cost model, `dag_export`, schema/type inference, -//! `asap-aware-mapping`'s summary binding) — for three concrete reasons: -//! -//! 1. **Names collide across joins.** Not hypothetical: `join_predicate_disambiguates_shared_column_name` -//! (`crates/frontend-sql/tests/sql_lowering.rs`) exists specifically because -//! `metrics.service` and `hosts.service` are both just `"service"` once their -//! schemas are concatenated. A bare name is ambiguous the moment two sources -//! share one; `ColumnId` is what makes "the second `service`, position 4, not -//! the first" a fact recorded once, instead of a lookup redone at every use site. -//! 2. **A name's meaning changes going up the DAG.** `Project` renames/aliases, -//! `Aggregate` collapses columns and introduces synthetic ones, `Join` -//! concatenates two schemas — a name valid at a `Scan` leaf isn't -//! automatically the right binding three nodes up; it has to be reinterpreted -//! against whatever schema is in scope at that node. Resolving bottom-up -//! pins each reference to "this exact column of this exact node's -//! already-derived output schema," so nothing downstream re-derives that scope. -//! 3. **It concentrates scoping logic in one place instead of ~6.** Every -//! downstream pass just compares/indexes `ColumnId`s — O(1), unambiguous. If -//! they worked on names instead, each would need its own qualifier-aware, -//! join-collision-aware name resolver, or risk silently binding to the wrong -//! `"service"`. -//! -//! Removing this resolution step and carrying `ColumnRef` everywhere would -//! therefore be a real regression for this repo's shape, not just a rename — -//! every one of those downstream passes would have to reimplement the scoping -//! this module already centralizes. - -use std::rc::Rc; - -use thiserror::Error; - -use super::agg_intent::AggIntent; -use super::column_resolution::{ - resolve_column_ref, resolve_column_refs, resolve_expr, resolve_group_keys_promql, ResolveError, -}; -use super::expr_ir::ColumnRef; -use super::query_expr::{ - aggregate_output_schema, any_measure_filtered, ConcatDiscriminatorKey, GroupKeys, Predicate, - ProjectItem, QueryExprError, Reduction, ResolvedQueryExpr, SortKey, UnresolvedQueryExpr, -}; -use super::schema::{ColumnId, Schema}; -use super::schema_resolver::SchemaResolver; - -/// Errors from resolving a canonical, unresolved [`UnresolvedQueryExpr`] DAG. -#[derive(Debug, Error)] -pub enum ResolveDAGError { - /// A column reference did not resolve against its in-scope schema. - #[error("column resolution failed: {0}")] - Resolve(#[from] ResolveError), - /// Deriving the schema of an already-resolved child failed (needed to - /// resolve positional column references against it). - #[error("schema derivation failed: {0}")] - Schema(#[from] QueryExprError), -} - -/// Resolve a whole [`UnresolvedQueryExpr`] DAG rooted at `dag` into canonical -/// [`ResolvedQueryExpr`]: binds every `ColumnRef` to a `ColumnId` via the -/// [`SchemaResolver`], then [`canonicalize`](super::canonicalize::canonicalize)s the -/// result. -pub fn resolve_root(dag: &UnresolvedQueryExpr) -> Result { - resolve_root_with_inherited(dag, &[]) -} - -/// [`resolve_root`] with label names inherited from an enclosing scope seeded -/// into the leaf schema, used when re-binding a `BinaryOp` side (issue #52). -fn resolve_root_with_inherited( - dag: &UnresolvedQueryExpr, - inherited: &[String], -) -> Result { - let fallback = SchemaResolver::new().resolve_schema_with_inherited(dag, inherited); - let l3 = resolve(dag, &fallback)?; - Ok(super::canonicalize::canonicalize(l3)) -} - -/// The generic substitution walk: converts children first (bottom-up), then -/// resolves this node's own `ColumnRef`s against the *converted child's* -/// derived output schema — so a `JOIN`'s concatenated schema and a cross- -/// series aggregate's frozen-closed output bind to the right positions. -fn resolve( - dag: &UnresolvedQueryExpr, - fallback: &Schema, -) -> Result { - use super::query_expr::QueryExpr as QE; - Ok(match dag { - QE::Scan { - source, - predicates, - schema, - } => { - let schema = schema.clone().unwrap_or_else(|| fallback.clone()); - let predicates = predicates - .iter() - .map(|Predicate(e)| Ok(Predicate(Rc::new(resolve_expr(e, &schema)?)))) - .collect::, ResolveError>>()?; - QE::Scan { - source: source.clone(), - predicates, - schema, - } - } - - // `PromqlScalarBridge`'s child is a scalar-sub-language node (issue - // #220) sitting at this operator-DAG position — resolved through - // `resolve_expr`, same as every other scalar position (`Predicate`, - // `ProjectItem.expr`, …), not the operator walk. In practice it's - // always a `Literal`, which has no `ColumnRef` to resolve, so - // `fallback` is never actually consulted here. - QE::PromqlScalarBridge(inner) => { - QE::PromqlScalarBridge(Rc::new(resolve_expr(inner, fallback)?)) - } - QE::EvalTimestamp => QE::EvalTimestamp, - QE::CurrentTimestamp => QE::CurrentTimestamp, - - QE::PromqlVectorFromScalar(child) => { - QE::PromqlVectorFromScalar(Rc::new(resolve(child, fallback)?)) - } - QE::PromqlScalarFromVector(child) => { - QE::PromqlScalarFromVector(Rc::new(resolve(child, fallback)?)) - } - - QE::PromqlRelabel { dst, value, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::PromqlRelabel { - dst: dst.clone(), - value: Rc::new(resolve_expr(value, &child_schema)?), - child: Rc::new(child), - } - } - - QE::PromqlInfoEnrich { selector, child } => QE::PromqlInfoEnrich { - selector: selector.clone(), - child: Rc::new(resolve(child, fallback)?), - }, - - QE::PromqlSeriesSample { by, kind, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::PromqlSeriesSample { - by: resolve_group_keys(by, &child_schema)?, - kind: *kind, - child: Rc::new(child), - } - } - - QE::Filter { pred, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::Filter { - pred: Predicate(Rc::new(resolve_expr(&pred.0, &child_schema)?)), - child: Rc::new(child), - } - } - - QE::Project { - cols, - qualifier, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let cols = cols - .iter() - .map(|item| -> Result { - Ok(ProjectItem { - alias: item.alias.clone(), - expr: resolve_expr(&item.expr, &child_schema)?, - }) - }) - .collect::, _>>()?; - QE::Project { - cols, - qualifier: qualifier.clone(), - child: Rc::new(child), - } - } - - QE::Aggregate { - reduction, - measures, - output_names, - filters, - having, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let reduction = resolve_reduction(reduction, &child_schema)?; - let measures = measures - .iter() - .map(|m| resolve_agg_intent(m, &child_schema)) - .collect::, ResolveError>>()?; - // A measure filter reads the rows being aggregated, so it binds - // against the child's schema, not the aggregate's output. - let filters = filters - .iter() - .map(|f| { - f.as_ref() - .map(|Predicate(p)| Ok(Predicate(Rc::new(resolve_expr(p, &child_schema)?)))) - .transpose() - }) - .collect::, ResolveError>>()?; - // One canonical spelling of "unfiltered" (empty), so structural - // equality and CSE never split on `[]` versus `[None, None]`. - let filters = if any_measure_filtered(&filters) { - filters - } else { - Vec::new() - }; - let having = having - .as_ref() - .map(|Predicate(h)| -> Result { - let out_schema = aggregate_output_schema( - &child_schema, - &reduction, - &measures, - output_names, - )?; - Ok(Predicate(Rc::new(resolve_expr(h, &out_schema)?))) - }) - .transpose()?; - QE::Aggregate { - reduction, - measures, - output_names: output_names.clone(), - filters, - having, - child: Rc::new(child), - } - } - - QE::Dedup { cols, child } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - QE::Dedup { - cols: resolve_column_refs(cols, &child_schema)?, - child: Rc::new(child), - } - } - - QE::Concat { - children, - discriminator_unique_key, - } => { - let children: Vec<_> = children - .iter() - .map(|c| resolve(c, fallback)) - .collect::, _>>()?; - // No front end asserts this today (issue #228 shipped the - // extension point ahead of a wired call site) — resolved here - // regardless, against the first resolved branch's own output - // schema, exactly the schema `output_schema`'s `Concat` arm - // derives the merged schema from, so a future direct - // `concat_with_discriminator` caller upstream of `resolve_root` - // gets a correctly positional `ConcatDiscriminatorKey` out the - // other side. - let discriminator_unique_key = discriminator_unique_key - .as_ref() - .map(|key| -> Result<_, ResolveDAGError> { - let schema = children - .first() - .ok_or(QueryExprError::EmptyConcat)? - .output_schema()?; - Ok(ConcatDiscriminatorKey::new( - resolve_column_ref(key.discriminator(), &schema)?, - resolve_column_refs(key.inner_key(), &schema)?, - )) - }) - .transpose()?; - QE::Concat { - children, - discriminator_unique_key, - } - } - - QE::Join { - kind, - pred, - left, - right, - } => { - // Each branch is bound independently, same reasoning as `BinaryOp` - // below — different leaves / label sets. - let left = resolve_root_with_inherited(left, &[])?; - let right = resolve_root_with_inherited(right, &[])?; - let mut concat = left.output_schema()?; - concat.fields.extend(right.output_schema()?.fields); - let pred = Predicate(Rc::new(resolve_expr(&pred.0, &concat)?)); - QE::Join { - kind: kind.clone(), - pred, - left: Rc::new(left), - right: Rc::new(right), - } - } - - QE::SetOp { - kind, - all, - left, - right, - } => QE::SetOp { - kind: kind.clone(), - all: *all, - left: Rc::new(resolve_root_with_inherited(left, &[])?), - right: Rc::new(resolve_root_with_inherited(right, &[])?), - }, - - QE::Sort { - keys, - partition_by, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let keys = keys - .iter() - .map(|k| -> Result { - Ok(SortKey { - expr: resolve_expr(&k.expr, &child_schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child_schema)?; - QE::Sort { - keys, - partition_by, - child: Rc::new(child), - } - } - - QE::Limit { n, offset, child } => QE::Limit { - n: *n, - offset: *offset, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::PromqlSubquery { - range, - resolution, - child, - } => QE::PromqlSubquery { - range: *range, - resolution: *resolution, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::TimeRange { range, child } => QE::TimeRange { - range: *range, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::TimeShift { shift, child } => QE::TimeShift { - shift: *shift, - child: Rc::new(resolve(child, fallback)?), - }, - - QE::SQLWindowFunc { - func, - args, - partition_by, - order_by, - frame, - output_name, - child, - } => { - let child = resolve(child, fallback)?; - let child_schema = child.output_schema()?; - let args = args - .iter() - .map(|a| resolve_expr(a, &child_schema)) - .collect::, _>>()?; - let partition_by = resolve_group_keys(partition_by, &child_schema)?; - let order_by = order_by - .iter() - .map(|k| -> Result { - Ok(SortKey { - expr: resolve_expr(&k.expr, &child_schema)?, - ascending: k.ascending, - nulls_first: k.nulls_first, - }) - }) - .collect::, _>>()?; - QE::SQLWindowFunc { - func: func.clone(), - args, - partition_by, - order_by, - frame: frame.clone(), - output_name: output_name.clone(), - child: Rc::new(child), - } - } - - QE::BinaryOp { - op, - lhs, - rhs, - vector_match, - } => { - // A binary op's two sides may scan different metrics with - // different label sets, so each branch resolves against its OWN - // bound schema; but an independently-bound side still has to see - // label names an *enclosing* node references (issue #52). - let own = super::schema_resolver::collect_referenced_columns(dag); - let inherited: Vec = inherited_names(fallback) - .into_iter() - .filter(|n| !own.contains(n)) - .collect(); - QE::BinaryOp { - op: op.clone(), - lhs: Rc::new(resolve_root_with_inherited(lhs, &inherited)?), - rhs: Rc::new(resolve_root_with_inherited(rhs, &inherited)?), - vector_match: vector_match.clone(), - } - } - - // The scalar variants (issue #205) are never reached here directly — - // `resolve` only ever recurses into `child`/operator positions; - // every scalar position (`Predicate`, `ProjectItem.expr`, …) goes - // through `resolve_expr` instead, at the operator arm that owns it. - other @ (QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. }) => { - unreachable!("resolve reached a scalar QueryExpr variant directly: {other:?}") - } - }) -} - -/// The label names an enclosing scope's schema carries beyond the `(ts, -/// value)` floor. -fn inherited_names(schema: &Schema) -> Vec { - schema - .fields - .iter() - .filter(|c| c.name != "ts" && c.name != "value") - .map(|c| c.name.clone()) - .collect() -} - -/// Resolve a name-based [`GroupKeys`] into positional -/// [`GroupKeys`], preserving its `by`/`without` mode. -fn resolve_group_keys( - keys: &GroupKeys, - schema: &Schema, -) -> Result, ResolveError> { - let ids = resolve_column_refs(keys.keys(), schema)?; - Ok(if keys.is_without() { - GroupKeys::without(ids) - } else { - GroupKeys::by(ids) - }) -} - -/// Resolve a name-based [`Reduction`] into positional -/// [`Reduction`]. -/// -/// Uses [`resolve_group_keys_promql`] rather than the strict -/// [`resolve_group_keys`], unlike every other group-key site in `resolve` -/// (`PromqlSeriesSample.by`, `Sort.partition_by`, `SQLWindowFunc.partition_by`): a key -/// absent from a **closed** schema (e.g. the output of a nested cross-series -/// aggregate that collapsed the label) is provably absent from every row, so -/// PromQL drops it from the grouping rather than rejecting the query (issue -/// #53) — `sum(sum by (group) (m)) by (job)` is the canonical case, `job` -/// absent from the inner aggregate's closed `[group, sum]` output. Applied -/// uniformly to every `Aggregate`, not just PromQL's: SQL's `GROUP BY` keys -/// are always genuinely present (DataFusion validates the plan), so the -/// "drop instead of reject" branch is simply never exercised there — the -/// lenient resolver is a no-op difference for a SQL DAG, not a behavior -/// change. -fn resolve_reduction( - reduction: &Reduction, - schema: &Schema, -) -> Result, ResolveError> { - Ok(match reduction { - Reduction::Reduce(by) => { - let ids = resolve_group_keys_promql(by.keys(), schema)?; - Reduction::Reduce(if by.is_without() { - GroupKeys::without(ids) - } else { - GroupKeys::by(ids) - }) - } - Reduction::PerEntity => Reduction::PerEntity, - }) -} - -/// Resolve a name-based [`AggIntent`] into positional -/// [`AggIntent`] — every `col: Option` resolves to -/// `Option` (`None` stays `None`, the sample-value convention); -/// every other field carries straight through unchanged. -fn resolve_agg_intent( - intent: &AggIntent, - schema: &Schema, -) -> Result, ResolveError> { - let col = |c: &Option| -> Result, ResolveError> { - c.as_ref() - .map(|r| resolve_column_ref(r, schema)) - .transpose() - }; - Ok(match intent { - AggIntent::Count { accuracy } => AggIntent::Count { - accuracy: accuracy.clone(), - }, - AggIntent::PearsonCorr { left, right } => AggIntent::PearsonCorr { - left: resolve_column_ref(left, schema)?, - right: resolve_column_ref(right, schema)?, - }, - AggIntent::Sum { col: c } => AggIntent::Sum { col: col(c)? }, - AggIntent::Min { col: c } => AggIntent::Min { col: col(c)? }, - AggIntent::Max { col: c } => AggIntent::Max { col: col(c)? }, - AggIntent::Avg { col: c } => AggIntent::Avg { col: col(c)? }, - AggIntent::StdDev { col: c, population } => AggIntent::StdDev { - col: col(c)?, - population: *population, - }, - AggIntent::Variance { col: c, population } => AggIntent::Variance { - col: col(c)?, - population: *population, - }, - AggIntent::Quantile { - col: c, - q, - accuracy, - } => AggIntent::Quantile { - col: col(c)?, - q: *q, - accuracy: accuracy.clone(), - }, - AggIntent::TopK { k, accuracy } => AggIntent::TopK { - k: *k, - accuracy: accuracy.clone(), - }, - AggIntent::Cardinality { cols, accuracy } => AggIntent::Cardinality { - cols: cols - .iter() - .map(|c| resolve_column_ref(c, schema)) - .collect::>()?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyL2 { col: c, accuracy } => AggIntent::FrequencyL2 { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::FrequencyEntropy { col: c, accuracy } => AggIntent::FrequencyEntropy { - col: col(c)?, - accuracy: accuracy.clone(), - }, - AggIntent::Rate => AggIntent::Rate, - AggIntent::IRate => AggIntent::IRate, - AggIntent::Increase => AggIntent::Increase, - AggIntent::Changes => AggIntent::Changes, - AggIntent::Delta => AggIntent::Delta, - AggIntent::IDelta => AggIntent::IDelta, - AggIntent::Deriv => AggIntent::Deriv, - AggIntent::Resets => AggIntent::Resets, - AggIntent::PredictLinear { seconds } => AggIntent::PredictLinear { seconds: *seconds }, - AggIntent::DoubleExpSmoothing { smoothing, trend } => AggIntent::DoubleExpSmoothing { - smoothing: *smoothing, - trend: *trend, - }, - AggIntent::HistogramCount => AggIntent::HistogramCount, - AggIntent::HistogramSum => AggIntent::HistogramSum, - AggIntent::HistogramAvg => AggIntent::HistogramAvg, - AggIntent::HistogramStdDev => AggIntent::HistogramStdDev, - AggIntent::HistogramStdVar => AggIntent::HistogramStdVar, - AggIntent::HistogramFraction { lower, upper } => AggIntent::HistogramFraction { - lower: *lower, - upper: *upper, - }, - AggIntent::HistogramQuantile { q, le } => AggIntent::HistogramQuantile { - q: *q, - le: resolve_column_ref(le, schema)?, - }, - AggIntent::Math(f) => AggIntent::Math(f.clone()), - AggIntent::Absent => AggIntent::Absent, - AggIntent::AbsentOverTime => AggIntent::AbsentOverTime, - AggIntent::PresentOverTime => AggIntent::PresentOverTime, - AggIntent::TimeFn(f) => AggIntent::TimeFn(*f), - AggIntent::Group => AggIntent::Group, - AggIntent::CountValues { label } => AggIntent::CountValues { - label: label.clone(), - }, - AggIntent::LastOverTime => AggIntent::LastOverTime, - AggIntent::FirstOverTime => AggIntent::FirstOverTime, - AggIntent::MadOverTime => AggIntent::MadOverTime, - AggIntent::TsOfMinOverTime => AggIntent::TsOfMinOverTime, - AggIntent::TsOfMaxOverTime => AggIntent::TsOfMaxOverTime, - AggIntent::TsOfFirstOverTime => AggIntent::TsOfFirstOverTime, - AggIntent::TsOfLastOverTime => AggIntent::TsOfLastOverTime, - AggIntent::Extension { ext_kind, payload } => AggIntent::Extension { - ext_kind: ext_kind.clone(), - payload: payload.clone(), - }, - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::pre_asap::expr_ir::CompareOpKind; - use crate::pre_asap::query_expr::{ - BinaryOpKind, QueryExpr, Source, VectorMatch, VectorMatchKind, - }; - - // A measure filter (#466) binds positionally against the aggregate's - // input, and a vector with no set entry collapses to the empty spelling. - #[test] - fn resolve_measure_filters_against_the_child_schema() { - use crate::pre_asap::expr_ir::ScalarValue; - use crate::pre_asap::query_expr::Predicate; - use crate::pre_asap::{DataType, Field, GroupKeys}; - use crate::types::AccuracyTarget; - let scan = || UnresolvedQueryExpr::Scan { - source: Source::Table { - table_ref: "metrics".into(), - }, - predicates: vec![], - schema: Some(Schema::new(vec![ - Field::plain("service", DataType::Utf8, false), - Field::plain("latency", DataType::Float64, false), - Field::plain("bytes", DataType::Int64, false), - ])), - }; - let aggregate = |filters| UnresolvedQueryExpr::Aggregate { - reduction: Reduction::Reduce(GroupKeys::by(vec![ColumnRef::Named("service".into())])), - measures: vec![ - AggIntent::Count { - accuracy: AccuracyTarget::Exact, - }, - AggIntent::Sum { - col: Some(ColumnRef::Named("bytes".into())), - }, - ], - output_names: vec![], - filters, - having: None, - child: Rc::new(scan()), - }; - let latency_gt_one = Predicate(Rc::new(UnresolvedQueryExpr::Compare { - left: Rc::new(UnresolvedQueryExpr::Column(ColumnRef::Named( - "latency".into(), - ))), - op: CompareOpKind::Gt, - right: Rc::new(UnresolvedQueryExpr::Literal(ScalarValue::Float64(1.0))), - })); - - let resolved = resolve_root(&aggregate(vec![Some(latency_gt_one), None])).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &resolved else { - unreachable!() - }; - let [Some(Predicate(first)), None] = filters.as_slice() else { - panic!("expected one filtered and one unfiltered measure, got {filters:?}"); - }; - assert!( - matches!(first.as_ref(), QueryExpr::Compare { left, .. } - if matches!(left.as_ref(), QueryExpr::Column(1))), - "latency is input column 1, got {first:?}" - ); - - let resolved = resolve_root(&aggregate(vec![None, None])).unwrap(); - let QueryExpr::Aggregate { filters, .. } = &resolved else { - unreachable!() - }; - assert!(filters.is_empty()); - } - - // Both sides resolve with qualifiers; an unknown right input is an error. - #[test] - fn resolve_pearson_corr_inputs() { - use crate::pre_asap::{DataType, Field}; - let schema = Schema::new(vec![ - Field::plain("x", DataType::Float64, true).with_table("a"), - Field::plain("x", DataType::Float64, true).with_table("b"), - ]); - let intent = AggIntent::PearsonCorr { - left: ColumnRef::Qualified { - table: "a".into(), - name: "x".into(), - }, - right: ColumnRef::Qualified { - table: "b".into(), - name: "x".into(), - }, - }; - assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::PearsonCorr { left: 0, right: 1 } - ); - let missing = AggIntent::PearsonCorr { - left: ColumnRef::Qualified { - table: "a".into(), - name: "x".into(), - }, - right: ColumnRef::Named("missing".into()), - }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); - } - - // Every leg resolves independently, qualifiers included; one unknown leg - // fails rather than silently shortening the tuple. - #[test] - fn resolve_distinct_tuple_columns() { - use crate::pre_asap::{DataType, Field}; - use crate::types::AccuracyTarget; - let schema = Schema::new(vec![ - Field::plain("k", DataType::Int64, true).with_table("a"), - Field::plain("k", DataType::Int64, true).with_table("b"), - ]); - let qualified = |table: &str| ColumnRef::Qualified { - table: table.into(), - name: "k".into(), - }; - let intent = AggIntent::Cardinality { - cols: vec![qualified("b"), qualified("a")], - accuracy: AccuracyTarget::Exact, - }; - assert_eq!( - resolve_agg_intent(&intent, &schema).unwrap(), - AggIntent::Cardinality { - cols: vec![1, 0], - accuracy: AccuracyTarget::Exact, - } - ); - let missing = AggIntent::Cardinality { - cols: vec![qualified("a"), ColumnRef::Named("missing".into())], - accuracy: AccuracyTarget::Exact, - }; - assert!(resolve_agg_intent(&missing, &schema).is_err()); - } - - /// `resolve_root` over a `BinaryOp { , PromqlScalarBridge, vector_match }` - /// (issue #220): the bridged scalar operand resolves through the same - /// generic walk as every other node (its `Literal` child has no - /// `ColumnRef` to resolve, so it comes through unchanged), the vector - /// side's `ColumnRef`s resolve positionally, and the `VectorMatch` - /// modifier on the relational binary-op path survives resolution - /// untouched — Instance 2 of #220 (`BinaryOp` vs `Compare`/`Arithmetic`) - /// is out of scope for this change, so this pins that its behavior is - /// unaffected by the Instance-1 collapse. - #[test] - fn resolve_root_threads_a_scalar_bridge_operand_and_preserves_vector_match() { - let vm = VectorMatch { - kind: VectorMatchKind::Ignoring, - labels: vec!["job".into()], - grouping: None, - }; - let unresolved: UnresolvedQueryExpr = QueryExpr::BinaryOp { - op: BinaryOpKind::Compare(CompareOpKind::Gt), - lhs: Rc::new(UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: "up".into(), - }, - predicates: vec![], - schema: None, - }), - rhs: Rc::new(UnresolvedQueryExpr::promql_scalar(1.0)), - vector_match: Some(vm.clone()), - }; - - let resolved = resolve_root(&unresolved).expect("resolves"); - let QueryExpr::BinaryOp { - lhs, - rhs, - vector_match, - .. - } = &resolved - else { - panic!("expected a resolved BinaryOp, got {resolved:?}"); - }; - assert!(matches!(lhs.as_ref(), QueryExpr::Scan { .. })); - assert_eq!(rhs.as_promql_scalar(), Some(1.0)); - assert_eq!(vector_match.as_ref(), Some(&vm)); - - // Schema derivation still follows the vector side post-resolution. - assert_eq!( - resolved.output_schema().unwrap(), - lhs.output_schema().unwrap() - ); - } - - /// Issue #228 review, end-to-end: `resolve_root` over a `Concat` whose - /// discriminator column is referenced *nowhere else* in the DAG, with a - /// schema-less (usage-derived) leaf `Scan` in the first branch — exactly - /// the scenario the review flagged. Before the `schema_resolver.rs` fix, the - /// SchemaResolver's fallback schema wouldn't contain `phi` at all, and this - /// `resolve_column_ref` call would fail `NotFound` for a column the - /// caller correctly named. It must resolve cleanly, and the resolved - /// `ConcatDiscriminatorKey` must carry the *positional* `ColumnId`s of - /// the branch's own (usage-derived) schema. - #[test] - fn resolve_root_seeds_and_resolves_an_otherwise_unreferenced_discriminator_column() { - let branch = || UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { metric: "m".into() }, - predicates: vec![], - schema: None, - }; - let unresolved = UnresolvedQueryExpr::concat_with_discriminator( - vec![branch(), branch()], - ColumnRef::Named("phi".into()), - vec![ColumnRef::Named("host".into())], - ); - - let resolved = resolve_root(&unresolved).expect("resolves"); - let QueryExpr::Concat { - children, - discriminator_unique_key, - } = &resolved - else { - panic!("expected a resolved Concat, got {resolved:?}"); - }; - let schema = children[0].output_schema().unwrap(); - let key = discriminator_unique_key - .as_ref() - .expect("discriminator key survives resolution"); - assert_eq!(*key.discriminator(), schema.column_id("phi").unwrap()); - assert_eq!( - key.inner_key().to_vec(), - vec![schema.column_id("host").unwrap()] - ); - } -} diff --git a/crates/types/src/pre_asap/scalar_type_rules.rs b/crates/types/src/pre_asap/scalar_type_rules.rs index 44eaa250c..3efad3f7e 100644 --- a/crates/types/src/pre_asap/scalar_type_rules.rs +++ b/crates/types/src/pre_asap/scalar_type_rules.rs @@ -1,5 +1,5 @@ -//! Shared type rules for structural map scalar expressions. -//! Execution must separately implement the documented ordering/default semantics. +//! Shared type and nullability rules used to validate scalar expressions. +//! These rules do not evaluate expressions or define physical representations. use super::schema::DataType; /// Names are resolved once against this closed builtin set; unknown functions @@ -123,6 +123,21 @@ fn common_type(left: &DataType, right: &DataType) -> Result { )) } +/// Closed, namespaced contracts for PromQL pointwise float functions. +/// Date functions consume Unix seconds; `timestamp` remains a sample-selection +/// operation because its operand is a sample timestamp rather than its value. +pub fn promql_function_arity(name: &str) -> Option { + Some(match name.strip_prefix("promql_")? { + "abs" | "ceil" | "floor" | "exp" | "ln" | "log2" | "log10" | "sqrt" | "sgn" | "sin" + | "cos" | "tan" | "asin" | "acos" | "atan" | "sinh" | "cosh" | "tanh" | "asinh" + | "acosh" | "atanh" | "deg" | "rad" | "minute" | "hour" | "day_of_week" + | "day_of_month" | "day_of_year" | "month" | "year" | "days_in_month" => 1, + "round" | "clamp_min" | "clamp_max" => 2, + "clamp" => 3, + _ => return None, + }) +} + #[cfg(test)] mod tests { use super::*; @@ -196,324 +211,3 @@ mod tests { .is_err()); } } - -#[cfg(test)] -mod projection_tests { - use super::*; - use crate::pre_asap::{Field, ProjectItem, QueryExpr, ScalarValue, Schema, Source}; - use std::rc::Rc; - fn project(expr: QueryExpr) -> QueryExpr { - QueryExpr::Project { - cols: vec![ProjectItem { - alias: Some("result".into()), - expr, - }], - qualifier: None, - child: Rc::new(QueryExpr::Scan { - source: Source::Table { - table_ref: "t".into(), - }, - predicates: vec![], - schema: Schema::new(vec![ - Field::plain("k", DataType::Utf8, false), - Field::plain("v", DataType::Int64, true), - ]), - }), - } - } - #[test] - fn canonical_projection_uses_map_signature_and_rejects_invalid_arity() { - let map = QueryExpr::FunctionCall { - name: "map".into(), - args: vec![QueryExpr::Column(0), QueryExpr::Column(1)], - }; - let schema = project(map.clone()).output_schema().unwrap(); - assert_eq!( - schema.fields[0].dtype, - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: true - } - ); - assert!(!schema.fields[0].nullable); - let lookup = QueryExpr::FunctionCall { - name: "asap_map_access".into(), - args: vec![map, QueryExpr::Literal(ScalarValue::Utf8("missing".into()))], - }; - assert_eq!( - project(lookup).output_schema().unwrap().fields[0], - Field::plain("result", DataType::Int64, true) - ); - assert!(project(QueryExpr::FunctionCall { - name: "map".into(), - args: vec![QueryExpr::Column(0)] - }) - .output_schema() - .is_err()); - } -} - -/// Resolve the bounded canonical `asap_struct_field(struct, selector)` operation. -/// Selectors are positive 1-based literal ordinals or exact literal field names. -/// The existing Struct fields remain the sole authority for type/nullability. -/// Dynamic/negative/defaulted selectors and nullable containers are intentionally -/// unsupported here; this is not a claim of complete native tupleElement support. -pub fn struct_field_type( - args: &[super::QueryExpr], - schema: &super::Schema, -) -> Result<(DataType, bool), String> { - use super::{QueryExpr, ScalarValue}; - let [input, selector] = args else { - return Err("struct field access requires a struct and constant selector".into()); - }; - let (dtype, nullable) = input - .scalar_type(schema) - .map_err(|error| error.to_string())?; - if nullable { - return Err("nullable struct container access is unsupported".into()); - } - let DataType::Struct { fields } = dtype else { - return Err("struct field access requires a Struct input".into()); - }; - let field = match selector { - QueryExpr::Literal(ScalarValue::Int64(index)) if *index > 0 => usize::try_from(*index - 1) - .ok() - .and_then(|index| fields.get(index)) - .ok_or("struct field ordinal is out of bounds")?, - QueryExpr::Literal(ScalarValue::Utf8(name)) => { - let mut matches = fields.iter().filter(|field| field.name == *name); - let field = matches.next().ok_or("struct field name does not exist")?; - if matches.next().is_some() { - return Err("struct field name is ambiguous".into()); - } - field - } - _ => { - return Err( - "struct field selector must be a positive ordinal or field-name literal".into(), - ) - } - }; - Ok((field.dtype.clone(), field.nullable)) -} - -#[cfg(test)] -mod struct_field_tests { - use super::*; - use crate::pre_asap::{Field, FieldDataType, QueryExpr, ScalarValue, Schema}; - fn schema() -> Schema { - Schema::new(vec![Field::plain( - "record", - DataType::Struct { - fields: vec![ - Field::new("ts", DataType::Int64, false), - Field::new( - "values", - DataType::List { - element: Box::new(Field::new("item", DataType::Float64, true)), - }, - true, - ), - ], - }, - false, - )]) - } - fn access(selector: QueryExpr) -> QueryExpr { - QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![QueryExpr::Column(0), selector], - } - } - #[test] - fn field_access_reuses_nested_field_type_and_nullability() { - let schema = schema(); - assert_eq!( - access(QueryExpr::Literal(ScalarValue::Int64(1))) - .scalar_type(&schema) - .unwrap(), - (DataType::Int64, false) - ); - let named = access(QueryExpr::Literal(ScalarValue::Utf8("values".into()))); - let ordinal = access(QueryExpr::Literal(ScalarValue::Int64(2))); - assert_eq!( - named.scalar_type(&schema).unwrap(), - ordinal.scalar_type(&schema).unwrap() - ); - assert_eq!( - named.scalar_type(&schema).unwrap(), - ( - DataType::List { - element: Box::new(Field::new("item", DataType::Float64, true)) - }, - true - ) - ); - let roundtrip: QueryExpr = - serde_json::from_str(&serde_json::to_string(&named).unwrap()).unwrap(); - assert_eq!(roundtrip, named); - } - #[test] - fn unsupported_field_access_is_an_error_not_placeholder_typing() { - for selector in [ - QueryExpr::Column(0), - QueryExpr::Literal(ScalarValue::Int64(0)), - QueryExpr::Literal(ScalarValue::Int64(-1)), - QueryExpr::Literal(ScalarValue::Int64(3)), - QueryExpr::Literal(ScalarValue::Utf8("missing".into())), - ] { - assert!(access(selector).scalar_type(&schema()).is_err()); - } - let mut ambiguous = schema(); - if let FieldDataType::Plain(DataType::Struct { fields }) = &mut ambiguous.fields[0].dtype { - fields.push(Field::new("ts", DataType::Utf8, false)); - } - assert!(access(QueryExpr::Literal(ScalarValue::Utf8("ts".into()))) - .scalar_type(&ambiguous) - .is_err()); - let mut nullable = schema(); - nullable.fields[0].nullable = true; - assert!(access(QueryExpr::Literal(ScalarValue::Int64(1))) - .scalar_type(&nullable) - .is_err()); - } -} - -/// Canonical element lookup over a declared Map or List. Map lookup retains its -/// existing key/default contract. List lookup is one-based, supports negative -/// indices, and returns the declared element default when a dynamic index is -/// out of range. Literal zero is conservatively rejected because native array -/// behavior depends on whether the input array is constant. Nullable containers -/// are unsupported; nullable indices produce nullable results. -pub fn element_access_type( - args: &[super::QueryExpr], - schema: &super::Schema, -) -> Result<(DataType, bool), String> { - use super::{QueryExpr, ScalarValue}; - let [input, index] = args else { - return Err("element access requires a collection and index".into()); - }; - let source = input.scalar_type(schema).map_err(|e| e.to_string())?; - let key = index.scalar_type(schema).map_err(|e| e.to_string())?; - match &source.0 { - DataType::Map { .. } => MapScalarFunction::Access.output_type(&[source, key]), - DataType::List { element } => { - if source.1 { - return Err("nullable List container access is unsupported".into()); - } - if !matches!(key.0, DataType::Int64 | DataType::Null) { - return Err("List index must have integer type".into()); - } - if matches!(index, QueryExpr::Literal(ScalarValue::Int64(0))) { - return Err( - "literal zero List index is unsupported without constant-array proof".into(), - ); - } - Ok(( - element.dtype.clone(), - element.nullable || key.1 || key.0 == DataType::Null, - )) - } - _ => Err("element access requires a Map or List".into()), - } -} - -#[cfg(test)] -mod element_access_tests { - use super::*; - use crate::pre_asap::{Field, QueryExpr, ScalarValue, Schema}; - fn access(index: QueryExpr) -> QueryExpr { - QueryExpr::FunctionCall { - name: "asap_element_access".into(), - args: vec![QueryExpr::Column(0), index], - } - } - #[test] - fn list_index_preserves_nested_element_metadata() { - let element = DataType::Struct { - fields: vec![ - Field::new("ts", DataType::Int64, false), - Field::new("value", DataType::Float64, true), - ], - }; - let schema = Schema::new(vec![ - Field::plain( - "samples", - DataType::List { - element: Box::new(Field::new("item", element.clone(), false)), - }, - false, - ), - Field::plain("i", DataType::Int64, true), - ]); - for index in [1, -1, 100] { - assert_eq!( - access(QueryExpr::Literal(ScalarValue::Int64(index))) - .scalar_type(&schema) - .unwrap(), - (element.clone(), false) - ); - } - assert_eq!( - access(QueryExpr::Column(1)).scalar_type(&schema).unwrap(), - (element.clone(), true) - ); - assert!(access(QueryExpr::Literal(ScalarValue::Int64(0))) - .scalar_type(&schema) - .is_err()); - assert!(access(QueryExpr::Literal(ScalarValue::Float64(1.0))) - .scalar_type(&schema) - .is_err()); - let nested = QueryExpr::FunctionCall { - name: "asap_struct_field".into(), - args: vec![ - access(QueryExpr::Literal(ScalarValue::Int64(1))), - QueryExpr::Literal(ScalarValue::Int64(2)), - ], - }; - assert_eq!( - nested.scalar_type(&schema).unwrap(), - (DataType::Float64, true) - ); - let roundtrip: QueryExpr = - serde_json::from_value(serde_json::to_value(&nested).unwrap()).unwrap(); - assert_eq!(roundtrip, nested); - } - #[test] - fn generic_map_lookup_reuses_legacy_signature() { - let schema = Schema::new(vec![Field::plain( - "m", - DataType::Map { - key: Box::new(DataType::Utf8), - value: Box::new(DataType::Int64), - value_nullable: false, - }, - false, - )]); - let key = QueryExpr::Literal(ScalarValue::Utf8("k".into())); - let legacy = QueryExpr::FunctionCall { - name: "asap_map_access".into(), - args: vec![QueryExpr::Column(0), key.clone()], - }; - assert_eq!( - access(key).scalar_type(&schema).unwrap(), - legacy.scalar_type(&schema).unwrap() - ); - } -} - -/// Closed, namespaced contracts for PromQL pointwise float functions. -/// Date functions consume Unix seconds; `timestamp` remains a sample-selection -/// operation because its operand is a sample timestamp rather than its value. -pub fn promql_function_arity(name: &str) -> Option { - Some(match name.strip_prefix("promql_")? { - "abs" | "ceil" | "floor" | "exp" | "ln" | "log2" | "log10" | "sqrt" | "sgn" | "sin" - | "cos" | "tan" | "asin" | "acos" | "atan" | "sinh" | "cosh" | "tanh" | "asinh" - | "acosh" | "atanh" | "deg" | "rad" | "minute" | "hour" | "day_of_week" - | "day_of_month" | "day_of_year" | "month" | "year" | "days_in_month" => 1, - "round" | "clamp_min" | "clamp_max" => 2, - "clamp" => 3, - _ => return None, - }) -} diff --git a/crates/types/src/pre_asap/schema.rs b/crates/types/src/pre_asap/schema.rs index 77c8ebe92..7eec788b7 100644 --- a/crates/types/src/pre_asap/schema.rs +++ b/crates/types/src/pre_asap/schema.rs @@ -325,82 +325,6 @@ impl TryFrom for Schema { /// label map. `$` cannot occur in a user PromQL label name. pub const PROMQL_SERIES_IDENTITY: &str = "$promql_series_identity"; -/// Resolve a PromQL root to rows carrying [`PROMQL_SERIES_IDENTITY`] before -/// candidate search. `closed` describes physical columns here: the final -/// column contains every dynamic source label. It does not assert that the -/// query's projected labels are the full label set. -/// -/// This realization supports explicit `by` grouping and per-series computation. -/// Operators that rewrite or implicitly match dynamic label sets require their -/// own realization; they must not accidentally treat the opaque identity as a -/// user label or silently discard it. -pub fn with_promql_series_identity(root: &super::QueryExpr) -> Result { - use super::{QueryExpr, Source}; - use std::rc::Rc; - let mut root = root.clone(); - fn visit(node: &mut QueryExpr) -> Result<(), String> { - match node { - QueryExpr::Scan { - source: Source::TimeSeries { .. }, - schema, - .. - } => { - if schema - .fields - .iter() - .any(|column| column.name == PROMQL_SERIES_IDENTITY) - { - return Err("source already contains a physical series identity".into()); - } - if schema.closed { - return Err("dynamic series identity requires an open PromQL source".into()); - } - schema - .fields - .push(Field::plain(PROMQL_SERIES_IDENTITY, DataType::Utf8, false)); - schema.closed = true; - Ok(()) - } - QueryExpr::TimeRange { child, .. } - | QueryExpr::Limit { child, .. } - | QueryExpr::TimeShift { child, .. } - | QueryExpr::PromqlSubquery { child, .. } - | QueryExpr::PromqlScalarFromVector(child) - | QueryExpr::PromqlRelabel { child, .. } => visit(Rc::make_mut(child)), - // Constants read no series. - QueryExpr::PromqlScalarBridge(_) - | QueryExpr::EvalTimestamp - | QueryExpr::Literal(super::ScalarValue::Float64(_)) => Ok(()), - QueryExpr::PromqlVectorFromScalar(child) => visit(Rc::make_mut(child)), - QueryExpr::BinaryOp { lhs, rhs, .. } => { - visit(Rc::make_mut(lhs))?; - visit(Rc::make_mut(rhs)) - } - QueryExpr::Concat { children, .. } => { - for child in children { - visit(child)?; - } - Ok(()) - } - QueryExpr::Aggregate { child, .. } => visit(Rc::make_mut(child)), - QueryExpr::Sort { - child, - partition_by, - .. - } => { - if partition_by.is_without() { - return Err("dynamic without ranking requires label-set projection".into()); - } - visit(Rc::make_mut(child)) - } - _ => Err("operator has no dynamic series-identity realization".into()), - } - } - visit(&mut root)?; - root.output_schema().map_err(|error| error.to_string())?; - Ok(root) -} - impl Schema { pub fn has_promql_series_identity(&self) -> bool { self.closed @@ -605,12 +529,4 @@ mod tests { assert_eq!(back, c); assert_eq!(back.table.as_deref(), Some("hosts")); } - // Direct scalar literals remain valid vector inputs when series typing runs. - #[test] - fn series_identity_accepts_direct_vector_literal() { - let root = super::super::QueryExpr::PromqlVectorFromScalar(std::rc::Rc::new( - super::super::QueryExpr::Literal(super::super::ScalarValue::Float64(1.0)), - )); - assert!(with_promql_series_identity(&root).is_ok()); - } } diff --git a/crates/types/src/pre_asap/schema_resolver.rs b/crates/types/src/pre_asap/schema_resolver.rs deleted file mode 100644 index a9afff2cb..000000000 --- a/crates/types/src/pre_asap/schema_resolver.rs +++ /dev/null @@ -1,492 +0,0 @@ -//! The **SchemaResolver** — name resolution as an explicit pass. -//! -//! [`SchemaResolver::resolve_schema`] produces the complete, self-contained [`Schema`] every -//! `ColumnId` in the canonical DAG indexes into. [`resolve`](super::resolve) -//! then becomes purely structural: it threads the SchemaResolver's schema and -//! positional resolution downstream is **total**. -//! -//! The default [`UsageDerivedCatalog`] knows nothing — every schema is derived -//! purely from the query's own usage. That is the honest state for the -//! observability domain (metric label sets are open-ended). A registry-backed -//! `SchemaCatalog` is future work; the `SchemaResolver` pass does not change when it -//! lands, only the catalog impl swaps. - -use super::expr_ir::ColumnRef; -use super::query_expr::UnresolvedQueryExpr; -use super::schema::{DataType, Field, Schema}; - -/// The DB / source-schema metadata source — resolves a source (metric / -/// table) name to its known columns. -/// Source of truth for a source's columns — the "catalog". `SqlCatalog` backs -/// it for SQL; PromQL uses [`UsageDerivedCatalog`] (returns `None`) until a -/// registry-backed impl (returning a metric's known label set) drops in here. -/// Distinct from `Scan.schema`, which is the *resolved* binding schema this -/// feeds — the catalog is the input, the schema is the result. Even a -/// registry-backed PromQL catalog yields an **open** schema -/// ([`Schema::closed`] `= false`): a metric's -/// labels are per-series and time-varying, so the registry is a superset hint, -/// not a per-row contract. -pub trait SchemaCatalog { - /// Columns known for `source`. `None` when unknown — the [`SchemaResolver`] then - /// falls back to a usage-derived column set. - fn columns_for(&self, source: &str) -> Option>; -} - -/// The default catalog: knows nothing. Every schema the [`SchemaResolver`] produces -/// is derived purely from the query's own usage. -pub struct UsageDerivedCatalog; - -impl SchemaCatalog for UsageDerivedCatalog { - fn columns_for(&self, _source: &str) -> Option> { - None - } -} - -/// The explicit name-resolution pass. -pub struct SchemaResolver { - catalog: C, -} - -impl Default for SchemaResolver { - fn default() -> Self { - Self::new() - } -} - -impl SchemaResolver { - pub fn new() -> Self { - Self { - catalog: UsageDerivedCatalog, - } - } -} - -impl SchemaResolver { - pub fn with_catalog(catalog: C) -> Self { - Self { catalog } - } - - /// Resolve the complete [`Schema`] in scope for a query rooted at `dag`. - /// - /// Contains the time axis, the synthetic `value` column, and one column - /// per distinct name referenced anywhere in the DAG — so positional - /// `ColumnId` resolution downstream is total. - pub fn resolve_schema(&self, dag: &UnresolvedQueryExpr) -> Schema { - self.resolve_schema_with_inherited(dag, &[]) - } - - /// Like [`resolve_schema`](Self::resolve_schema), but also seeds `inherited` label names that are - /// referenced by an **enclosing** scope rather than by `dag` itself. This is - /// how an independently-bound `BinaryOp` side (each side re-binds against its - /// own sub-DAG) still sees an outer aggregate's group keys — e.g. the - /// `__name__` / `job` in `sum by (__name__)(a or b)`, which appear in neither - /// side's own matchers (issue #52). - pub fn resolve_schema_with_inherited( - &self, - dag: &UnresolvedQueryExpr, - inherited: &[String], - ) -> Schema { - let mut columns: Vec = leftmost_scan_name(dag) - .and_then(|name| self.catalog.columns_for(name)) - .unwrap_or_else(default_leaf_columns); - - // Ensure the (ts, value) floor is present. - for floor in default_leaf_columns() { - if !columns.iter().any(|c| c.name == floor.name) { - columns.push(floor); - } - } - - // Append one column per referenced-but-unknown name (group keys etc.), - // plus any inherited-from-enclosing-scope names. - let referenced = collect_referenced_columns(dag); - for name in referenced.iter().chain(inherited) { - if !columns.iter().any(|c| c.name == *name) { - columns.push(Field::plain(name.clone(), DataType::Utf8, true)); - } - } - - let time_index = columns.iter().position(|c| c.name == "ts"); - Schema { - fields: columns, - time_index, - unique_keys: Vec::new(), - // Usage-derived (schemaless PromQL): the metric's full label set is - // open and runtime-only, so this lists only what the query references. - closed: false, - } - } -} - -/// The conventional PromQL leaf shape: `(ts: Timestamp, value: Float64)`. -fn default_leaf_columns() -> Vec { - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - ] -} - -/// Push a `ColumnRef`'s bare name (the schema-seedable identifier). `Qualified` -/// collapses to its `name`; `SampleValue`/`Wildcard` carry no name. -fn push_ref_name(c: &ColumnRef, out: &mut Vec) { - match c { - ColumnRef::Named(n) => out.push(n.clone()), - ColumnRef::Qualified { name, .. } => out.push(name.clone()), - ColumnRef::SampleValue | ColumnRef::Wildcard => {} - } -} - -/// The leftmost `Scan`'s source name in a canonical (`UnresolvedQueryExpr`) DAG — -/// the [`collect_referenced_columns`] counterpart to what a dedicated -/// `Source` leaf type would carry as a method; the canonical DAG's `Scan` -/// leaf needs this walk written out instead. -fn leftmost_scan_name(dag: &UnresolvedQueryExpr) -> Option<&str> { - use UnresolvedQueryExpr as QE; - match dag { - QE::Scan { source, .. } => Some(match source { - super::query_expr::Source::TimeSeries { metric } => metric.as_str(), - super::query_expr::Source::Table { table_ref } => table_ref.as_str(), - }), - // A scalar bridge's child is a scalar-sub-language leaf (in practice - // always a `Literal`, issue #220) — never a `Scan`, same as - // `EvalTimestamp`. - QE::PromqlScalarBridge(_) | QE::EvalTimestamp | QE::CurrentTimestamp => None, - QE::PromqlVectorFromScalar(child) | QE::PromqlScalarFromVector(child) => { - leftmost_scan_name(child) - } - QE::PromqlRelabel { child, .. } - | QE::PromqlInfoEnrich { child, .. } - | QE::PromqlSeriesSample { child, .. } - | QE::Filter { child, .. } - | QE::Project { child, .. } - | QE::Aggregate { child, .. } - | QE::Dedup { child, .. } - | QE::Sort { child, .. } - | QE::Limit { child, .. } - | QE::PromqlSubquery { child, .. } - | QE::TimeRange { child, .. } - | QE::TimeShift { child, .. } - | QE::SQLWindowFunc { child, .. } => leftmost_scan_name(child), - QE::Concat { children, .. } => children.first().and_then(leftmost_scan_name), - QE::Join { left, .. } | QE::SetOp { left, .. } | QE::BinaryOp { lhs: left, .. } => { - leftmost_scan_name(left) - } - // The scalar variants (issue #205) never appear as a direct - // `leftmost_scan_name` target — every reachable one sits behind a - // wrapper field (`Predicate`, `ProjectItem`, …) this walk never - // descends into; it only follows the relational skeleton. - QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. } => None, - } -} - -/// Collect every distinct column name referenced anywhere in `dag` that -/// resolves positionally — every place a front end constructing -/// [`QueryExpr`](super::query_expr::QueryExpr) directly (issue -/// #179) puts a name-based reference: `Scan.predicates`, `Aggregate`'s -/// `reduction`/`having`/per-measure `col`, `Dedup.cols`, `PromqlSeriesSample.by`, -/// `Filter.pred`, `Project.cols`, `Sort.keys`/`partition_by`, -/// `SQLWindowFunc.args`/`partition_by`/`order_by`, `Join.pred`, `PromqlRelabel.value`. -/// The SchemaResolver seeds these into the usage-derived leaf so positional -/// resolution downstream is total. -pub(crate) fn collect_referenced_columns(dag: &UnresolvedQueryExpr) -> Vec { - use UnresolvedQueryExpr as QE; - fn named(expr: &UnresolvedQueryExpr, out: &mut Vec) { - for c in expr.columns_referenced() { - push_ref_name(c, out); - } - } - fn group_keys(g: &super::query_expr::GroupKeys, out: &mut Vec) { - g.keys().iter().for_each(|k| push_ref_name(k, out)); - } - fn measure_cols(measures: &[super::agg_intent::AggIntent], out: &mut Vec) { - for m in measures { - for c in m.input_cols() { - push_ref_name(&c, out); - } - } - } - fn walk(node: &UnresolvedQueryExpr, out: &mut Vec) { - match node { - QE::Scan { predicates, .. } => { - for super::query_expr::Predicate(p) in predicates { - named(p, out); - } - } - QE::Aggregate { - reduction, - measures, - filters, - having, - child, - .. - } => { - if let super::query_expr::Reduction::Reduce(by) = reduction { - group_keys(by, out); - } - measure_cols(measures, out); - for super::query_expr::Predicate(f) in filters.iter().flatten() { - named(f, out); - } - if let Some(super::query_expr::Predicate(h)) = having { - named(h, out); - } - walk(child, out); - } - QE::Dedup { cols, child } => { - cols.iter().for_each(|c| push_ref_name(c, out)); - walk(child, out); - } - QE::PromqlSeriesSample { by, child, .. } => { - group_keys(by, out); - walk(child, out); - } - QE::Filter { pred, child } => { - named(&pred.0, out); - walk(child, out); - } - QE::Project { cols, child, .. } => { - for item in cols { - named(&item.expr, out); - } - walk(child, out); - } - QE::Sort { - keys, - partition_by, - child, - } => { - for k in keys { - named(&k.expr, out); - } - group_keys(partition_by, out); - walk(child, out); - } - QE::SQLWindowFunc { - args, - partition_by, - order_by, - child, - .. - } => { - for a in args { - named(a, out); - } - group_keys(partition_by, out); - for k in order_by { - named(&k.expr, out); - } - walk(child, out); - } - QE::PromqlRelabel { value, child, .. } => { - named(value, out); - walk(child, out); - } - QE::Join { - pred, left, right, .. - } => { - named(&pred.0, out); - walk(left, out); - walk(right, out); - } - QE::EvalTimestamp | QE::CurrentTimestamp => {} - // The bridged child is a genuine scalar-sub-language position now - // (issue #220) — peel its column refs off with `named`, same as - // every other scalar-typed field (`Scan.predicates`, - // `Filter.pred`, …). In practice it's always a `Literal`, which - // references no columns, so this is a no-op today. - QE::PromqlScalarBridge(inner) => named(inner, out), - QE::PromqlVectorFromScalar(child) | QE::PromqlScalarFromVector(child) => { - walk(child, out) - } - QE::PromqlInfoEnrich { child, .. } - | QE::Limit { child, .. } - | QE::PromqlSubquery { child, .. } - | QE::TimeRange { child, .. } - | QE::TimeShift { child, .. } => walk(child, out), - QE::Concat { - children, - discriminator_unique_key, - } => { - // Same treatment as `Dedup.cols` above: an own-field - // `ColumnRef` must be seeded here too, or a discriminator - // column that isn't otherwise referenced anywhere else in - // the DAG (plausible — a raw usage-derived label, not one a - // `Project`/relabel freshly created) is absent from the - // SchemaResolver's usage-derived fallback schema, and - // `resolve.rs`'s later `resolve_column_ref` call fails with - // `NotFound` for a column the caller correctly named. - if let Some(key) = discriminator_unique_key { - push_ref_name(key.discriminator(), out); - key.inner_key().iter().for_each(|c| push_ref_name(c, out)); - } - children.iter().for_each(|c| walk(c, out)); - } - QE::SetOp { left, right, .. } => { - walk(left, out); - walk(right, out); - } - QE::BinaryOp { lhs, rhs, .. } => { - walk(lhs, out); - walk(rhs, out); - } - // The scalar variants (issue #205) never appear as a direct - // `walk` target — every reachable one is peeled off first by - // `named` at whichever operator field holds it (`Scan.predicates`, - // `Filter.pred`, `Project.cols`, …). - QE::Column(_) - | QE::Literal(_) - | QE::Compare { .. } - | QE::BoolAnd(_) - | QE::BoolOr(_) - | QE::Not(_) - | QE::IsNull(_) - | QE::IsNotNull(_) - | QE::Cast { .. } - | QE::InList { .. } - | QE::FunctionCall { .. } - | QE::Arithmetic { .. } - | QE::Case { .. } => { - unreachable!("walk reached a scalar QueryExpr variant directly: {node:?}") - } - } - } - let mut out: Vec = Vec::new(); - walk(dag, &mut out); - out.sort(); - out.dedup(); - out -} - -#[cfg(test)] -mod tests { - use std::rc::Rc; - - use super::super::query_expr::{GroupKeys, Source}; - use super::*; - - fn src(name: &str) -> UnresolvedQueryExpr { - UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: name.into(), - }, - predicates: vec![], - schema: None, - } - } - - // Both correlation inputs must seed a usage-derived schema before positional resolution. - #[test] - fn pearson_corr_inputs_seed_usage_derived_schema() { - use crate::pre_asap::{AggIntent, Reduction}; - let dag = UnresolvedQueryExpr::Aggregate { - reduction: Reduction::by(vec![]), - measures: vec![AggIntent::PearsonCorr { - left: ColumnRef::Named("x".into()), - right: ColumnRef::Named("y".into()), - }], - output_names: vec![], - filters: vec![], - having: None, - child: Rc::new(src("m")), - }; - assert_eq!(collect_referenced_columns(&dag), vec!["x", "y"]); - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("x").is_some()); - assert!(schema.column_id("y").is_some()); - } - - #[test] - fn bare_source_yields_ts_value_floor() { - let schema = SchemaResolver::new().resolve_schema(&src("m")); - assert_eq!(schema.fields.len(), 2); - assert_eq!(schema.fields[0].name, "ts"); - assert_eq!(schema.fields[1].name, "value"); - assert_eq!(schema.time_index, Some(0)); - } - - #[test] - fn sort_partition_keys_land_in_schema() { - // Per-group ranking keys (`topk by (host)` → `Sort.partition_by`) must be - // seeded into the usage-derived leaf so they resolve positionally. - let dag = UnresolvedQueryExpr::Sort { - keys: vec![super::super::query_expr::SortKey { - expr: UnresolvedQueryExpr::Column(ColumnRef::SampleValue), - ascending: false, - nulls_first: false, - }], - partition_by: GroupKeys::by(vec![ColumnRef::Named("host".into())]), - child: Rc::new(src("hits")), - }; - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("host").is_some()); - } - - /// Issue #228 review: a `Concat`'s `discriminator_unique_key` columns — - /// even one referenced nowhere else in the DAG — must be seeded into - /// the usage-derived fallback schema, exactly like `Dedup.cols`, or - /// `resolve.rs`'s later `resolve_column_ref` fails `NotFound` for a - /// column the caller correctly named. - #[test] - fn concat_discriminator_key_is_seeded_into_the_resolver_schema() { - let dag = UnresolvedQueryExpr::concat_with_discriminator( - vec![src("m")], - ColumnRef::Named("phi".into()), - vec![ColumnRef::Named("host".into())], - ); - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!( - schema.column_id("phi").is_some(), - "discriminator column must be seeded" - ); - assert!( - schema.column_id("host").is_some(), - "inner_key column must be seeded" - ); - } - - #[test] - fn inherited_names_are_seeded_alongside_referenced() { - // A `BinaryOp` side re-binds against its own sub-DAG, but must still see - // an enclosing aggregate's group key (`__name__` / `job`) that appears in - // neither side's own matchers (issue #52). `resolve_schema_with_inherited` seeds it. - let schema = - SchemaResolver::new().resolve_schema_with_inherited(&src("m"), &["__name__".into()]); - assert!(schema.column_id("__name__").is_some()); - // `resolve_schema` (no inheritance) does not conjure it. - let plain = SchemaResolver::new().resolve_schema(&src("m")); - assert!(plain.column_id("__name__").is_none()); - } - - #[test] - fn custom_catalog_supplies_base_columns() { - struct FixedCatalog; - impl SchemaCatalog for FixedCatalog { - fn columns_for(&self, source: &str) -> Option> { - (source == "known").then(|| { - vec![ - Field::plain("ts", DataType::Timestamp, false), - Field::plain("value", DataType::Float64, false), - Field::plain("datacenter", DataType::Utf8, false), - ] - }) - } - } - let schema = SchemaResolver::with_catalog(FixedCatalog).resolve_schema(&src("known")); - let dc = schema - .column_id("datacenter") - .and_then(|id| schema.fields.get(id)); - assert!(matches!(dc, Some(c) if !c.nullable)); - } -} diff --git a/crates/types/tests/physical_export.rs b/crates/types/tests/physical_export.rs index e396fbaa8..485182384 100644 --- a/crates/types/tests/physical_export.rs +++ b/crates/types/tests/physical_export.rs @@ -295,3 +295,114 @@ fn an_empty_batch_is_an_error() { )) )); } + +/// A summary over another summary exports a typed maintenance edge between +/// them, and the document round-trips and rejects unknown fields and versions. +#[test] +fn exports_summary_over_summary_as_typed_precompute_edges() { + use asap_types::ir::physical_export::{ + compile_physical_asap_dag_with_node_ids, GroupingEdgeCompatibility, WindowEdgeCompatibility, + }; + use asap_types::post_asap::ExecutionDataState; + let scan = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::TimeSeries { metric: "m".into() }, + predicates: vec![], + schema: Schema::with_time_index( + vec![ + Field::plain("ts", DataType::Timestamp, false), + Field::plain("value", DataType::Float64, false), + ], + 0, + vec![], + ), + })) + .unwrap(); + let sum = |child, column| { + OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { + child, + family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(column), + reduction: Reduction::by(vec![]), + grouping: Default::default(), + filter: None, + })) + .unwrap() + }; + let inner = sum(scan, ColumnRef::SampleValue); + let outer = sum(Rc::clone(&inner), ColumnRef::Named("state".into())); + let root = OperatorNode::new_shared(Operator::ASAP(ASAPOp::FinalizeExactAccumulator { + child: outer, + })) + .unwrap(); + let mut memo = TimingMemo::new(); + let timed = + apply_lifecycle_timings(&root, &LifecycleAssignment::default_maintained(), &mut memo) + .unwrap(); + let compiled = compile_physical_asap_dag_with_node_ids(&timed).unwrap(); + assert_eq!(compiled.node_ids.node_id(&timed), Some(3)); + assert_eq!( + compiled.node_ids.node_id(memo.timed(&inner).unwrap()), + Some(1) + ); + let dag = compiled.dag; + assert_eq!(dag.roots, vec![3]); + assert_eq!( + dag.nodes[1].output_state, + ExecutionDataState::INGESTION_SUMMARY + ); + assert_eq!( + dag.nodes[2].output_state, + ExecutionDataState::INGESTION_SUMMARY + ); + let dependency = dag + .edges + .iter() + .find(|e| e.producer == 1 && e.consumer == 2) + .unwrap(); + assert_eq!(dependency.data_state, ExecutionDataState::INGESTION_SUMMARY); + assert_eq!(dependency.grouping, GroupingEdgeCompatibility::Identical); + assert_eq!( + dependency.window, + WindowEdgeCompatibility::RequiresAlignedPanePhaseOrExactWindowEdgeResidual + ); + assert!(matches!( + dependency.intermediate_schema.fields[0].dtype, + FieldDataType::ExactAggregate(ExactKind::Sum, _) + )); + assert!(matches!( + dag.nodes[2].payload, + Operator::ASAP(ASAPOp::SummaryAgg { + family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + reduction: Reduction::Reduce(_), + .. + }) + )); + + let document = PhysicalASAPDAGDocument::new(dag); + let decoded: PhysicalASAPDAGDocument = + serde_json::from_str(&serde_json::to_string(&document).unwrap()).unwrap(); + assert_eq!(decoded, document); + decoded.validate().unwrap(); + let mut old_version = document.clone(); + old_version.schema_version = 1; + assert_eq!( + old_version.validate(), + Err(PhysicalASAPDAGValidationError::UnsupportedVersion(1)) + ); + let mut unknown = serde_json::to_value(&document).unwrap(); + unknown["unexpected"] = serde_json::json!(true); + assert!(serde_json::from_value::(unknown).is_err()); +} + +/// Node-id-keyed bindings serialize in numeric id order. +#[test] +fn physical_node_ids_serialize_in_deterministic_binding_order() { + use asap_types::ir::physical_export::PhysicalASAPNodeId; + let mut bindings = std::collections::BTreeMap::::new(); + bindings.insert(10, "materialization-10"); + bindings.insert(2, "query-2"); + assert_eq!( + serde_json::to_string(&bindings).unwrap(), + r#"{"2":"query-2","10":"materialization-10"}"# + ); +} diff --git a/crates/types/tests/planner_vocabulary.rs b/crates/types/tests/planner_vocabulary.rs index f3d53cf59..82f1cbc1a 100644 --- a/crates/types/tests/planner_vocabulary.rs +++ b/crates/types/tests/planner_vocabulary.rs @@ -1,6 +1,5 @@ use asap_types::ir::physical_export::WindowEdgeCompatibility; use asap_types::post_asap::{validate_pane_coverage, PaneLayout, WindowEdgeCoverage}; -use asap_types::pre_asap::{SchemaResolver, Source, UnresolvedQueryExpr}; use asap_types::resources::{PhysicalHandoffBytes, PhysicalHandoffKind}; // Renamed pane APIs still read and emit the deployed wire contract. @@ -29,18 +28,11 @@ fn window_edge_names_preserve_wire_values() { ); } -// External consumers can use the new resolver and resource names without changing behavior. +// External consumers can use the new resource names without changing behavior. +// (The schema-resolver half moved with the resolver to `asap-frontend-common`; +// `schema_resolver::tests::bare_source_yields_ts_value_floor` covers it.) #[test] -fn renamed_schema_and_handoff_apis_are_public() { - let dag = UnresolvedQueryExpr::Scan { - source: Source::TimeSeries { - metric: "requests".into(), - }, - predicates: vec![], - schema: None, - }; - let schema = SchemaResolver::new().resolve_schema(&dag); - assert!(schema.column_id("value").is_some()); +fn renamed_handoff_apis_are_public() { let bytes = PhysicalHandoffBytes { network_bytes: 12, materialization_bytes: 4, diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index f5f8f5cc2..181301237 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -52,7 +52,7 @@ fn values_contract_is_checked() { } /// Scalar typing validates every branch and never assigns placeholder types. #[test] -fn scalar_type_ruless_fail_closed() { +fn scalar_type_rules_fail_closed() { for expr in [ ScalarExpr::Column(99), ScalarExpr::FunctionCall { diff --git a/docs/design_docs/architecture/README.md b/docs/design_docs/architecture/README.md index 8b936eafe..e5f8ff2e7 100644 --- a/docs/design_docs/architecture/README.md +++ b/docs/design_docs/architecture/README.md @@ -17,7 +17,7 @@ flowchart TD W["PlanningWorkload: query demand + optional data facts"] F["Frontend dependencies: SQL catalog or PromQL time"] E["Strategy, accuracy model, and applicable evidence"] - PRE["Frontend lowering → canonical Pre-ASAP QueryExpr roots"] + PRE["Frontend lowering → canonical Pre-ASAP OperatorNode roots"] SEARCH["Whole-workload candidate search: sharing, legality, accuracy"] SPACE["CandidateLogicalASAPDAGs: compact logical candidate DAG space"] RANK["Optional cost_sorted: ranked inspection view"] @@ -58,7 +58,8 @@ an unsupported physical alternative into a deployable plan. | Area | Main crate or module | Responsibility | |---|---|---| -| Shared IR | `asap-types` | Pre-ASAP and Post-ASAP expressions, schemas, workloads, guarantees, and exported plan data | +| Shared IR | `asap-types` | The unified operator IR (`ir`: one `OperatorNode` before and after ASAP optimization), schemas, workloads, guarantees, and exported plan data | +| Front-end common | `frontend-common` | Name-based `UnresolvedOp` tree shared by the front ends, and `resolve_root` into the operator IR | | Query frontends | `frontend-sql`, `frontend-promql`, `frontend-metricsql` | Parse source languages and produce canonical Pre-ASAP queries | | ASAP-aware mapping | `asap-aware-mapping` | Candidate generation, CSE, legality, accuracy propagation, lifecycle expansion, costing, and ranking | | Developer inspection | `devtools` | Expose planner DAGs, alternatives, decisions, and explanations for inspection | diff --git a/docs/design_docs/architecture/asap-aware-mapping.md b/docs/design_docs/architecture/asap-aware-mapping.md index 2600fc42d..78d3e0f9d 100644 --- a/docs/design_docs/architecture/asap-aware-mapping.md +++ b/docs/design_docs/architecture/asap-aware-mapping.md @@ -44,7 +44,7 @@ budgets; deployment belongs to a later stage. - **Replacement Sub-DAG**: A candidate post-ASAP sub-DAG to replace a target sub-DAG. For example, a quantile aggregation may have KLL, DDSketch, and exact aggregation as alternatives. - **ReplacementStrategy**: A rule to recognize a target Sub-DAG and produces one or more valid replacement Sub-DAGs. - **Candidate Plan**: A complete post-ASAP plan formed by choosing compatible replacement alternatives across the plan. -- **Maintained population**: A multiset of qualifying records retained across evaluations and updated as members enter, change, leave or expire; multiple readouts can share this state. +- **Maintained population**: A multiset of qualifying records retained across evaluations and updated as members enter, change, leave or expire; multiple evaluations can share this state. - **Cost Model**: A model used to compare valid candidate plans according to criteria such as storage, update cost, query latency, and accuracy. The distinction between **ReplacementStrategy** and **Candidate Plan** is important. A ReplacementStrategy generates alternatives at a decision point, while a candidate plan is a complete plan that combines choices across all relevant decision points. diff --git a/docs/design_docs/architecture/evidence-dependent-candidates.md b/docs/design_docs/architecture/evidence-dependent-candidates.md index abdfbdb90..9dcbba3fd 100644 --- a/docs/design_docs/architecture/evidence-dependent-candidates.md +++ b/docs/design_docs/architecture/evidence-dependent-candidates.md @@ -15,7 +15,7 @@ target)` returns `true`. **Uncertified** means Planner cannot make that claim: the guarantee is absent, contains unknown terms, or is known not to meet the target. An uncertified summary may still be a well-formed logical candidate; this label says nothing about whether the backend can physically execute it. -The exact `KeepPreAsap` path has an exact guarantee. +The exact path (the pre-ASAP sub-DAG kept by `retain_exact`) has an exact guarantee. | State | Planner representation | Consequence / next step | |---|---|---| @@ -88,7 +88,7 @@ The default `global_selection()` skips summaries that `has_missing_accuracy_evidence()` identifies as uncertified. Its `GlobalSelection::assemble_selected_dag()` result is a selected logical plan, not an instruction to deploy every candidate in `CandidateLogicalASAPDAGs`. If no alternative -is chosen at a site, DAG assembly retains the exact `KeepPreAsap` path. The +is chosen at a site, DAG assembly retains the exact pre-ASAP sub-DAG. The backend can inspect alternatives, apply its own evidence and policy, then choose a physically supported one; it must not equate candidate presence with approval. Models may explicitly opt into qualitative candidate ranking when no @@ -109,7 +109,7 @@ backend. | PromQL input | Before this PR | After this PR | |---|---|---| | `count by(job)(up)` with an ε/δ target | Hydra's shared CMS/CountSketch alternatives are absent: missing shared-grid bounds make the strategy decline the target. | Both Hydra alternatives remain in `CandidateLogicalASAPDAGs` with symbolic unknown bound/probability terms. `has_missing_accuracy_evidence()` is true; default `global_selection()` does not choose either as a certified answer. | -| `entropy_over_time(m[5m])` with an ε target | The uncalibrated frequency readout has no `SummaryEstimate` candidate. | Its `SummaryEstimate` remains inspectable with `guarantee: None`. Default selection still skips it, so candidate visibility is not an accuracy certificate. | +| `entropy_over_time(m[5m])` with an ε target | The uncalibrated frequency evaluation has no `SummaryEstimate` candidate. | Its `SummaryEstimate` remains inspectable with `guarantee: None`. Default selection still skips it, so candidate visibility is not an accuracy certificate. | | `quantile_over_time(0.9,data[5m]) / quantile_over_time(0.5,data[5m])` with an ε target | The uncertified direct DDSketch ratio is **already** visible because of #449. | Still visible with `guarantee: None`, and still skipped by default selection. This is a regression/control example, not a new candidate introduced by this PR. | For the first two rows, the observable change is the alternative set delivered @@ -121,7 +121,7 @@ evidence (for example a failure probability of `1.5`) instead produces a The corresponding reproducible checks are `cargo test -p asap-frontend-promql grouped_count_keeps_uncertified_hydra_candidates_for_backend_review`, -`cargo test -p asap-frontend-promql uncalibrated_frequency_readouts_do_not_bypass_accuracy_targets`, +`cargo test -p asap-frontend-promql uncalibrated_frequency_evaluations_do_not_bypass_accuracy_targets`, and `cargo test -p asap-integration-tests ddsketch_ratio_without_domain_proof_is_uncertified`. All three start from PromQL text and exercise frontend lowering and planning. None runs a deployed query. diff --git a/docs/design_docs/architecture/input-output-workflow.md b/docs/design_docs/architecture/input-output-workflow.md index 65ee43141..4120811ca 100644 --- a/docs/design_docs/architecture/input-output-workflow.md +++ b/docs/design_docs/architecture/input-output-workflow.md @@ -66,12 +66,12 @@ flowchart TD D["data_workload: continuous arrival; declared ingestion interval 15 s"] T["Frontend argument: now_ms"] F["PromQL lowering"] - R["One canonical QueryExpr root"] + R["One canonical OperatorNode root"] S["Candidate search"] P["CandidateLogicalASAPDAGs: logical choices for this root"] I["cost_sorted: inspect choices"] G["global_selection + assemble_selected_dag(root)"] - L["One selected Post-ASAP DAG; exact KeepPreAsap if no optimization is selected"] + L["One selected Post-ASAP DAG; the exact pre-ASAP sub-DAG if no optimization is selected"] X["Extra lifecycle inputs: horizon; update rate; capabilities; comparable summary/raw costs"] H["Summary-maintenance-lifecycle-aware selection"] HM["Assemble one selected DAG and decide summary maintenance"] @@ -102,7 +102,7 @@ flowchart LR Q["query_batch: SELECT COUNT(*) FROM metrics; invocations 1; AdHoc"] C["SqlCatalog: resolves metrics and its columns"] F["SQL lowering"] - R["One QueryExpr root"] + R["One OperatorNode root"] P["Candidate search → CandidateLogicalASAPDAGs"] Q --> F C --> F @@ -214,7 +214,7 @@ fields expand as follows: | `TimeSelection` | `lookback` | Optional event-time duration selected before the upper bound. | | `TimeSelection` | `as_of` | Optional fixed upper-bound timestamp; `None` means planning/evaluation time. | -Frontend lowering produces one Pre-ASAP `QueryExpr` root for each normalized +Frontend lowering produces one Pre-ASAP `Rc` root for each normalized query entry. The caller must retain each root's association with its workload entry for later recurrence and lifecycle planning. @@ -334,7 +334,7 @@ below. A future higher-level API could hide `CandidateLogicalASAPDAGs` behind th the current interface lets an integrator own them. DAG assembly connects choices after selection and does not replace this candidate interface. -Here, a **root** is the top-level `Rc` for a workload query. A +Here, a **root** is the top-level `Rc` for a workload query. A **target** is any discovered sub-DAG that may be replaced, including roots. For `count(up) + 1`, the addition is a root and `count(up)` can be an inner target. `TargetSubDAGCandidates` holds the alternatives for one such target. @@ -361,7 +361,7 @@ All paths start by lowering the workload and searching for candidates: ```text PlanningWorkload + frontend dependencies + planning models/evidence - -> frontend lowering: one QueryExpr root per normalized query entry + -> frontend lowering: one OperatorNode root per normalized query entry -> search_workload_with_targets -> CandidateLogicalASAPDAGs ``` @@ -390,7 +390,7 @@ The return type is `Vec>`; each element has thi ```rust struct RankedTargetSubDAGCandidates<'a> { - target: &'a Rc, + target: &'a Rc, consumer_count: usize, candidates: Vec<&'a ReplacementSubDAG>, costs: Vec, // costs[i] describes candidates[i] @@ -430,7 +430,8 @@ the result for one query root. | **Output:** one selected logical [Post-ASAP DAG](../concepts/post-asap-ir.md) per query root | Each output DAG specifies the chosen operators, parameters, and accuracy -guarantees. Its root is represented by `Rc`; the +guarantees. Its root is an `Rc` (the same IR as the input, +with some nodes now ASAP operators) and carries no execution timing yet; the [API reference](../../develop_docs/library-api.md#api-definition-and-example) describes the function signatures and return handling. @@ -456,7 +457,7 @@ there is no need to run the ordinary selection/assembly workflow first: `Result, SummaryMaintenanceLifecycleAssemblyError>`. When a summary does not beat a known raw cost, or a required comparable cost is unavailable, the result - retains the exact `KeepPreAsap` root and no summary deployments. + retains the exact pre-ASAP root (`retain_exact`) and no summary deployments. As in ordinary selection, one selection call serves the workload and assembly is per root. The second helper calls `assemble_selected_dag` internally; callers @@ -489,7 +490,7 @@ facts remain unknown rather than being treated as zero. The per-query output, `SummaryMaintenanceLifecyclePlan`, **contains** the Post-ASAP DAG rather than being a parallel representation. It records: -* the assembled Post-ASAP DAG root (`Rc`); +* the assembled Post-ASAP DAG root (`Rc`, with execution timing written); * lifecycle choices for summary state; * planning horizon and expected reads/updates; * selected window implementation and guarantees; diff --git a/docs/design_docs/architecture/metricsql-frontend.md b/docs/design_docs/architecture/metricsql-frontend.md index ddc463239..ffb45b728 100644 --- a/docs/design_docs/architecture/metricsql-frontend.md +++ b/docs/design_docs/architecture/metricsql-frontend.md @@ -9,16 +9,17 @@ on VictoriaMetrics and models MetricsQL syntax directly, including `WITH`, rollup expressions, step-relative durations, MetricsQL binary operators, aggregate limits, or-delimited matchers, and `keep_metric_names`. -The frontend walks that AST directly and emits the existing canonical -`QueryExpr`. It does not add MetricsQL fields to `QueryExpr`, SDS descriptors, -or the physical summary DAG. +The frontend walks that AST directly into the shared name-based `UnresolvedOp` +tree (`asap-frontend-common`) and calls `resolve_root`, which returns the +canonical `Rc` DAG. It does not add MetricsQL fields to the +operator IR, SDS descriptors, or the physical summary DAG. ```text MetricsQL source | MetricsqlExpr (extension semantics retained) | -canonical QueryExpr +UnresolvedOp tree --resolve_root--> canonical OperatorNode DAG | existing ASAP-aware mapping and physical Summary DAG ``` @@ -33,8 +34,8 @@ existing ASAP-aware mapping and physical Summary DAG | Common rollups: rate/increase/derivatives and statistical `*_over_time` | Existing per-entity canonical intents over the lowered range. | | PromQL arithmetic, comparison, and set binary operators without modifiers | Existing canonical `BinaryOp`. | | `default_rollup(selector[range])` | Lower to `Aggregate(LastOverTime)` over the explicit `TimeRange`. | -| `default_rollup(selector)` | Reject for exact fallback because the implicit lookbehind window depends on the runtime evaluation step, which is not a property of canonical `QueryExpr`. | -| `expr keep_metric_names` | Parsed natively, then rejected for exact fallback because canonical `QueryExpr` does not carry metric-name lineage. | +| `default_rollup(selector)` | Reject for exact fallback because the implicit lookbehind window depends on the runtime evaluation step, which is not a property of the canonical operator IR. | +| `expr keep_metric_names` | Parsed natively, then rejected for exact fallback because the canonical operator IR does not carry metric-name lineage. | | `if`, `ifnot`, `default`, aggregate `limit`, or-delimited matchers, binary match modifiers | Parsed natively and rejected until the canonical executor has the exact semantics. | | `WITH` | Expanded by the native parser; the expanded expression lowers when every resulting node is supported. | diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index 057ba7c17..ffbf62239 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -5,14 +5,13 @@ This document defines the boundary between ASAPPlanner's logical plans, physical lowering, statistics resolution, and analytical resource estimation. It answers which representation is authoritative at each stage and prevents -the cost model from being coupled directly to either logical IR. +the cost model from being coupled directly to the logical IR. The integration pipeline is: ```text -pre-ASAP QueryExpr ─┐ - ├─ physical lowering ─> PhysicalOperator DAG -post-ASAP SummaryExpr┘ │ +logical OperatorNode DAG ─ physical lowering ─> PhysicalOperator DAG +(NonASAPOp + ASAPOp nodes) │ v OperatorStatistics │ @@ -30,8 +29,8 @@ Each representation is authoritative for a different concern: | Representation | Authoritative concern | |---|---| -| `QueryExpr` | Original exact query semantics: sources, predicates, relational and PromQL operations, and output shape. | -| `SummaryExpr` | Logical summary semantics: selected family, grouping strategy, summary composition, and summary readout. | +| `NonASAPOp` nodes | Exact query semantics: sources, predicates, relational and PromQL operations, and output shape. | +| `ASAPOp` nodes | Logical summary semantics: selected family, grouping strategy, summary composition, and summary evaluation. | | `PhysicalOperator` DAG | Selected executable algorithms, their configuration, physical identity, edges, and execution multiplicity. | | `OperatorStatistics` | Workload-dependent evidence required by each selected physical operator's resource formula. | | `ResourceEstimate` | Estimated CPU operations, peak live memory, and physical source/disk reads over one comparison scope. | @@ -39,7 +38,7 @@ Each representation is authoritative for a different concern: `PhysicalOperator` is therefore the source of truth for the operator vocabulary consumed by analytical costing. `OperatorStatistics` corresponds one-to-one with that vocabulary. It must not independently invent operator kinds or copy -all variants from either logical IR. +all variants from the logical IR. The canonical physical-plan types should live at a neutral boundary shared by lowering, costing, explanation, and downstream compilation. Their conceptual @@ -51,7 +50,7 @@ being established. One logical operation may choose between algorithms or expand into a physical sub-DAG. Conversely, one physical operator may implement nodes originating -from either logical IR. +from either operator category (`NonASAPOp` or `ASAPOp`). Examples include: @@ -61,14 +60,14 @@ Examples include: supported join algorithm. - `SummaryAgg` may lower to an exact accumulator build, CMS build, KLL build, or another physical summary algorithm selected by the candidate. -- `SummaryEstimate` must lower to a readout operator compatible with the +- `SummaryEstimate` must lower to an evaluation operator compatible with the concrete summary state it consumes. - shared logical sub-DAGs become shared physical nodes only when they refer to the same physical identity and compatible evidence. -For this reason, aligning `OperatorStatistics` directly with `QueryExpr` would -lose post-ASAP summary implementations, while aligning it directly with -`SummaryExpr` would lose raw query operators and physical algorithm choices. +For this reason, aligning `OperatorStatistics` directly with the logical +operators would lose physical algorithm choices, and with only one category +would lose either summary implementations or raw query operators. ## Lowering obligations @@ -91,14 +90,16 @@ its modeled descendants is invalid because it undercounts the candidate. ### Pre-ASAP lowering -`KeepPreAsap` recursively lowers its contained `QueryExpr`. Typical physical +Every `NonASAPOp` node lowers recursively, whether it is in a raw query or +kept exact inside a post-ASAP plan. Typical physical operators include scans, filters, projections, hash aggregates, joins, ordering, bounded Top-K, limits, and PromQL-specific operators. The selected physical algorithm, rather than the logical spelling, determines the formula. ### Post-ASAP lowering -Every `SummaryExpr` operation also needs explicit physical realization: +Every `ASAPOp` node, and every exact operator composed with one, also needs +explicit physical realization: | Logical summary operation | Required physical realization | |---|---| @@ -107,12 +108,14 @@ Every `SummaryExpr` operation also needs explicit physical realization: | `SummaryMerge` | merge operator over compatible concrete summary states | | `SummarySubtract` | subtract operator supported by the selected state representation | | `SummaryDelete` | physical deletion/update operator supported by the selected representation | -| `SummaryEstimate` | family- and query-specific readout operator | -| `KeepPreAsap` | recursive lowering of the contained `QueryExpr` | -| `BinaryOp` | binary evaluation preserving operand order, execution timing and any typed finite/relative-division guard | -| `ValueOperation` | concrete realization of the value operation with its required execution timing and data state | -| `RelationalJoin` | concrete row-join algorithm preserving join kind and predicate | -| `RelationalJoin` with `JoinKind::Semi` | retain left rows matching explicit right-side keys; candidate pruning carries completeness evidence and ordinary TopK ranks the result | +| `SummaryEstimate` | family- and query-specific evaluation operator | +| `FinalizeExactAccumulator` | exact-state finalization before value consumers | +| `MaintainPopulation` / `EvaluatePopulation` | maintained-population update and its aggregate or TopK-prefix evaluation | +| retained `NonASAPOp` sub-DAG | recursive lowering of the exact operators (see above) | +| `BinaryOp` | binary evaluation preserving operand order, the node's execution timing and any typed finite/relative-division guard | +| `Project` / `Filter` / `Sort` / `Limit` / `Aggregate` over an evaluation | concrete realization at the node's execution timing and data state | +| `Join` | concrete row-join algorithm preserving join kind and predicate | +| `Join` with `JoinKind::Semi` | retain left rows matching explicit right-side keys; candidate pruning carries completeness evidence and ordinary TopK ranks the result | This table is a completeness requirement, not a claim that every realization already exists. Until lowering introduces an explicit physical operator, @@ -120,13 +123,13 @@ statistics contract, validation rule, and resource formula for an operation, a candidate containing it is unavailable. The streaming integration can consume a complete binding through -`SummaryNodeEvidence`. That binding is keyed to exact `SummaryNode` +`SummaryNodeEvidence`. That binding is keyed to exact `OperatorNode` identities and uses structured evidence for aggregate state, join, merge, -subtract, delete, readout, and retained pre-ASAP work. It is a physical +subtract, delete, evaluation, and retained pre-ASAP work. It is a physical evidence boundary, not automatic physical lowering: a deployment must still select each concrete implementation and provide all edges, resource facts, multiplicities, source ownership, and stable physical identities. The planner -fails closed when any reachable `SummaryExpr` node lacks that binding. +fails closed when any reachable ASAP node lacks that binding. The raw/query portion of a streaming comparison remains a `PhysicalDAG` using the canonical `PhysicalOperator` and `OperatorStatistics` pairing. Summary @@ -137,7 +140,7 @@ summary-family semantics. Lifecycle choice affects the physical DAG but does not replace it. Ephemeral, prepared, shared, and continuously maintained alternatives determine when -build, update, readout, merge, subtract, or delete nodes execute. The physical +build, update, evaluation, merge, subtract, or delete nodes execute. The physical operators still determine how each execution consumes CPU, memory, and I/O. ## Statistics contract @@ -427,7 +430,7 @@ recovering average semantics from query text. ### Candidate pruning is a sub-DAG -Candidate-based TopK uses a summary key readout, a general semi-join over +Candidate-based TopK uses a summary key evaluation, a general semi-join over explicit matching key columns, grouped Sort by the authoritative score, and grouped Limit. Sort and Limit carry the same partition keys. The join preserves authoritative left-side values and does not rank or limit @@ -440,10 +443,12 @@ fields. The phase assignment API updates producer edge states and rejects an ingestion computation that depends on query-time work. Deployment capability, storage readiness, schemas and approximation guarantees remain separate checks. -Post-ASAP DAG wire version 4 removes the special membership operator, its edge +Post-ASAP DAG wire version 4 removed the special membership operator, its edge roles and the duplicate operator phase fields without compatibility aliases. +Version 6 (current) exports one node per operator: retained exact operators are +`Relational` nodes, not embedded sub-DAGs. -Post-ASAP DAG wire version 6 adds a per-measure row predicate to the aggregate +Post-ASAP DAG wire version 7 adds a per-measure row predicate to the aggregate operators (#466): `filters` on the exact aggregate value operation, parallel to its measures, and `filter` on `SummaryAgg`, gating which rows update the summary state. The version bump makes an older reader fail loudly instead of @@ -465,8 +470,9 @@ a numeric entity key is not a score. Exact accumulator inputs are explicitly finalized before row operators consume them. None of these operations proves candidate completeness; that evidence belongs to the semi-join's pruning step. -The semantic `SummaryExpr` constructors still propose an initial execution -layout. Uniform phase assignment applies to the exported post-ASAP DAG; +The logical DAG carries no execution layout: `apply_lifecycle_timings` writes +each node's timing from the lifecycle assignment before export. Uniform phase +assignment applies to the exported post-ASAP DAG; it is not a claim that every deployment has implemented every placement. diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 58e3f06b6..5dfe5f760 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -53,7 +53,7 @@ backend still owns how the selected algorithms are physically realized. The same contract applies when ASAPPlanner selects a summary algorithm. Planner can choose KLL rather than DDSketch, while downstream chooses the concrete KLL implementation and runtime configuration that satisfies the selected parameter -and accuracy contract. Empirical KLL error, update work, state size, and readout +and accuracy contract. Empirical KLL error, update work, state size, and evaluation work observed on a particular workload can be fed back as evidence for later Planner comparisons. @@ -124,7 +124,7 @@ The ASAPQuery configuration and MIP formulations can supply physical alternatives and coefficients. Their general principles also inform Planner costing: arrival rate scales ingestion work, overlapping active windows multiply update work and live state, retained windows consume memory, and -merge/subtract/readout work scales with query recurrence. Disagreement between +merge/subtract/evaluation work scales with query recurrence. Disagreement between formulations must become distinct explicit alternatives, not hidden assumptions in one cost formula. diff --git a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md index 72e626a2e..4820b6226 100644 --- a/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md +++ b/docs/design_docs/architecture/updated_interface_with_pluggable_optimization.md @@ -185,7 +185,7 @@ for (index, entry) in workload.query_workload.entries().enumerate() { let accuracy = entry.requirements.accuracy.target(); let expr = lower_sql_dialect(&entry.query.0, &catalog, dialect.clone(), accuracy.clone()) .await?; - roots.push((index, Rc::new(expr), Some(accuracy))); + roots.push((index, expr, Some(accuracy))); entry_indices.push(index); } diff --git a/docs/design_docs/concepts/accuracy-models.md b/docs/design_docs/concepts/accuracy-models.md index 5353ef9ef..baeadbe9e 100644 --- a/docs/design_docs/concepts/accuracy-models.md +++ b/docs/design_docs/concepts/accuracy-models.md @@ -69,7 +69,7 @@ query text or cost estimates. flowchart TD Request[Query semantics and accuracy target] --> Generate[Generate candidates and size parameters] Evidence[Scoped source contracts and evidence] --> Generate - Generate --> Local[Derive local readout guarantees] + Generate --> Local[Derive local evaluation guarantees] Evidence --> Local Local --> Compose[Propagate guarantees through the DAG] Evidence --> Compose @@ -100,7 +100,7 @@ is ready, or that a complete deployment cost is available. ## Local estimator models and parameter sizing -A local model describes a specific readout of a specific estimator with +A local model describes a specific evaluation of a specific estimator with committed parameters and applicable assumptions. A family name or a parameter such as HLL precision is not, by itself, a confidence certificate. @@ -121,8 +121,8 @@ The built-in models currently include: | CMS | L1-normalized frequency bound from width and depth; does not by itself certify TopK membership | | CountSketch | L2-normalized frequency bound and median concentration bound, requiring valid odd depth | | KMV / Theta | Parameter-derived cardinality bounds using the registered variance/Chebyshev model at 99% confidence | -| UnivMon | Exact unit-update total for the supported readout; no universal guarantee for all its statistics | -| Other families/readouts | No default certificate where no accuracy model is registered | +| UnivMon | Exact unit-update total for the supported evaluation; no universal guarantee for all its statistics | +| Other families/evaluations | No default certificate where no accuracy model is registered | This table describes Planner's registered contracts, not independent mathematical verification of every estimator or permission to substitute @@ -214,7 +214,7 @@ an observation into a guarantee. A deployment supplies `EstimatorContract::ClassicHll` for the complete aggregate expression. It asserts the classic estimator, independent uniform bucket -hashing and an enforced maximum distinct population per readout, including +hashing and an enforced maximum distinct population per evaluation, including all merged panes. Planner combines this contract with the query or allocated local target, selects a supported precision, derives the guarantee and uses the normal propagation and selection checks. @@ -290,7 +290,7 @@ accuracy/ ├── allocation.rs # End-to-end budget allocation ├── reconciliation.rs # Accuracy coordination across consumers └── estimators/ - ├── mod.rs # Family/readout dispatch and source-contract integration + ├── mod.rs # Family/evaluation dispatch and source-contract integration ├── kll.rs ├── ddsketch.rs ├── hll.rs # Generic HLL and bounded Classic HLL @@ -308,7 +308,7 @@ share the same contract. Adding an estimator or composition requires: -1. A precisely defined error metric, estimator/readout semantics and assumptions. +1. A precisely defined error metric, estimator/evaluation semantics and assumptions. 2. Sizing behavior and a guarantee derived from the committed parameters, including unsupported parameter domains. 3. Explicit evidence requirements, population scope and provenance. 4. Propagation rules where supported; rejection or retained unknowns elsewhere. diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 08c670af4..f7592f3d6 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -1,86 +1,171 @@ # Post-ASAP IR The goal of the post-ASAP IR is to represent operations using ASAP primitives -such as sketches, exact summaries, samples and wavelets. Post-ASAP IR also -retains exact Pre-ASAP sub-DAGs and supports operations over summary readouts, -since only some query operations can be satisfied using summaries. - -The lists below cover every current variant of -[`SummaryExpr`](../../../crates/types/src/post_asap/expr.rs). A node's presence -in the IR does not imply that every summary family, cost model or downstream -runtime supports it. - -## ASAP-specific nodes operated over a summary structure, not raw data - -- `SummaryAgg`: produce summary state from input data using the selected family, - parameters, update input, reduction and grouping layout. -- `SummaryEstimate`: read the requested statistic from summary state and return - query values. Exact accumulators can expose results without a separate sketch - readout. -- `SummaryMerge`: merge compatible summary states when the family supports merging. +such as sketches, exact summaries, samples and wavelets, while retaining the +exact query operators that no summary replaces, and supporting operations over +summary evaluations. + +ASAPPlanner has one operator IR before and after ASAP optimization +([`crates/types/src/ir/`](../../../crates/types/src/ir/)). A post-ASAP plan is +the same `Rc` DAG a front end produced, in which some nodes now +carry `Operator::ASAP(ASAPOp)` instead of `Operator::NonASAP(NonASAPOp)`. There +is no wrapper around retained exact work: an unreplaced `Filter`, `Join` or +`Aggregate` is the same node it was before, and either category can consume +the other's output. The node structure, the `Schema`, scalar expressions and +the catalog of non-ASAP operators are described once in the +[Pre-ASAP IR reference](../../develop_docs/pre-asap-ir.md); this document covers +what optimization adds: the ASAP operators, the accuracy guarantee, execution +timing, and the exported DAG. + +A node's presence in the IR does not imply that every summary family, cost +model or downstream runtime supports it. + +## ASAP operators + +Every variant of [`ASAPOp`](../../../crates/types/src/ir/asap.rs) operates +over summary state rather than raw data. The summary family, kind/algorithm and +parameters are committed in the node; the state itself is typed by the +`FieldDataType` of the output field that carries it (`ExactAggregate`, +`Sketch`, `Sample`, `Wavelet`, `StatModel`). + +Implemented: + +- `SummaryAgg { child, family, input, reduction, grouping }`: produce summary + state from input rows using the selected family, parameters, update input, + reduction and grouping layout. Output: the grouping columns plus one `state` + field typed `family`; result kind `State`. +- `SummaryEstimate { summary_input, query }`: read the requested statistic + (`SketchStatistic`) from summary state and return query values in a row-shaped + schema. +- `FinalizeExactAccumulator { child }`: read an exact accumulator's state as + its finalized value — the maintenance-to-read boundary before query-time + operators consume it. +- `MaintainPopulation { child, population }`: maintain the full declared + population, including membership changes. +- `EvaluatePopulation { child, evaluation }`: read an aggregate or TopK prefix from a + maintained population. +- `SummaryMerge { children }`: merge summary states with identical schemas + (same family, parameters and grouping) into the state of their union; its + coverage check rejects inputs whose selections overlap. The timing pass does + not place it yet (`ExecutionDataStateError::UnimplementedOperator`). + +Reserved (migrated but unimplemented; schema derivation rejects them with +`UNIMPLEMENTED_ASAP_OP` and the timing pass with `UnimplementedOperator`): + - `SummarySubtract`: subtract one summary state from another when supported by the selected representation. - `SummaryDelete`: delete a key from summary state when the representation supports deletion. - `SummaryJoin`: combine summary states for join estimation; this is distinct from joining ordinary rows. +- `Extension`: a deployment-defined operator. The earlier draft listed `SummaryCreate` and `SummaryInsert`. These are not -separate variants in the current IR. `SummaryAgg` describes the state-producing +separate variants. `SummaryAgg` describes the state-producing computation and its update input. The [summary-maintenance lifecycle](../proposals/asap-aware-mapping/workload-demand-and-summary-lifecycle.md) separately describes when state is created, retained, shared, updated and retired. Physical binding and runtime execution implement the actual build and update -operations. This is not a one-to-one rename of the old nodes, and not every -summary family supports incremental maintenance. - -## Exact work and composition nodes - -- `KeepPreAsap`: retain an exact Pre-ASAP sub-DAG when it is not rewritten. -- `BinaryOp`: combine independently planned operands with the specified binary - semantics and execution timing. -- `ValueOperation`: apply aggregate, exact-function, population, projection, - filter, sort, limit or extension semantics with explicit execution timing. -- `RelationalJoin`: join row-producing children using the specified join kind - and predicate. -- Candidate pruning uses `RelationalJoin` with `JoinKind::Semi` and an explicit +operations. Not every summary family supports incremental maintenance. + +## Exact work and composition + +Exact work is represented by the ordinary operators, unchanged: + +- A sub-DAG the planner does not rewrite keeps its `NonASAPOp` nodes. Plan + assembly marks such a sub-DAG with an exact `ResultGuarantee` + (`asap_aware_mapping::replacement::retain_exact`); a sub-DAG with no ASAP + operator and no guarantee is a logical rewrite candidate that has not been + assessed yet (`is_logical_rewrite`). +- `BinaryOp` combines independently planned operands. Summary planning may set + its typed division guards (`checked_finite_division`, + `checked_relative_division`); the operator's timing comes from the lifecycle + assignment, not from the operator. +- Aggregate, projection, filter, sort and limit over an evaluation are the ordinary + `Aggregate`, `Project`, `Filter`, `Sort` and `Limit` operators reading an ASAP + node. Exact-accumulator state may pass through the projection-like + operators unchanged; a value consumer needs a `FinalizeExactAccumulator` + boundary first. +- Candidate pruning uses `Join` with `JoinKind::Semi` and an explicit equality predicate on key columns. The left input supplies authoritative - values; the right input supplies keys. Grouped Sort followed by grouped Limit ranks - and selects the joined rows. Completeness evidence belongs to pruning, not ranking. + values; the right input supplies keys. Grouped `Sort` followed by grouped + `Limit` (both with the same `partition_by`) ranks and selects the joined + rows. Completeness evidence belongs to pruning, not ranking. -A `SummaryNode` carries its expression, schema and optional result guarantee. +Every `OperatorNode` carries its schema and an optional `ResultGuarantee`. State and query values have different contracts; see [Schema and physical data for ASAP primitives](../proposals/asap-primitive-schema.md). Exact operations over -approximate readouts still require composed accuracy guarantees. See the +approximate evaluations still require composed accuracy guarantees. See the [accuracy implementation companion](../../develop_docs/end-to-end-accuracy-guarantees.md) and [physical-plan integration](../architecture/physical-plan-integration.md) for the corresponding correctness and realization requirements. -## In-memory and exported DAG forms - -The Pre-ASAP DAG and the Post-ASAP DAG are both logical: they describe what is -computed, not which physical operators execute it. The Post-ASAP DAG has two -forms of the same content. Planning builds and shares `SummaryNode` DAGs. -`compile_post_asap_dag` converts a selected DAG into a -[`PostAsapDAG`](../../../crates/types/src/post_asap/post_asap_dag.rs) with -stable node IDs and typed edges; `PostAsapDAGDocument` is its versioned wire -envelope. Physical compilation consumes `PostAsapDAG` and produces a separate -physical DAG. - -## Execution phase +## Execution timing An operator defines what computation happens. The plan decides when it happens: **ingestion time** or **query time**. Operator identity must not imply one of these phases. Backend capability restrictions are implementation gaps, not definitions of the operator. -Every post-ASAP operator payload supports both phase assignments. Phase is -stored on the `PostAsapDAG` node, independently of its operator payload. -`PostAsapDAG::with_execution_phases` assigns a phase to every node and updates -its edges. Ingestion work cannot depend on a future query result. Default -semantic realization still proposes an initial layout; it does not restrict -which phase an operator may use. Deployments must separately check that -they have an implementation and a valid data source for the chosen placement. +The logical DAG carries no timing: `OperatorNode::timing` is `None` on every +front-end node and every candidate, and `map_children` clears it. Summary +materialization chooses a lifecycle per summary state and records it in a +[`LifecycleAssignment`](../../../crates/types/src/ir/timing.rs) (ingestion-time +maintenance or query-time recomputation per `SummaryAgg`; a state absent from +the assignment defaults to ingestion-time maintenance). +`apply_lifecycle_timings(root, &assignment, &mut TimingMemo)` then writes a +timing into every node, top-down: + +- a node of fixed kind takes its kind's timing — `SummaryEstimate` and + `EvaluatePopulation` run at query time, `MaintainPopulation` at ingestion time; +- a `SummaryAgg` takes the assignment's timing, unless something below it can + only exist at query time (an evaluation); +- every other node runs when its consumer runs: everything that feeds a + maintained state runs at ingestion time, everything above an evaluation at + query time. + +The pass then validates every edge (rows or exact-accumulator state into a +`SummaryAgg`, state into an evaluation, an ingestion-time `MaintainPopulation` under +an `EvaluatePopulation`, no ingestion work reading a query-time value) and rejects a +node reached from two consumers that need different timings; +`split_shared_by_phase` copies such a sub-DAG for one side before the +assignment is applied. `validate_default` and `planned_data_state` answer the +same questions for a candidate at planning time without keeping anything. + +## Exported DAG + +The pre-ASAP DAG and the post-ASAP DAG are both logical: they describe what is +computed, not which physical operators execute it. Planning builds and shares +`OperatorNode` trees; +[`asap_types::ir::physical_export::compile_physical_asap_dag`](../../../crates/types/src/ir/physical_export.rs) +converts a selected, timed tree into a `PhysicalASAPDAG` with stable node IDs +and typed edges (`compile_physical_asap_workload` does the same for a batch, +with one root per query), and `PhysicalASAPDAGDocument` is its versioned wire +envelope (`schema_version` = `PHYSICAL_ASAP_DAG_WIRE_VERSION`). +Physical compilation consumes `PhysicalASAPDAG` and produces a separate +physical DAG. + +The DAG has **one node per operator** — relational operators included — with +children as edges and no embedded sub-DAGs: + +- A node's payload is its `Operator`: the same `NonASAPOp` or `ASAPOp` + variant as in the IR, with each child replaced by its node ID. +- Edges carry a role: `Input`, `Left`/`Right` for the two sides of a `Join`, + `SetOp`, `BinaryOp`, `SummarySubtract` or `SummaryJoin`, and `ScalarRef` + when the consumer reads the producer from inside one of its scalar + expressions (`scalar(v)`). Every edge records the intermediate schema, the + producer's data state and grouping/window compatibility. +- Each node records `output_state` (timing plus `Raw` or `SummaryState`), + `output_schema` and `guarantee`. Export reads the timing written by + `apply_lifecycle_timings` and rejects an untimed node + (`ExecutionDataStateError::UntimedNode`); it does not re-run data-state + validation. + +Phase is stored on the `PhysicalASAPDAG` node, independently of its payload. +`PhysicalASAPDAG::with_execution_phases` reassigns a phase to every node and +updates its edges; ingestion work cannot depend on a future query result. +Deployments must separately check that they have an implementation and a +valid data source for the chosen placement. ## Weighted grouped TopK @@ -91,19 +176,19 @@ the update weight is the series rate. Summing updates for one item implements the logical grouped sum without first constructing all exact grouped sums. The DAG is per-series rate → finalized values → partitioned summary construction -→ typed candidate/score readout → output projection → grouped Sort → grouped +→ typed candidate/score evaluation → output projection → grouped Sort → grouped Limit. The output count is two per job. The candidate capacity is a separate parameter, provisionally `max(k, ceil(1 / epsilon))`; this sizing choice is not a membership theorem. Missing evidence retains a logical candidate with symbolic unknown guarantees; default selection does not certify or choose it. -The row readout restores job and service identities and returns estimated sums. +The row evaluation restores job and service identities and returns estimated sums. There is no mandatory exact scoring branch or candidate semi-join in this path. The old raw counter-delta update expression is removed rather than retained as a compatibility option: counter increments are not complete windowed rate results. -The direct readout represents both score error and membership. A source provider +The direct evaluation represents both score error and membership. A source provider supplies an enforced upper bound on distinct partition/item identities for the -complete readout. Planner uses this bound to size confidence and union-bound +complete evaluation. Planner uses this bound to size confidence and union-bound score errors over adaptively selected items. Membership evidence is evaluated for the query's output count, not the candidate capacity. Score and membership failure probabilities are combined, and the score guarantee remains in the @@ -111,8 +196,8 @@ membership guarantee's child provenance. An exact request does not accept this approximate output path merely because its selected identities are certified. Deployment chooses ingestion time or query time for these operators. The -semantic constructor proposes a layout; `with_execution_phases` assigns the -placement. Either deployment must give each evaluation a complete +lifecycle assignment writes the placement; `with_execution_phases` can +reassign it on the exported DAG. Either deployment must give each evaluation a complete rate window and an isolated summary state, or maintain an equivalent replacement strategy. Appending successive rate snapshots to one cumulative state is invalid. An ingestion execution can compute a window before the query and store its state; diff --git a/docs/design_docs/concepts/pre-asap-ir.md b/docs/design_docs/concepts/pre-asap-ir.md index 735f522f3..0b12766b1 100644 --- a/docs/design_docs/concepts/pre-asap-ir.md +++ b/docs/design_docs/concepts/pre-asap-ir.md @@ -12,16 +12,18 @@ Only semantics that affect correctness, summary applicability, or cost become fi ### Time -- TimeRange — a PromQL range-vector lookback such as [5m]. +- TimeRange — PromQL sample selection: an instant selector's lookback, or a range selector such as [5m]. - TimeShift — moves when a selector is evaluated (offset or @). - PromqlSubquery — re-evaluates an instant-vector expression over a range. ### Relational - Scan — identifies a logical data source. +- Values — literal rows; one empty row is the input of a `SELECT` without `FROM`. +- ScalarBridge — a scalar expression at an operator position: a bare scalar query, or the scalar operand of ` op `. - Filter — restricts rows using a predicate. - Project — selects or derives output columns. -- BinaryOp — composes two inputs with arithmetic, comparison, or boolean logic. +- BinaryOp — composes two inputs with arithmetic, comparison, or boolean logic. A PromQL `bool` comparison returns 0/1 instead of filtering. - Sort — orders rows without expressing a heavy-hitter intent. - Limit — caps a row count, optionally after an offset. - Dedup — removes duplicate rows. @@ -31,10 +33,7 @@ Only semantics that affect correctness, summary applicability, or cost become fi ### PromQL-specific -- PromqlScalarBridge — holds a scalar sub-expression at an operator-DAG position. -- EvalTimestamp — provides the evaluation timestamp as a scalar. -- PromqlVectorFromScalar — promotes a scalar to a label-less instant vector. -- PromqlScalarFromVector — collapses a single-series vector to a scalar. +- PromqlVectorFromScalar — promotes a scalar to a label-less instant vector. Its inverse, PromQL `scalar(v)`, is a scalar expression that reads `v`. - PromqlRelabel — rewrites labels on each series. - PromqlInfoEnrich — enriches labels from an info metric. - PromqlSeriesSample — selects whole series without reducing them. diff --git a/docs/design_docs/decisions/concat-unique-keys.md b/docs/design_docs/decisions/concat-unique-keys.md index 9c0148abf..08b811b94 100644 --- a/docs/design_docs/decisions/concat-unique-keys.md +++ b/docs/design_docs/decisions/concat-unique-keys.md @@ -2,6 +2,9 @@ > Status: accepted decision for the implementation described here. +> Historical: `crates/types/src/pre_asap/query_expr.rs` was removed in #543; +> `Concat` is now `NonASAPOp::Concat` (`crates/types/src/ir/non_asap.rs`). + > **Update**: the investigation below found no current call site paying for a > redundant `Dedup` that this override would remove, and the original version > of this document recommended deferring Option 1 on that basis. The repo @@ -14,7 +17,7 @@ ## Context -[`QueryExpr::Concat`](../../../crates/types/src/pre_asap/query_expr.rs) (the +[`QueryExpr::Concat`](../../../crates/types/src/ir/non_asap.rs) (the n-ary exact `UNION ALL` node, renamed from `Merge` in #226) always drops `unique_keys` on its output — `merge_drops_the_branches_unique_keys` and `merge_and_setop_agree_on_unique_keys` pin this down. Issue #228 asks whether diff --git a/docs/design_docs/decisions/cse-cost-model.md b/docs/design_docs/decisions/cse-cost-model.md index 7ada47bc5..10bb5739a 100644 --- a/docs/design_docs/decisions/cse-cost-model.md +++ b/docs/design_docs/decisions/cse-cost-model.md @@ -10,7 +10,7 @@ elimination) to identify eligible, structurally identical computations. A shared logical node records an opportunity for reuse; selecting maintained state or independent execution is a separate planning decision. -[`asap_types::pre_asap::cse::share_common_sub_dags`](../../../crates/types/src/pre_asap/cse.rs) +[`asap_types::pre_asap::cse::share_common_sub_dags`](../../../crates/types/src/ir/cse.rs) (issue #223 stages 1-2, PR #235) already *detects* every structurally-identical, legally-shareable (`Schema::unique_keys`-gated) sub-DAG and shares it **unconditionally** — there is no cost gate on top of legality. This document diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 00caf5d51..7bb8a93f6 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -28,11 +28,10 @@ implementation library. Deployment systems such as ASAPQuery and asap-fusion own deployment compilation and operation. The lifecycle is a planning contract associated with the logical DAG, not a separate computation IR. -The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG (`QueryExpr`), the +The Logical Post-ASAP DAG is preceded by the Pre-ASAP DAG, the language-independent query semantics before summary selection. Both are -logical. Planning builds Post-ASAP `SummaryNode` DAGs; `compile_post_asap_dag` -exports the selected DAG as a `PostAsapDAG`, which is the Physical Plan -Compiler's input. Its per-node execution phase (ingestion or query time) is +logical `OperatorNode` DAGs. `compile_physical_asap_dag` exports the selected +DAG as a `PhysicalASAPDAG`, which is the Physical Plan Compiler's input. Its per-node execution phase (ingestion or query time) is decided by the selected summary maintenance lifecycle, as the layer contract below states. @@ -139,7 +138,7 @@ readiness; those require runtime checks. Physical location, encoding, scheduling and retention are separate execution/deployment contracts. Persisted semantic identity, its wire format and any tenant or dataset binding -belong to the deployment. Planner provides the typed `PostAsapDAG` that a +belong to the deployment. Planner provides the typed `PhysicalASAPDAG` that a deployment canonicalizes; it does not define a stored-definition format. ### Running example @@ -307,7 +306,7 @@ The **Physical Plan Compiler** consumes both computation semantics and maintenan requirements: ```text -Logical Post-ASAP DAG (PostAsapDAG) +Timed Post-ASAP DAG (PhysicalASAPDAG) + Summary Maintenance Lifecycle + physical capabilities ↓ diff --git a/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md b/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md index f71dde2ea..8bb452783 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md +++ b/docs/design_docs/proposals/asap-aware-mapping/ddsketch-quantile-ratios.md @@ -14,7 +14,7 @@ The final guarantee records both input ranges and their contract identifiers. Th ## Candidate generation without evidence -The default `SketchAlgorithmStrategy` permits a direct DDSketch quantile-ratio +The default `ASAPStrategies` permits a direct DDSketch quantile-ratio candidate when domain evidence is absent, but leaves the root guarantee unset. This is useful for the v1 integration path; it does not turn missing evidence into evidence. Other approximate divisions still require their own composition diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 7a7ffb84d..5f2856b49 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1300,7 +1300,7 @@ pub enum PopulationStatistic { Sum, Count, Average, } /// A population kept in full (§5.4). -pub struct MaintainedPopulation { +pub struct MaintainedPopulation { /// Which rows or series the population contains. pub input: PopulationInput, /// The largest `k` a `TopK` read may ask for. @@ -1308,7 +1308,7 @@ pub struct MaintainedPopulation { /// Whether `Quantile` reads are supported. pub quantiles: bool, } -pub enum PopulationInput { +pub enum PopulationInput { /// The current value of each PromQL series (see `CurrentSeriesInput`). CurrentSeries(CurrentSeriesInput), /// Rows of a table. `input`: the table `Scan` node. `value_column`: the diff --git a/docs/design_docs/proposals/asapquery-rule-coverage.md b/docs/design_docs/proposals/asapquery-rule-coverage.md index e496c9fe9..634878696 100644 --- a/docs/design_docs/proposals/asapquery-rule-coverage.md +++ b/docs/design_docs/proposals/asapquery-rule-coverage.md @@ -21,7 +21,7 @@ cost, and selection rules under `optimizer/`. The reviewed source is | Temporal aggregate functions | Lowering covered; realization varies | `Aggregate(PerEntity)` over `TimeRange` represents the full family. Sum, count, min, max, quantile, rate, and increase have summary realizations; `avg_over_time` is currently exact `PassThrough`, matching ASAPQuery's exact-only multi-stat fallback rather than claiming a maintained summary. | | Spatial aggregate functions | Lowering covered; realization varies | `Aggregate(Reduce(GroupKeys))` is shared by SQL and PromQL. Supported single accumulators and ordinary `by(...)` avg rewrites generate candidates; shapes such as `avg without(...)` retain the same exact raw fallback that ASAPQuery uses for multi-stat AQEs. | | Collapsible temporal + spatial aggregates | Semantic-equivalent rewriting | The existing rewrite strategy uses accumulator algebra: sum∘sum, sum∘count, min∘min, and max∘max. It rejects all other pairs and requires identical output schemas. | -| Sketch alternatives and exact fallback | Covered more generally | `SketchAlgorithmStrategy` enumerates legal summary realizations. The enclosing memo group always retains the original raw expression as the exact fallback; the strategy does not falsely label an approximate sketch as exact. | +| Sketch alternatives and exact fallback | Covered more generally | `ASAPStrategies` enumerates legal summary realizations. The enclosing memo group always retains the original raw expression as the exact fallback; the strategy does not falsely label an approximate sketch as exact. | | Subpopulation label placement | Covered more generally | `HydraGroupingStrategy` and `GroupingStrategy` express per-subpopulation and shared multi-subpopulation realizations. | | Shared computation | Covered more generally | workload-wide CSE and `SharedSubDAGStrategy` operate on physical DAG identity rather than AQE names. | | Average decomposition | Semantic-equivalent rewriting | The same rewrite strategy exposes independently optimizable sum/count accumulators when null semantics and schema permit it. | @@ -38,7 +38,7 @@ does not create a new strategy category. | Decision | Existing owner | |---|---| -| Which summary algorithm can implement one aggregate intent | `SketchAlgorithmStrategy` | +| Which summary algorithm can implement one aggregate intent | `ASAPStrategies` | | How grouping/subpopulation state is laid out | `HydraGroupingStrategy` | | Whether an equivalent logical expression exposes better accumulators | `SemanticEquivalentRewriteStrategy` (the broadened existing avg rewrite; `AvgToSumOverCountStrategy` remains a compatibility name) | | Whether identical physical work is shared | `SharedSubDAGStrategy` | @@ -52,7 +52,7 @@ does not create a new strategy category. Accordingly, ASAPQuery's four collapsible temporal/spatial patterns extend the existing semantic-rewrite owner. Temporal and spatial function recognition is already front-end lowering into `AggIntent`; sketch compatibility remains in -`SketchAlgorithmStrategy`; labels remain in `HydraGroupingStrategy`; and +`ASAPStrategies`; labels remain in `HydraGroupingStrategy`; and maintenance lifecycle legality remains in the lifecycle planner. Window framework selection is separate physical-planning work. None of these become a parallel syntax-oriented `PatternStrategy`. diff --git a/docs/design_docs/proposals/planner-layering.md b/docs/design_docs/proposals/planner-layering.md index 9f2395996..93f353f56 100644 --- a/docs/design_docs/proposals/planner-layering.md +++ b/docs/design_docs/proposals/planner-layering.md @@ -336,7 +336,7 @@ A summary-based candidate uses three kinds of summary nodes: summaries into a coarser one. * A **summary estimation node** computes an answer from a summary, for example the p99 estimate from a KLL, or the entropy estimate from a UnivMon. -* **summary subtract node** and **summary delete node** design is TODO. +* **summary subtract node** and **summary delete node** design is TODO. One summary build node can feed several estimation nodes, which is what Pass 2 exploits. diff --git a/docs/develop_docs/asap-aware-mapping-architecture.md b/docs/develop_docs/asap-aware-mapping-architecture.md index aee8fe53a..154c131ff 100644 --- a/docs/develop_docs/asap-aware-mapping-architecture.md +++ b/docs/develop_docs/asap-aware-mapping-architecture.md @@ -55,18 +55,20 @@ The diagram below follows a workload of one or more query roots through target d Terminology used in the diagram: - A **workload** is the set of named queries planned together. A **query root** - is the top-level `QueryExpr` (the logical query-expression type) for one of - those queries. **Pre-ASAP** means this logical input form, before the planner - realizes an operation as a concrete ASAP realization; **post-ASAP** means - the resulting realization form. + is the top-level `Rc` (the unified operator IR) for one of + those queries. **Pre-ASAP** means a DAG that contains only ordinary + `NonASAPOp` operators, before the planner realizes an operation with ASAP + primitives; **post-ASAP** means the same IR after some nodes became `ASAPOp` + summary operators. - A **DAG** (directed acyclic graph) represents query operators whose sub-DAGs may be shared. See [sub-DAG sharing and ASAP-aware CSE](../design_docs/proposals/planner-layering.md#pass-2-asap-aware-common-subexpression-elimination) for the sharing rules. Rust's `Rc` (reference-counted pointer) records shared node identity. - A **target** is one replaceable site. A **candidate** is one valid alternative - for it. `Replacement::Summary` is a constructed post-ASAP summary—maintained state - such as an exact accumulator or an approximate sketch—while - `Replacement::Rewrite` is another pre-ASAP logical expression. + for it. `Replacement::SubDAG` is a replacement sub-DAG: either a constructed + post-ASAP summary (it contains an `ASAPOp`, e.g. an exact accumulator or an + approximate sketch) or a logical rewrite with no ASAP operator + (`is_logical_rewrite` tells them apart). `Replacement::ExactComposition` refers to a child target whose realization must remain undecided until compatible selection. A **sketch** is a compact data structure that trades exactness for bounded error. A @@ -86,15 +88,15 @@ flowchart TB classDef report fill:#f2eafe,stroke:#7950b3,color:#34204f subgraph DISCOVERY[1. Discover every replaceable site] - WL["Input workload
one or more named pre-ASAP QueryExpr roots"]:::input + WL["Input workload
one or more named pre-ASAP OperatorNode roots"]:::input SEARCH["search_workload_with
run CSE once, then visit every node in every root DAG"]:::generate - TARGET["TargetSubDAG
one candidate site plus the number of workload locations
that reference the same Rc<QueryExpr>"]:::generate + TARGET["TargetSubDAG
one candidate site plus the number of workload locations
that reference the same Rc<OperatorNode>"]:::generate WL -->|"roots"| SEARCH -->|"one target per distinct node"| TARGET end subgraph GENERATION[2. Generate all legal alternatives at each site] STRATEGY["ReplacementStrategy
when a target matches, enumerate every legal replacement;
implementations generate but do not choose"]:::generate - CAND["ReplacementSubDAG candidates
each contains a Summary, Rewrite or ExactComposition
plus typed provenance and rationale;
no alternative is removed solely on cost"]:::store + CAND["ReplacementSubDAG candidates
each contains a Subtree (summary or logical rewrite) or ExactComposition
plus typed provenance and rationale;
no alternative is removed solely on cost"]:::store TARGET -->|"try every registered strategy"| STRATEGY --> CAND CM(["CostModel
orders candidates and supplies
deployment-specific parameters"]):::choose CM -. "rank and parameterize; accuracy checks remain required" .-> STRATEGY @@ -157,7 +159,7 @@ flowchart LR classDef workload fill:#e7f7ef,stroke:#31835e,color:#173f2d classDef common fill:#fff6dd,stroke:#b78922,color:#513d0c - ROOTS["Input
one or more named QueryExpr roots"]:::workload + ROOTS["Input
one or more named OperatorNode roots"]:::workload ROOTS --> CSE["Canonicalize sharing
merge structurally identical, legally shareable sub-DAGs"]:::workload CSE --> WALK["Discover sites
walk the complete DAG, including nodes below unshared parents"]:::workload WALK --> T["Build TargetSubDAG
retain the sub-DAG's Rc identity and measured consumer_count"]:::workload @@ -199,8 +201,8 @@ cost. The default context-free registry contains five `ReplacementStrategy` implementations: -- `SketchAlgorithmStrategy` matches supported aggregate and binary shapes. Its - `replacements(target)` method constructs every legal post-ASAP `SummaryNode`, +- `ASAPStrategies` matches supported aggregate and binary shapes. Its + `replacements(target)` method constructs every legal post-ASAP summary sub-DAG, including applicable sketch, exact-accumulator, and pass-through realizations. Candidates are sized and ordered for the target's accuracy requirement; candidates without a sufficient guarantee are rejected before diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 0e16d4116..02630e91f 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -10,25 +10,25 @@ first; use the [extension guide](extend-asap-aware-mapping.md) when changing one ### `TargetSubDAG` -A pre-ASAP `QueryExpr` node that a strategy may replace. +A pre-ASAP `OperatorNode` that a strategy may replace. ```rust pub struct TargetSubDAG<'a> { - pub root: &'a Rc, + pub root: &'a Rc, pub consumer_count: usize, } ``` -`root` is the actual `Rc` from the workload. +`root` is the actual `Rc` from the workload. -`consumer_count` counts structural references, not runtime executions. It is the number of places in the workload DAG that point to this exact `Rc` node. +`consumer_count` counts structural references, not runtime executions. It is the number of places in the workload DAG that point to this exact `Rc` node. For example, consider two top-level queries: - `sum by (service) (rate(m[5m]))` - `avg by (service) (rate(m[5m]))` -After `share_common_sub_dags` merges their identical `rate(m[5m])` sub-DAGs, both query DAGs point to the same `Rc`. That node's `consumer_count` is `2`, regardless of how often either query executes. +After `share_common_sub_dags` merges their identical `rate(m[5m])` sub-DAGs, both query trees point to the same `Rc`. That node's `consumer_count` is `2`, regardless of how often either query executes. Use: @@ -54,19 +54,24 @@ when the caller already knows the real number of consumers. The actual object that substitutes the target. -There are currently three forms: +There are currently two forms: ```rust pub enum Replacement { - Summary(Rc), - Rewrite(Rc), + SubDAG(Rc), ExactComposition(ExactComposition), } ``` -Use `Replacement::Summary` when the alternative is a constructed post-ASAP summary plan. +Use `Replacement::SubDAG` for a replacement sub-DAG. It is one of: -Use `Replacement::Rewrite` when the alternative is still a logical pre-ASAP `QueryExpr`. +- a constructed post-ASAP summary plan: the sub-DAG contains an `ASAPOp` + (`SummaryAgg`, `SummaryEstimate`, ...); +- a logical rewrite: only `NonASAPOp` nodes and no guarantee yet. + +`is_logical_rewrite(&node)` tells the two apart. A kept pre-ASAP sub-DAG +(`retain_exact`) has no ASAP operator but carries an exact guarantee, so it +counts as a bound decision, not a rewrite. Use `Replacement::ExactComposition` when an exact operation refers to a child target whose realization must remain undecided. Selection coordinates the @@ -77,10 +82,10 @@ Examples: ```text Quantile(...) - -> KLL SummaryNode + -> SummaryEstimate(SummaryAgg(KLL)) ``` -is a `Summary`; KLL (Karnin–Lang–Liberty) is a quantile-sketch algorithm. +is a summary `Subtree`; KLL (Karnin–Lang–Liberty) is a quantile-sketch algorithm. ```text compute independently @@ -88,7 +93,7 @@ compute independently reuse an already shared logical sub-DAG ``` -is represented as a `Rewrite`. +is represented as two logical-rewrite `Subtree`s. --- @@ -157,7 +162,7 @@ aggregation must compute without committing to a physical summary algorithm. A realization may be an approximate sketch, an exact mergeable accumulator, or a pass-through that keeps the original operation instead of building a summary. `realizations_for_intent` enumerates these concrete -realizations; `SketchAlgorithmStrategy::replacements()` constructs each one as +realizations; `ASAPStrategies::replacements()` constructs each one as a `ReplacementSubDAG`. It returns all candidates in preferred order without selecting a winner. At workload scale, `search_workload`/`search_workload_with` preserve all supported legal alternatives @@ -170,8 +175,8 @@ This guide uses the Cascades/Volcano terminology: realization. For example, a quantile `AggIntent` may have KLL and DDSketch `Realization` values. - A **transformation rule** maps a logical operation to another logical - operation. In this crate, that kind of candidate is represented by - `Replacement::Rewrite`. + operation. In this crate, that kind of candidate is a logical-rewrite + `Replacement::SubDAG`. - A **replacement candidate** packages either kind of result as a `ReplacementSubDAG` for search. `CandidateLogicalASAPDAGs` stores and ranks these candidates. - **Physical commitment and placement** happen downstream. An `Realization` @@ -183,7 +188,7 @@ The concrete flow is: ```text AggIntent -> realizations_for_intent(): enumerate Realization values - -> SketchAlgorithmStrategy: construct ReplacementSubDAG candidates + -> ASAPStrategies: construct ReplacementSubDAG candidates -> CandidateLogicalASAPDAGs: store and rank candidates -> downstream deployment: select and place a final choice ``` @@ -206,7 +211,7 @@ bounds, but does not execute workloads or own deployment measurements. Most hook | `rank_candidates` | Order valid sketch algorithms | No | | `size_params` | Convert an accuracy target into sketch parameters | Yes | | `realize_extension` | Map a custom intent to a realization | Yes | -| `readout_extension` | Query a custom extension summary | Panics until paired with a custom realization | +| `evaluation_extension` | Query a custom extension summary | Panics until paired with a custom realization | | `cse_recompute_cost` | Estimate independent recomputation | Yes | | `cse_shared_maintenance_cost` | Estimate shared maintenance | Yes | | `cse_share_decision` | Choose sharing or recomputation | Yes | @@ -247,10 +252,10 @@ bounds, but does not execute workloads or own deployment measurements. Most hook fn realize_extension(&self, ext_kind: &str, payload: &serde_json::Value) -> Realization; ``` -- **`readout_extension`** — define how queries read an extension summary that `realize_extension` mapped to a `Sketch`. The two hooks are a pair: realization defines what is maintained; readout defines how it is queried. Override both for the same `ext_kind`. The default readout panics to prevent a silent wrong answer. +- **`evaluation_extension`** — define how queries read an extension summary that `realize_extension` mapped to a `Sketch`. The two hooks are a pair: realization defines what is maintained; evaluation defines how it is queried. Override both for the same `ext_kind`. The default evaluation panics to prevent a silent wrong answer. ```rust - fn readout_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchStatistic; + fn evaluation_extension(&self, ext_kind: &str, payload: &serde_json::Value, col: &ColumnRef) -> SketchStatistic; ``` - **`cse_recompute_cost`** — estimate the one-time cost of recomputing a CSE candidate's sub-DAG independently at a single consumer. Default: `default_cse_recompute_cost`, a structural-size proxy. @@ -297,14 +302,14 @@ A custom cost model does not necessarily need to override every hook. The curren // One TargetSubDAGCandidates per distinct TargetSubDAG in the whole workload — // never a flat list of fully assembled plans. pub struct TargetSubDAGCandidates { - pub target: Rc, + pub target: Rc, pub consumer_count: usize, pub candidates: Vec, // accepted alternatives, unranked pub rejected: Vec, // failed accuracy checks } pub struct RankedTargetSubDAGCandidates<'a> { - pub target: &'a Rc, + pub target: &'a Rc, pub consumer_count: usize, pub candidates: Vec<&'a ReplacementSubDAG>, // same candidates, ranked pub costs: Vec, // costs[i] <-> candidates[i] @@ -313,7 +318,7 @@ pub struct RankedTargetSubDAGCandidates<'a> { `search_workload(roots)` runs the shared-sub-DAG pass once, discovers every target across every root's whole DAG (not just root-level sharing — a `SharedSubDAGStrategy` candidate three levels under an unshared `Filter` is exactly as real a site as a shared whole root), and asks every registered strategy to a fixpoint. Two logically different candidates at two different targets are never copied into two separate plans — they're two entries in two different `TargetSubDAGCandidates`s, sharing every other node in the workload by construction. -`CandidateLogicalASAPDAGs::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — a same-shape `Rewrite` pair (a `SharedSubDAGStrategy` share/recompute choice) goes through `CostModel::cse_share_decision`; a same-shape run of `Summary` candidates realizing sketches (a `SketchAlgorithmStrategy` choice) goes through `CostModel::rank_candidates`; and a mixed candidate set is ordered by each candidate's `CostModel::estimate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks +`CandidateLogicalASAPDAGs::cost_sorted(cost_model)` is the one ranking step: for each candidate set, it dispatches by candidate shape — the `SharedSubDAGStrategy` share/recompute pair (recognized by `ReplacementProvenance::CseShare`/`CseRecompute`) goes through `CostModel::cse_share_decision`; a set with a Hydra shared-grid alternative goes through `CostModel::grouping_state_cost`; a set whose candidates all realize sketches (a `ASAPStrategies` choice) goes through `CostModel::rank_candidates`; and any other mixed set is ordered by `CostModel::candidate_cost`. Every candidate gets a numeric cost aligned index-for-index in `costs`. Count in, count out—nothing is dropped to produce a ranking. Legality checks may already have removed proposals before this boundary. In particular, `search_workload_with_targets` checks explicit per-root targets, while retaining direct DDSketch ratios with missing domain evidence and no root guarantee for @@ -330,7 +335,7 @@ category (`SketchCategory`), algorithm (`SketchAlgorithm`), and the validated committed choice (`SketchKind`). The levels and their validation are specified in [Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#3-proposed-schema-design). -Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `FieldDataType::Sketch(SketchKind, GroupingStrategy)`. +Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `ASAPStrategies::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `FieldDataType::Sketch(SketchKind, GroupingStrategy)`. --- @@ -366,15 +371,15 @@ The crate provides no default `Matcher` implementation because the answer depend Concretely, `explanation.rs` reports three candidate kinds from each `TargetSubDAGCandidates`: -- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `FieldDataType::Sketch(..)`, not just an exact/pass-through candidate. -- `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the set's `target`). +- `ExplanationKind::SketchApproximation` — the set contains a summary `Replacement::SubDAG` that realizes `FieldDataType::Sketch(..)`, not just an exact/pass-through candidate. +- `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::SubDAG` whose `Rc` is the set's `target`). - `ExplanationKind::ExactComposition` — the candidate set contains an exact operation composed with a child target whose realization remains a coordinated choice. Each `ReplacementExplanation::reason` is copied verbatim from the matching candidate's own `ReplacementSubDAG::rationale`. Nothing in `explanation.rs` re-explains why a candidate is valid; that explanation already exists exactly once, on the candidate itself. -`ReplacementExplanation` carries both `node_hash` and `target`. A downstream consumer first compares `node_hash` with an exported `DAGNode::hash` to narrow the search, then compares the exact target expression with the node's in-process source expression. This preserves the hash's role as a fast filter while making the final association collision-safe; `location` remains human-readable presentation text rather than a machine identifier. +`ReplacementExplanation` carries both `node_hash` and `target`. A downstream consumer first compares `node_hash` with an exported `DAGNode::hash` to narrow the search, then compares the exact `target` node with the exported node's in-process `DAGNode::source_node`. This preserves the hash's role as a fast filter while making the final association collision-safe; `location` remains human-readable presentation text rather than a machine identifier. ### Why there is no `ExplanationRule` trait @@ -382,6 +387,6 @@ Explanations are derived from candidates already present in `CandidateLogicalASA ### How it derives `location` text -`CandidateLogicalASAPDAGs`/`TargetSubDAGCandidates` track `Rc` pointer identity, not human-readable breadcrumbs. `ReplacementExplanation::location` provides prose such as `root "dash_a" > lhs` so reporting consumers can identify the relevant part of the query without interpreting pointer identity. Location derivation does not make replacement or costing decisions. +`CandidateLogicalASAPDAGs`/`TargetSubDAGCandidates` track `Rc` pointer identity, not human-readable breadcrumbs. `ReplacementExplanation::location` provides prose such as `root "dash_a" > lhs` so reporting consumers can identify the relevant part of the query without interpreting pointer identity. Location derivation does not make replacement or costing decisions. --- diff --git a/docs/develop_docs/extend-asap-aware-mapping.md b/docs/develop_docs/extend-asap-aware-mapping.md index 5a5e069a8..b9a830140 100644 --- a/docs/develop_docs/extend-asap-aware-mapping.md +++ b/docs/develop_docs/extend-asap-aware-mapping.md @@ -44,7 +44,7 @@ There are four decisions to make. `matches` should contain the minimum structural and semantic checks needed to determine whether the strategy applies. -For example, the aggregate path in `SketchAlgorithmStrategy` requires a +For example, the aggregate path in `ASAPStrategies` requires a supported shape: - the node is an `Aggregate`, @@ -112,23 +112,20 @@ and let costing decide later. --- -### Choose `Summary` vs. `Rewrite` +### Summary sub-DAG vs. logical rewrite -Return: +Both are returned as: ```rust -Replacement::Summary(...) +Replacement::SubDAG(node) ``` -when the candidate is a fully constructed post-ASAP summary. +- A fully constructed post-ASAP summary: `node` contains an `ASAPOp`. +- A logical pre-ASAP rewrite: `node` has only `NonASAPOp` nodes and no + guarantee. `is_logical_rewrite(&node)` checks this. -Return: - -```rust -Replacement::Rewrite(...) -``` - -when the candidate is a logical pre-ASAP rewrite. +Set `provenance` to say which one it is (`ReplacementProvenance::SummaryRealization`, +`LogicalRewrite`, ...); selection reads provenance, not the sub-DAG's shape. Use `Replacement::ExactComposition` when a candidate depends on a child target whose implementation must be selected compatibly later. Do not bind it to the @@ -204,7 +201,7 @@ how to realize it, wrap that logic. Do not create a second implementation of the same semantics inside the strategy. -The existing `SketchAlgorithmStrategy` is the model to follow: it reuses +The existing `ASAPStrategies` is the model to follow: it reuses `replacement.rs`'s existing candidate list and summary-construction path. --- @@ -229,23 +226,23 @@ If your transformation requires context not currently represented in `TargetSubD --- -### Example: current `SketchAlgorithmStrategy` +### Example: current `ASAPStrategies` -`SketchAlgorithmStrategy` is the reference implementation for a strategy that +`ASAPStrategies` is the reference implementation for a strategy that produces constructed post-ASAP summaries. Construction: ```rust let strategy = - SketchAlgorithmStrategy::default_cost_model(); + ASAPStrategies::default_cost_model(); ``` or with a custom cost model: ```rust let model = MyCostModel; // illustrative -let strategy = SketchAlgorithmStrategy::new(&model); +let strategy = ASAPStrategies::new(&model); ``` The strategy matches supported aggregate nodes. @@ -254,10 +251,10 @@ At a high level: ```mermaid flowchart LR - A["Input TargetSubDAG
root is a supported Aggregate"] --> B["SketchAlgorithmStrategy::matches
check whether the target shape can produce summaries"] - B -->|"true"| C["SketchAlgorithmStrategy::replacements
use CostModel preferences and sizing while preserving
every semantically valid realization"] + A["Input TargetSubDAG
root is a supported Aggregate"] --> B["ASAPStrategies::matches
check whether the target shape can produce summaries"] + B -->|"true"| C["ASAPStrategies::replacements
use CostModel preferences and sizing while preserving
every semantically valid realization"] B -->|"false"| NONE["Empty candidate list"] - C --> F["Output Vec<ReplacementSubDAG>
each entry contains a constructed SummaryNode and rationale;
all candidates retained in preferred order"] + C --> F["Output Vec<ReplacementSubDAG>
each entry contains a constructed summary sub-DAG and rationale;
all candidates retained in preferred order"] ``` For an approximate quantile, both KLL and DDSketch remain candidates when @@ -271,16 +268,17 @@ even if the cost model prefers one. When only one realization is legal, such as Call the public strategy interface and inspect every returned candidate: ```rust -let strategy = SketchAlgorithmStrategy::new(&cost_model); +let strategy = ASAPStrategies::new(&cost_model); let candidates = strategy.replacements(&target); for candidate in candidates { match candidate.replacement { - Replacement::Summary(summary) => { - // Inspect or execute this constructed SummaryNode. + Replacement::SubDAG(node) => { + // A constructed summary sub-DAG (`node.is_asap()`), or a kept + // pre-ASAP sub-DAG with an exact guarantee for pass-through. } - Replacement::Rewrite(_) => unreachable!( - "SketchAlgorithmStrategy produces summary candidates" + Replacement::ExactComposition(_) => unreachable!( + "ASAPStrategies produces sub-DAG candidates" ), } } @@ -310,16 +308,16 @@ and returns two alternatives: 2. Build independently for each consumer. ``` -The shared candidate reuses the same `Rc`: +The shared candidate reuses the same `Rc`: ```rust -Replacement::Rewrite(Rc::clone(target.root)) +Replacement::SubDAG(Rc::clone(target.root)) ``` The independent candidate creates a structurally equal but separately allocated node: ```rust -Replacement::Rewrite( +Replacement::SubDAG( Rc::new((**target.root).clone()) ) ``` @@ -353,7 +351,7 @@ The basic calling pattern is: ```rust let target = TargetSubDAG::new(&root); let strategy = - SketchAlgorithmStrategy::default_cost_model(); + ASAPStrategies::default_cost_model(); if strategy.matches(&target) { let candidates = @@ -546,13 +544,13 @@ Then inject it into code that accepts a `&dyn CostModel`: let model = PreferDDSketch; let strategy = - SketchAlgorithmStrategy::new(&model); + ASAPStrategies::new(&model); let replacements = strategy.replacements(&target); ``` -Important: changing `rank_candidates` changes the preferred ordering, but `SketchAlgorithmStrategy` still enumerates every valid sketch candidate. +Important: changing `rank_candidates` changes the preferred ordering, but `ASAPStrategies` still enumerates every valid sketch candidate. A custom cost model should not change which alternatives are semantically legal. @@ -641,12 +639,12 @@ Use it for implementation families that are intentionally outside the built-in e --- -#### `readout_extension` +#### `evaluation_extension` -Use when an extension-defined summary also needs custom query/readout behavior. +Use when an extension-defined summary also needs custom query/evaluation behavior. ```rust -fn readout_extension( +fn evaluation_extension( &self, ext_kind: &str, payload: &serde_json::Value, @@ -654,7 +652,7 @@ fn readout_extension( ) -> SketchStatistic; ``` -This complements `realize_extension`: realization defines what gets maintained; readout defines how it is queried (see the [CostModel reference](asap-aware-mapping-contracts.md#costmodel)). +This complements `realize_extension`: realization defines what gets maintained; evaluation defines how it is queried (see the [CostModel reference](asap-aware-mapping-contracts.md#costmodel)). --- @@ -741,7 +739,7 @@ For example: ```rust let strategy = - SketchAlgorithmStrategy::new(&model); + ASAPStrategies::new(&model); let replacements = strategy.replacements(&target); @@ -770,7 +768,7 @@ Declare built-in sketch applicability through the public candidate registry: summary_candidates(intent) ``` -`SketchAlgorithmStrategy` consumes this registry through its public `replacements` method. +`ASAPStrategies` consumes this registry through its public `replacements` method. Therefore, when adding a new built-in sketch algorithm, the intended flow is: @@ -778,9 +776,9 @@ Therefore, when adding a new built-in sketch algorithm, the intended flow is: flowchart LR MAP["1. Declare legality
add the algorithm to summary_candidates
for each AggIntent it can answer"] MAP --> MODEL["2. Define costing
rank it, derive its SketchParams,
and provide a comparable numeric cost"] - MODEL --> BUILD["3. Define realization behavior
ensure the public strategy output contains a valid SummaryNode
with the correct maintained state and readout"] + MODEL --> BUILD["3. Define realization behavior
ensure the public strategy output contains a valid summary sub-DAG
with the correct maintained state and evaluation"] BUILD --> ACC["4. Certify accuracy
derive from committed parameters;
propagate and check the final target"] - ACC --> ENUM["5. Verify integration
SketchAlgorithmStrategy includes it automatically;
tests confirm enumeration, ordering, sizing, and cost"] + ACC --> ENUM["5. Verify integration
ASAPStrategies includes it automatically;
tests confirm enumeration, ordering, sizing, and cost"] ``` This keeps one source of truth for sketch applicability. Applicability alone @@ -793,9 +791,9 @@ ranking; preserve exact fallback and structured rejection information. See the [accuracy implementation companion](end-to-end-accuracy-guarantees.md) for formulas and evidence requirements. For a new algorithm, also update its -parameter, readout, schema and serialization definitions in `asap-types`. +parameter, evaluation, schema and serialization definitions in `asap-types`. -Do not special-case the new sketch inside `SketchAlgorithmStrategy` unless the strategy itself needs fundamentally new behavior. +Do not special-case the new sketch inside `ASAPStrategies` unless the strategy itself needs fundamentally new behavior. ### Verifying a new sketch algorithm @@ -804,7 +802,7 @@ or malformed evidence, incompatible metrics and unsupported composition. Test root-target checking before cost ranking, exact fallback, and exported rejection or guarantee data. A cheaper estimate must never admit an accuracy-illegal plan. -After wiring the new algorithm into `summary_candidates` and giving the cost model a real `rank_candidates`/`size_params` opinion about it, check two things. First, that `SketchAlgorithmStrategy::replacements()` for a matching `TargetSubDAG` actually includes a candidate realizing the new algorithm — extend a test shaped like `replacement.rs`'s own test-module coverage-matrix tests (e.g. `agg_intent_to_summary_kind_coverage_matrix`) to cover the new algorithm's `AggIntent`. Second, that `cost_sorted`/`estimate_cost` produce sane, comparable numbers for the new candidate rather than a `NaN` placeholder or an outlier that swamps every other candidate. +After wiring the new algorithm into `summary_candidates` and giving the cost model a real `rank_candidates`/`size_params` opinion about it, check two things. First, that `ASAPStrategies::replacements()` for a matching `TargetSubDAG` actually includes a candidate realizing the new algorithm — extend a test shaped like `replacement.rs`'s own test-module coverage-matrix tests (e.g. `agg_intent_to_summary_kind_coverage_matrix`) to cover the new algorithm's `AggIntent`. Second, that `cost_sorted`/`estimate_cost` produce sane, comparable numbers for the new candidate rather than a `NaN` placeholder or an outlier that swamps every other candidate. --- @@ -895,10 +893,11 @@ silently disagree. ### Mistake: reimplementing summary construction inside a strategy -If the candidate should produce a normal `SummaryNode`, use the existing +If the candidate should produce a normal summary sub-DAG (`SummaryAgg` / +`SummaryEstimate`), use the existing summary-construction path. -A strategy should steer or wrap that path when necessary, not recreate schema derivation, column resolution, readout construction, or parameter sizing. +A strategy should steer or wrap that path when necessary, not recreate schema derivation, column resolution, evaluation construction, or parameter sizing. --- @@ -922,7 +921,7 @@ Workload-wide target discovery, deduplication, and consumer counting are separat For CSE-style decisions, pointer identity can encode actual sharing. -Two `Rc` values can be structurally equal but deliberately represent independent computation. +Two `Rc` values can be structurally equal but deliberately represent independent computation. Use the distinction intentionally. @@ -937,8 +936,8 @@ When adding a new strategy: - [ ] Implement `ReplacementStrategy::replacements`. - [ ] Return every semantically valid replacement. - [ ] Return an empty vector for non-matching targets. -- [ ] Use `Replacement::Summary` for constructed post-ASAP output. -- [ ] Use `Replacement::Rewrite` for logical pre-ASAP alternatives. +- [ ] Return `Replacement::SubDAG` for both constructed post-ASAP output and + logical pre-ASAP alternatives, with the matching `provenance`. - [ ] Add a useful rationale to every candidate. - [ ] Reuse existing legality and implementation logic instead of duplicating it. - [ ] Keep ranking and cost-based pruning out of the strategy. @@ -953,11 +952,11 @@ When adding a new cost model: - [ ] Keep semantic applicability outside the cost model. - [ ] Use `rank_candidates` for algorithm preference; return every input candidate exactly once. - [ ] Use `size_params` for accuracy-to-parameter mapping. -- [ ] Use extension hooks for extension-defined implementations/readouts. +- [ ] Use extension hooks for extension-defined implementations/evaluations. - [ ] Use CSE hooks for recompute-vs.-sharing costs. - [ ] Override `estimate_cost` if consumers require numeric costs instead of `NaN`. - [ ] Test the hook directly. -- [ ] Test integration through a consumer such as `SketchAlgorithmStrategy`. +- [ ] Test integration through a consumer such as `ASAPStrategies`. - [ ] Verify that changing cost preferences does not silently remove valid replacement candidates. --- @@ -975,12 +974,12 @@ Use this table to find the right place for a change. | Prefer one sketch algorithm over another | `CostModel::rank_candidates` | | Change sketch sizing for an accuracy target | `CostModel::size_params` | | Add extension-defined implementation behavior | `CostModel::realize_extension` | -| Add extension-defined readout behavior | `CostModel::readout_extension` | +| Add extension-defined evaluation behavior | `CostModel::evaluation_extension` | | Change CSE recomputation cost | `CostModel::cse_recompute_cost` | | Change shared-maintenance cost | `CostModel::cse_shared_maintenance_cost` | | Change current share/recompute choice | `CostModel::cse_share_decision` | | Decide whether an available implementation satisfies a required one | `impl Matcher` | -| Produce a normal (ranked-first) post-ASAP summary for one target | `SketchAlgorithmStrategy::replacements(...).into_iter().next()` | +| Produce a normal (ranked-first) post-ASAP summary for one target | `ASAPStrategies::replacements(...).into_iter().next()` | | Search a whole workload for supported legal candidates | `search_workload`/`search_workload_with` | | Enforce per-root result accuracy requirements | `search_workload_with_targets` | | Coordinate compatible choices across groups | `CandidateLogicalASAPDAGs::global_selection` | diff --git a/docs/develop_docs/library-api.md b/docs/develop_docs/library-api.md index 719de6ba2..a6eaaea70 100644 --- a/docs/develop_docs/library-api.md +++ b/docs/develop_docs/library-api.md @@ -38,14 +38,16 @@ asap-types = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "e7fdb2 | Public function | Required input | Output | | --- | --- | --- | -| `asap_frontend_promql::lower_promql_workload` | PromQL `PlanningWorkload` with a nonzero `data_ingestion_interval` | All-or-nothing `Result, PromqlError>` for normalized batch and repeating entries | -| `asap_frontend_metricsql::lower_metricsql` | Query string, `AccuracyTarget` | `Result` | -| `asap_frontend_sql::lower_sql` | Query string, `SqlCatalog`, accuracy | Async `Result`; default SQL dialect is DataFusionSQL | +| `asap_frontend_promql::lower_promql_workload` | PromQL `PlanningWorkload` with a nonzero `data_ingestion_interval` | All-or-nothing `Result>, PromqlError>` for normalized batch and repeating entries | +| `asap_frontend_metricsql::lower_metricsql` | Query string, `AccuracyTarget` | `Result, MetricsqlError>` | +| `asap_frontend_sql::lower_sql` | Query string, `SqlCatalog`, accuracy | Async `Result, SqlError>`; default SQL dialect is DataFusionSQL | | `asap_frontend_sql::lower_sql_dialect` | Same inputs plus `SqlDialect` | Async resolved Pre-ASAP query or error | | `asap_frontend_sql::lower_sql_batch` | `QueryWorkload` and catalog | Per-query results for `query_batch`; does not iterate `repeating_queries` | Lowering resolves the supported source language into the canonical query -representation. It does not enumerate Post-ASAP alternatives. A frontend may +representation: an `asap_types::ir::OperatorNode` DAG containing only +`NonASAPOp` operators, with no timing (see the +[Pre-ASAP IR reference](pre-asap-ir.md)). It does not enumerate Post-ASAP alternatives. A frontend may reject unsupported syntax or semantics; a declared language/dialect enum does not imply complete support. PromQL workload lowering uses normalized `PlanningWorkload::query_workload.entries()` order, preserving entry-to-root associations for later @@ -58,7 +60,7 @@ PromQL's public signature (types are imported from their respective crates): ```text lower_promql_workload(workload: &PlanningWorkload, now_ms: u64) - -> Result, PromqlError> + -> Result>, PromqlError> ``` `DataWorkload.data_ingestion_interval` must contain a nonzero `Evidence`. @@ -125,9 +127,9 @@ For SQL, the corresponding signatures are: ```text async lower_sql(query: &str, catalog: &SqlCatalog, accuracy: AccuracyTarget) - -> Result + -> Result, SqlError> async lower_sql_dialect(query: &str, catalog: &SqlCatalog, - dialect: SqlDialect, accuracy: AccuracyTarget) -> Result + dialect: SqlDialect, accuracy: AccuracyTarget) -> Result, SqlError> ``` | `SqlDialect` value | Current behavior | @@ -162,7 +164,7 @@ It keeps the alternatives available; it does not select an entire workload plan. ```text search_workload_with_targets<'s, Id>( - roots: Vec<(Id, Rc, Option)>, + roots: Vec<(Id, Rc, Option)>, strategies: &[Box], accuracy_model: &dyn AccuracyModel, ) -> CandidateLogicalASAPDAGs @@ -197,7 +199,6 @@ accuracy target, and prints every ranked candidate instead of selecting a winner The default cost model is suitable for inspection, not deployment calibration. ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -235,7 +236,7 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let cost_model = DefaultCostModel; let strategies = default_strategies_with(&cost_model); let space = search_workload_with_targets( @@ -254,12 +255,12 @@ fn main() -> Result<(), Box> { | API (`asap_aware_mapping`, unless qualified) | Inputs | Output and limits | | --- | --- | --- | -| `search_workload` | `(query_id, Rc)` roots | `CandidateLogicalASAPDAGs` with built-in strategies/model; no explicit per-root target argument | +| `search_workload` | `(query_id, Rc)` roots | `CandidateLogicalASAPDAGs` with built-in strategies/model; no explicit per-root target argument | | `search_workload_with` | Roots, strategy slice | `CandidateLogicalASAPDAGs`; callers choose context-free replacement strategies | | `search_workload_with_targets` | Roots with optional end-to-end targets, strategies, accuracy model | Candidate space with supplied root-target checks; `None` does not supply a root-level requirement; uncertified direct DDSketch ratios remain available for backend selection | | `CandidateLogicalASAPDAGs::cost_sorted` | Cost model | `Vec`; retains alternatives and pairs `candidates[i]` with `costs[i]` | | `CandidateLogicalASAPDAGs::cost_sorted_with_recurrence` | Cost model, recurrence profiles, optional horizon | Ranked per-target candidate sets or `RecurrenceError`; uses recurrence for applicable share/recompute comparisons | -| `SketchAlgorithmStrategy::replacements` through `ReplacementStrategy` | One `TargetSubDAG` | Alternatives at that target; not whole-workload search | +| `ASAPStrategies::replacements` through `ReplacementStrategy` | One `TargetSubDAG` | Alternatives at that target; not whole-workload search | `cost_sorted` is a ranking view, not a request to discard all but the first candidate. Display costs follow model hooks and may be unavailable/non-finite; @@ -279,12 +280,12 @@ choices are not multiplied in. Exceeding `expansion_limit` is an error, never a partial inventory. For PromQL roots that carry a target, `search_workload_with_targets` also asks -each strategy's `ReplacementStrategy::propose_for_root`. `SketchAlgorithmStrategy` +each strategy's `ReplacementStrategy::propose_for_root`. `ASAPStrategies` answers an instant-vector TopK with current-series heap realizations over rows carrying the complete series identity (`$promql_series_identity`). They are finalized, deduplicated, and marked `ReplacementProvenance::RootPhysicalRealization`. Callers do not apply `with_series_identity` themselves. Compile each with -`promql_rows::compile_current_series_readout`; other queries keep their previous +`promql_rows::compile_current_series_evaluation`; other queries keep their previous inventory. `global_selection` never commits these candidates; the backend compiles and prices them. CandidateLogicalASAPDAGs lists no placement variants: node timing comes from the summary maintenance lifecycle. @@ -300,11 +301,11 @@ pass. An omitted strategy contributes no proposals of its own. | Value to put inside `Box::new(...)` | Meaning | In default factories? | | --- | --- | --- | -| `SketchAlgorithmStrategy::new(&model)` | Enumerates supported exact/sketch implementations and parameter choices for aggregate targets | Yes | +| `ASAPStrategies::new(&model)` | Enumerates supported exact/sketch implementations and parameter choices for aggregate targets | Yes | | `HydraGroupingStrategy::new(&model)` | Considers a shared multi-subpopulation structure for supported grouped sketch families, subject to accuracy evidence | Yes | | `SharedSubDAGStrategy` | Proposes sharing versus independent recomputation at reused sub-DAGs | Yes | | `SemanticEquivalentRewriteStrategy` | Proposes supported equivalent aggregate rewrites, including decomposing average into sum/count | Yes | -| `ExactCompositionStrategy::new(&model)` | Proposes supported exact operations around summary readouts or in maintenance | Yes | +| `ExactCompositionStrategy::new(&model)` | Proposes supported exact operations around summary evaluations or in maintenance | Yes | | Your `ReplacementStrategy` implementation | Adds domain-specific legal replacement proposals | No | `AvgToSumOverCountStrategy` is an alias for `SemanticEquivalentRewriteStrategy` @@ -345,7 +346,6 @@ replacement::default_strategies_with_evidence<'a>( ### Example: supply two strategies and run search ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -353,7 +353,7 @@ use asap_types::workload::{ }; use asap_aware_mapping::{ search_workload_with_targets, DefaultAccuracyModel, DefaultCostModel, - ReplacementStrategy, SketchAlgorithmStrategy, SharedSubDAGStrategy, + ReplacementStrategy, ASAPStrategies, SharedSubDAGStrategy, }; use asap_types::types::AccuracyTarget; @@ -383,10 +383,10 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let model = DefaultCostModel; let strategies: Vec> = vec![ - Box::new(SketchAlgorithmStrategy::new(&model)), + Box::new(ASAPStrategies::new(&model)), Box::new(SharedSubDAGStrategy), ]; let space = search_workload_with_targets( @@ -442,7 +442,7 @@ accuracy guarantees. ```rust use asap_aware_mapping::{ DefaultAccuracyModel, DefaultCostModel, EqualSplitAllocator, - NoAccuracyEvidence, ReplacementStrategy, SketchAlgorithmStrategy, + NoAccuracyEvidence, ReplacementStrategy, ASAPStrategies, }; fn main() { @@ -451,7 +451,7 @@ fn main() { let allocation = EqualSplitAllocator; let evidence = NoAccuracyEvidence; let strategies: Vec> = vec![Box::new( - SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( + ASAPStrategies::new_with_planning_inputs_and_evidence( &cost, &accuracy, &allocation, &evidence, ), )]; @@ -463,16 +463,16 @@ fn main() { Constructor definition: ```text -SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence( +ASAPStrategies::new_with_planning_inputs_and_evidence( cost_model: &dyn CostModel, accuracy_model: &dyn AccuracyModel, allocator: &dyn AccuracyBudgetAllocator, evidence: &dyn AccuracyEvidenceProvider, -) -> SketchAlgorithmStrategy +) -> ASAPStrategies ``` All provider arguments are required for this constructor. They must outlive the -strategy vector. `SketchAlgorithmStrategy::new(&cost_model)` is the shorter +strategy vector. `ASAPStrategies::new(&cost_model)` is the shorter constructor using default accuracy/allocation and no extra evidence. | Extension point | What it controls | What it cannot establish alone | @@ -488,7 +488,7 @@ with the intended model/evidence; replacing only the final sorting model does no regenerate parameter choices. For evidence-aware defaults, use `asap_aware_mapping::replacement::default_strategies_with_evidence`. For custom accuracy/allocation/evidence on sketches, -`SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence` exposes these providers. +`ASAPStrategies::new_with_planning_inputs_and_evidence` exposes these providers. Keep each provider's evidence scope and freshness valid for the query population. ## Workload inputs and defaults @@ -525,7 +525,7 @@ Use this workflow when Planner owns summary-maintenance lifecycle decisions; otherwise the backend may make them from logical candidates. It includes both selection and DAG assembly, so callers do not first run the ordinary workflow. The first helper returns one `GlobalSelection`; the second is called per root -and returns a plan containing `root: Rc` plus maintenance decisions. +and returns a plan containing `root: Rc` (already timed) plus maintenance decisions. See the [workflow design](../design_docs/architecture/input-output-workflow.md#summary-maintenance-lifecycle-aware-helper). Two capabilities are distinct: the runtime can orchestrate a lifecycle, and the @@ -542,7 +542,7 @@ global_selection_with_summary_maintenance_lifecycles<'a, Id>( ) -> Result, SummaryMaintenanceLifecycleSelectionError> assemble_selected_dag_with_summary_maintenance_lifecycles( - selection: &GlobalSelection<'_>, target: &Rc, + selection: &GlobalSelection<'_>, target: &Rc, demand: WorkloadDemand<'_>, now_ms: u64, horizon: Option, capabilities: SummaryMaintenanceLifecycleCapabilities, cost_model: &dyn CostModel, ) -> Result, SummaryMaintenanceLifecycleAssemblyError> @@ -655,7 +655,7 @@ from that hook, as in Planner selection. A lifecycle choice then fixes each physical placement through timing: a continuously maintained state and its inputs run at ingestion time, while an -ephemeral one stays at query time. Compile each query's `PostAsapDAG` once and +ephemeral one stays at query time. Compile each query's `PhysicalASAPDAG` once and cut every chosen assignment from that result: ```rust @@ -735,7 +735,7 @@ workflow for those decisions. Downstream still owns physical commitment. | --- | --- | | `CandidateLogicalASAPDAGs::global_selection(&model)` | Compatible structural selection across targets; no recurrence or lifecycle planning implied | | `CandidateLogicalASAPDAGs::global_selection_with_recurrence(...)` | Compatible selection using supplied recurrence profiles/horizon; no lifecycle commitments implied | -| `GlobalSelection::assemble_selected_dag(&target)` | `Result>, RealizationError>`; constructs semantic IR, not stored summary data | +| `GlobalSelection::assemble_selected_dag(&target)` | `Result>, RealizationError>`; constructs untimed semantic IR, not stored summary data | Use a target associated with the searched space; DAG assembly can return `None` when that target is absent. A downstream integration can use these convenience @@ -747,8 +747,8 @@ for checking complete physical alternatives and deployment constraints. ```text CandidateLogicalASAPDAGs::global_selection(&self, cost_model: &dyn CostModel) -> GlobalSelection<'_> -GlobalSelection::assemble_selected_dag(&self, target: &Rc) - -> Result>, RealizationError> +GlobalSelection::assemble_selected_dag(&self, target: &Rc) + -> Result>, RealizationError> ``` For structural inspection only, this complete example selects a semantic root @@ -756,7 +756,6 @@ and exports its inspection DAG. It performs no lifecycle or deployment planning. Use lifecycle-aware selection above when the comparison needs those decisions. ```rust -use std::rc::Rc; use asap_frontend_promql::lower_promql_workload; use asap_types::workload::{ AccuracyRequirement, BatchEntry, DataWorkload, DurationMs, Evidence, Query, @@ -790,7 +789,7 @@ fn main() -> Result<(), Box> { ..Default::default() }), }; - let root = Rc::new(lower_promql_workload(&workload, 0)?.remove(0)); + let root = lower_promql_workload(&workload, 0)?.remove(0); let space = search_workload(vec![("q1", root)]); let selection = space.global_selection(&DefaultCostModel); // Search may canonicalize roots; use the root returned by CandidateLogicalASAPDAGs. @@ -806,11 +805,12 @@ fn main() -> Result<(), Box> { | Function/type | Purpose | | --- | --- | -| `asap_types::dag_export::export(&query)` | Pre-ASAP inspection DAG | -| `asap_types::dag_export::export_summary(&summary)` | Post-ASAP inspection DAG | -| `asap_types::post_asap::compile_post_asap_dag(&root)` | Compile a semantic DAG with execution-data-state validation; not a physical plan | -| `PostAsapDAGDocument::new(dag)` and `.validate()` | Versioned semantic envelope and explicit validation; constructing it alone does not validate | -| `asap_aware_mapping::export_summary_maintenance_plan(&plan)` | DAG plus lifecycle deployments, alternatives and available cost/guarantee information | +| `asap_types::dag_export::export(&query)` | Pre-ASAP inspection dag | +| `asap_types::dag_export::export_summary(&summary)` | Post-ASAP inspection dag | +| `asap_types::ir::apply_lifecycle_timings(&root, &assignment, &mut TimingMemo::new())` | Write execution timing into every node from a `LifecycleAssignment` and validate the data-state edges; a lifecycle plan's `root` is already timed | +| `asap_types::ir::physical_export::compile_physical_asap_dag(&timed_root)` | Export a timed DAG as a `PhysicalASAPDAG` (`PHYSICAL_ASAP_DAG_WIRE_VERSION`); rejects an untimed node; `compile_physical_asap_workload(&roots)` exports a batch with one root per query | +| `PhysicalASAPDAGDocument::new(dag)` and `.validate()` | Versioned semantic envelope and explicit validation; constructing it alone does not validate | +| `asap_aware_mapping::export_summary_maintenance_plan(&plan)` | Graph plus lifecycle deployments, alternatives and available cost/guarantee information | | `explain_replacements` / `explain_replacements_with` | Findings from default/custom-strategy search; not a complete physical feasibility report | Choose the export matching your intended handoff: an inspection DAG is not diff --git a/docs/develop_docs/metrics-observability-corpora.md b/docs/develop_docs/metrics-observability-corpora.md index d127e92ce..129ea0af1 100644 --- a/docs/develop_docs/metrics-observability-corpora.md +++ b/docs/develop_docs/metrics-observability-corpora.md @@ -49,19 +49,19 @@ o11y-bench, and awesome-prometheus-alerts. They are not duplicated here. The test prints totals, parse errors, lowering errors, pre-ASAP successes, post-ASAP candidates, unchanged queries, and post-ASAP errors. `Pre-ASAP` means -that parsing and lowering produced a `QueryExpr`. `Post-ASAP candidate` means -the isolated `SketchAlgorithmStrategy` produced a non-`KeepPreAsap` summary -candidate. `Unchanged` is a successful pre-ASAP query for which that strategy -returned only the pre-ASAP fallback. +that parsing and lowering produced an `OperatorNode` DAG. `Post-ASAP candidate` +means the isolated `ASAPStrategies` produced a candidate that contains +an ASAP operator (`contains_asap()`). `Unchanged` is a successful pre-ASAP query +for which that strategy returned only the kept pre-ASAP sub-DAG (`retain_exact`). ## Strategies The corpus measurement deliberately uses only -`SketchAlgorithmStrategy::default_cost_model().replacements(...)` on each +`ASAPStrategies::default_cost_model().replacements(...)` on each query root. It does not measure workload-wide search or the other default strategies. -The default workload search currently registers `SketchAlgorithmStrategy`, +The default workload search currently registers `ASAPStrategies`, `HydraGroupingStrategy`, `SharedSubDAGStrategy`, and `AvgToSumOverCountStrategy`. Workload context can additionally contribute `RollupStrategy` and `AccuracyReconciliationStrategy`. This baseline is diff --git a/docs/develop_docs/native-promql-inputs.md b/docs/develop_docs/native-promql-inputs.md index d263b7825..d2c10f535 100644 --- a/docs/develop_docs/native-promql-inputs.md +++ b/docs/develop_docs/native-promql-inputs.md @@ -23,7 +23,7 @@ operator's metric-name/result-label rules. Source selection, complete window coverage and revision admission remain deployment responsibilities. Planner's maintained-population candidate recognizes this explicit identity -representation. Its TopK readout compiles automatically to `CurrentSeries`, +representation. Its TopK evaluation compiles automatically to `CurrentSeries`, `Sort`, and `Limit`; deployment supplies the raw boundary or an already maintained population boundary. Compilation does not open either source. diff --git a/docs/develop_docs/offline-sketch-evidence.md b/docs/develop_docs/offline-sketch-evidence.md index cd331e57d..56c507de1 100644 --- a/docs/develop_docs/offline-sketch-evidence.md +++ b/docs/develop_docs/offline-sketch-evidence.md @@ -91,7 +91,7 @@ hooks. The provider's lifecycle helper returns available build/update CPU costs for a single independently instantiated state. It deliberately leaves retention, retirement and read costs unknown. In particular, a point-frequency benchmark read does not price a total-count read, even when both use CMS. A deployment must -match readout semantics and supply the missing lifecycle and raw-query evidence +match evaluation semantics and supply the missing lifecycle and raw-query evidence before selecting and pricing a complete physical plan. Never combine these nanosecond costs with CPU operation counts without explicit calibration. @@ -116,7 +116,7 @@ not be passed as these disjoint phase measurements. `MeasurementQueryBinding` is the producer's explicit assertion identifying the read/error probe population. The consumer checks that binding and the error -record's readout kind/value type; it cannot recover or certify the original +record's evaluation kind/value type; it cannot recover or certify the original probe set from an aggregate error number alone. The supported workload is an immutable i64 point-frequency snapshot, fully @@ -131,7 +131,7 @@ post-merge error and an exact merge baseline exist. The caller supplies an `EmpiricalAccuracyRequirement`: the exact observed error metric, maximum accepted mean, and minimum number of offline trials. This is -separate from `AccuracyTarget`. Every candidate must match the readout descriptor, +separate from `AccuracyTarget`. Every candidate must match the evaluation descriptor, error metric, trial count and all ordinary distribution/configuration/environment checks. A zero observed error is neither proof of exactness nor a per-key bound. diff --git a/docs/develop_docs/physical-compile-coverage.md b/docs/develop_docs/physical-compile-coverage.md index 5d048d7f7..920626e9f 100644 --- a/docs/develop_docs/physical-compile-coverage.md +++ b/docs/develop_docs/physical-compile-coverage.md @@ -6,12 +6,12 @@ Audience: developers moving computation from ASAPQuery-backend into ## Contract Logical selection decides what to compute. The maintenance lifecycle sets node -timing. `physical_planner::compile` turns a timed `PostAsapDAG` into physical +timing. `physical_planner::compile` turns a timed `PhysicalASAPDAG` into physical operator DAGs. The backend owns ingestion, panes, storage, stored-state -readout, external exact engines, pricing/selection, and execution scheduling. +evaluation, external exact engines, pricing/selection, and execution scheduling. A backend lowering is *covered* when `compile` accepts the corresponding -`PostAsapDAG` node and produces operators with the same result. The backend +`PhysicalASAPDAG` node and produces operators with the same result. The backend should then pass the timed DAG and its input contracts to `compile`. It should not rebuild operator choices from PromQL text or construct operators itself. @@ -20,6 +20,10 @@ not rebuild operator choices from PromQL text or construct operators itself. Surveyed backend: `ASAPQuery-backend` branch `perf/788-startup-search`. Planner base: `split/462-f-physical-planner` (#475). +`Fallback` below names the legacy wrapper node of that base. #543 removed it: +those shapes are ordinary operator nodes now, and the Planner has no +execution fallback. + Status values: - **Supported**: `compile` or `compile_node` already covers this computation. @@ -44,14 +48,14 @@ Status values: | 13 | `QueryPlanNode::Scalar`, `DAGCompiler::lower` scalar literal | Scalar constant | `Fallback{PromqlScalarBridge(Literal)}` | Missing | Only `promql_values::compile_scalar`. | | 14 | `DAGCompiler::lower` `ReduceSum`; `physical_values.rs` PerEntity projection | Sum over finalized values; per-entity identity | `SummaryAgg{ExactAggregate(Sum)}` | Supported | The backend builds an identity `Operator::project` itself for PerEntity. | | 15 | `DAGCompiler::lower` `ExactReadout`; `post_asap_readout.rs` ExactReadout | Finalize exact state (sum/count/min/max/rate/increase) | `Value::FinalizeExactAccumulator` | Partial | Count yields Int64 against a declared Float64 PromQL value. `compile` rejects it. | -| 16 | `post_asap_readout.rs` SummaryEstimate (`readout_bound`, `expand_item_rows`) | Sketch estimate per group; TopK item expansion | `SummaryEstimate` | Partial | The backend's label-map state layout and MetricsQL `__name__` rules have no Planner equivalent. `compile_exact_readout` has no sketch counterpart. | +| 16 | `post_asap_readout.rs` SummaryEstimate (`readout_bound`, `expand_item_rows`) | Sketch estimate per group; TopK item expansion | `SummaryEstimate` | Partial | The backend's label-map state layout and MetricsQL `__name__` rules have no Planner equivalent. `compile_exact_evaluation` has no sketch counterpart. | | 17 | `post_asap_readout.rs` SummaryMerge (`merge_bound_states`) | Merge states by group | `SummaryMerge` | Supported | Union plus `summary_merge`. | | 18 | `post_asap_readout.rs` counter range parameters | Counter lookback for rate/increase | `TimeRange` ancestor of finalization | Supported | Applied through `with_counter_lookback`. | | 19 | `post_asap_readout.rs` `execute_value_fragment` | Per-timestamp binding of a value fragment | n/a | Backend | Evaluation scheduling. | | 20 | `DAGCompiler::lower` SummaryJoin / Subtract / Delete | Summary algebra | `SummaryJoin`, `SummarySubtract`, `SummaryDelete` | Missing | The backend also rejects these (`ExactFallback`). | -| 21 | `current_series.rs` Snapshot + TopK | Current-series ranking | `ReadPopulation{TopK}` | Supported | | -| 22 | `current_series.rs` Sum / Count / Average | Current-series aggregates | `ReadPopulation{Sum,Count,Average}` | Missing | `compile` accepts only TopK. | -| 23 | `current_series.rs` Quantile | Current-series quantile | `ReadPopulation{Quantile}` | Missing | No exact quantile reduction. | +| 21 | `current_series.rs` Snapshot + TopK | Current-series ranking | `EvaluatePopulation{TopK}` | Supported | | +| 22 | `current_series.rs` Sum / Count / Average | Current-series aggregates | `EvaluatePopulation{Sum,Count,Average}` | Missing | `compile` accepts only TopK. | +| 23 | `current_series.rs` Quantile | Current-series quantile | `EvaluatePopulation{Quantile}` | Missing | No exact quantile reduction. | | 24 | `raw_dag.rs` weight `Column` | Summary update from a sample/projected value | `SummaryAgg` | Supported | | | 25 | `raw_dag.rs` weight `Constant` | Unit/constant-weight update | `SummaryAgg` | Missing | `compile_node` requires a column weight. | | 26 | `raw_dag.rs` item `Column` / `Tuple` | Keyed update item | `SummaryAgg{item}` | Supported | `keyed_summary_build`. | @@ -70,20 +74,21 @@ Totals at #475: 11 Supported, 4 Partial, 14 Missing, 2 Backend. | 4, 8, 13 | Query-time `Binary` folds a scalar-literal operand into a projection over grouped value rows. | | 5 | Query-time `Binary` over grouped value rows performs an inner equi-join on equal label columns, then applies the operator. Per-series rows remain Partial. | | 15 | Count finalization converts exactly to the declared Float64 value. | -| 22, 23 | `ReadPopulation` Sum/Count/Average/Quantile compile to grouped aggregation. `Reduction::Quantile` implements PromQL interpolation. | +| 22, 23 | `EvaluatePopulation` Sum/Count/Average/Quantile compile to grouped aggregation. `Reduction::Quantile` implements PromQL interpolation. | Totals after this change: 17 Supported, 4 Partial, 8 Missing, 2 Backend. ## Covered by PromQL fallback compilation -`compile` now lowers a `Fallback{QueryExpr}` node from its typed expression, -realized with `promql_rows::with_series_identity`. The deployment supplies the -raw rows of the `i`th selector returned by `promql_fallback::raw_series` at -`promql_fallback::raw_series_input(node, i)`, with that selector's schema. Each -selector has its own slot, even when two selectors read the same metric. The -node's own ID still names its output, so a deployment may instead supply the -whole result, for example from an external exact engine. A Fallback that reads -no selector, such as `vector(1)`, needs no input. +`compile` lowers each PromQL operator node (selector, range function, +subquery, binary operator) natively, realized with +`promql_rows::with_series_identity`. The deployment supplies the raw rows of +each selector as the input of its `Scan` node, by that node's ID and with its +schema. Each selector has its own `Scan`, even when two selectors read the same +metric. A node's own ID still names its output, so a deployment may instead +supply that whole result, for example from an external exact engine. A node +that reads no selector, such as `vector(1)`, needs no input. The Planner has no +execution fallback: a node `compile` cannot lower fails compilation. `Operator::series_window` evaluates each series at the query time, or at each subquery step, over the left-open window `(t - offset - range, t - offset]`. @@ -124,10 +129,10 @@ Totals are unchanged: 19 Supported, 5 Partial, 5 Missing, 2 Backend. | Row | Change | |---|---| -| 5 | Query-time `Binary` over rows with a series identity, such as per-series readouts of stored state, uses the Fallback's `series_labels` and `series_binary`. Examples: `avg_over_time` as stored sum/count, and `rate(a) / rate(b)`. Matching drops `__name__` and honors `on`/`ignoring` when the payload carries them. Only one-to-one arithmetic is covered; `group_left`/`group_right` stay rejected and comparisons are row 7. Now Supported. | +| 5 | Query-time `Binary` over rows with a series identity, such as per-series evaluations of stored state, uses the Fallback's `series_labels` and `series_binary`. Examples: `avg_over_time` as stored sum/count, and `rate(a) / rate(b)`. Matching drops `__name__` and honors `on`/`ignoring` when the payload carries them. Only one-to-one arithmetic is covered; `group_left`/`group_right` stay rejected and comparisons are row 7. Now Supported. | | 4, 8 | A literal operand also applies to per-series rows and drops `__name__`, in the Fallback too. Series whose label sets become equal are an error, as in Prometheus. | -Grouped `sum`/`avg`, current-series `Sum`/`Average` readouts, and +Grouped `sum`/`avg`, current-series `Sum`/`Average` evaluations, and `sum_over_time`/`avg_over_time` use Prometheus' Kahan-Neumaier summation. An average switches to an incremental mean once the running sum would overflow. The grouped path also serves SQL `SUM`/`AVG` over Float64, which are now @@ -175,7 +180,7 @@ and `group_left`, including a right-side series identity when needed. Thus |---|---| | 1 | Comparisons, `bool`, set operators, `group_left`/`group_right`, `scalar()` operands, and literals over aggregates whose value has another name, such as `sum by (job) (a) * 2`. Still Partial. | | 5 | Grouped `Binary` rows use the same operator instead of a relational join. A duplicate match group is now an error instead of a cross product. | -| 7 | Fallback, grouped `Binary`, and per-series comparisons and sets. Temporal stored readouts drop `__name__` before matching, including exact Count conversion, and reject duplicate output identities. | +| 7 | Fallback, grouped `Binary`, and per-series comparisons and sets. Temporal stored evaluations drop `__name__` before matching, including exact Count conversion, and reject duplicate output identities. | Totals after this change: 20 Supported, 5 Partial, 4 Missing, 2 Backend. @@ -188,7 +193,7 @@ Totals after this change: 20 Supported, 5 Partial, 4 Missing, 2 Backend. An argument whose output provably lacks `le`, such as `sum by (job) (rate(x_bucket[5m]))`, is rejected at lowering. Prometheus returns an empty vector for it. Candidate search keeps the classic form as one -exact `KeepPreAsap` sub-DAG for every accuracy target; it has no sketch +retained exact ordinary sub-DAG for every accuracy target; it has no sketch candidate. `histogram_quantiles` lowers each branch the same way; the Fallback compiler accepts its `Concat` of relabeled branches and rejects duplicate output label sets. Nested aggregation, such as @@ -200,7 +205,7 @@ Totals after this change: 21 Supported, 5 Partial, 3 Missing, 2 Backend. In order of backend usage: -1. Rows 1 and 12, the remaining `Fallback` shapes: +1. Rows 1 and 12, the remaining PromQL shapes: - Other scalar functions as operands. - Subquery operands other than one per-series function; implicit subquery resolution, which is a deployment default. @@ -208,8 +213,8 @@ In order of backend usage: After these shapes are covered, the backend can delete rows 28 and 30. 2. Rows 25 and 27: constant weights and `EntityIdentity` items for precompute `SummaryAgg`. -3. Row 16: a label-map sketch-state readout, the counterpart of - `compile_exact_readout`, and MetricsQL `__name__` retention rules. +3. Row 16: a label-map sketch-state evaluation, the counterpart of + `compile_exact_evaluation`, and MetricsQL `__name__` retention rules. 4. Row 20: summary join, subtract, and delete. `fill`, `fill_left`, and `fill_right` matching modifiers are rejected by the diff --git a/docs/develop_docs/planner-vocabulary-migration.md b/docs/develop_docs/planner-vocabulary-migration.md index ea46e992c..80ef6de72 100644 --- a/docs/develop_docs/planner-vocabulary-migration.md +++ b/docs/develop_docs/planner-vocabulary-migration.md @@ -12,7 +12,7 @@ names. | `replacement::ImplementError` | `replacement::RealizationError` | | `ReplacementProvenance::SummaryImplementation` | `ReplacementProvenance::SummaryRealization` | | `implementations_for_with` (internal) | `realizations_for_intent` | -| `pre_asap::binder` | `pre_asap::schema_resolver` | +| `pre_asap::binder` | `asap_frontend_common::schema_resolver` | | `Binder` | `SchemaResolver` | | `Binder::bind` | `SchemaResolver::resolve_schema` | | `Binder::bind_with_inherited` | `SchemaResolver::resolve_schema_with_inherited` | @@ -32,8 +32,9 @@ names. | Physical evidence/comparison `boundaries` fields | `handoffs` | | `BoundaryEstimate::per_boundary` | `PhysicalHandoffEstimate::per_handoff` | | Internal `Models` | `CandidatePlanningInputs` | -| `SketchAlgorithmStrategy::with_models` | `SketchAlgorithmStrategy::new_with_planning_inputs` | -| `SketchAlgorithmStrategy::with_models_and_evidence` | `SketchAlgorithmStrategy::new_with_planning_inputs_and_evidence` | +| `SketchAlgorithmStrategy::with_models` | `ASAPStrategies::new_with_planning_inputs` | +| `SketchAlgorithmStrategy::with_models_and_evidence` | `ASAPStrategies::new_with_planning_inputs_and_evidence` | +| `SketchAlgorithmStrategy` | `ASAPStrategies` | | `HydraGroupingStrategy::with_models_and_evidence` | `HydraGroupingStrategy::new_with_planning_inputs_and_evidence` | For example, `Binder::new().bind(&dag)` becomes diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 77d80c847..cae40d0c3 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -2,7 +2,15 @@ This is the detailed node reference. Start with the [Pre-ASAP IR concept](../design_docs/concepts/pre-asap-ir.md) for purpose and the compact catalog. -The goal of the pre-ASAP IR is represent operations from different query languages in a single representation, and make it easier to analyze how/where ASAP primitives can be used. +ASAPPlanner has **one operator IR before and after ASAP optimization**, defined in +`crates/types/src/ir/`. "Pre-ASAP" is not a separate type: it is this IR as a front end +emits it, before any ASAP operator has been introduced. This document covers what every +plan shares — the node, the schema, scalar expressions, how front ends produce the DAG, and +the catalog of ordinary (`NonASAPOp`) operators. The ASAP operators, execution timing and +the exported wire form are described in the [Post-ASAP IR](../design_docs/concepts/post-asap-ir.md) +document; the two do not repeat each other. + +The goal of the pre-ASAP form is to represent operations from different query languages in a single representation, and make it easier to analyze how/where ASAP primitives can be used. Only operations that are semantically relevant to answering the query and selecting an ASAP primitive need to become first-class nodes here. ## Design principles @@ -13,7 +21,135 @@ Only operations that are semantically relevant to answering the query and select > Notes: **SQL and PromQL use different schema models**. SQL typically uses a closed schema, where tables, columns, and types are predefined, while PromQL uses an open (schemaless) schema, where metrics and labels can evolve without a fixed table schema. Closed schemas provide stronger structure and validation; open schemas provide greater flexibility and makes it easier to evolve or ingest diverse data, but can require more care around naming conventions, label cardinality, and query consistency. -The pre-ASAP IR is defined using the `QueryExpr` enum. We discuss some of important enum types below. +## The node + +A plan is a DAG of `Rc` (`crates/types/src/ir/node.rs`). Nodes are immutable +and shared through `Rc`: a structurally identical sub-DAG referenced from several parents is +one node, and that pointer identity is what CSE, target discovery and plan assembly key on. + +```rust +pub struct OperatorNode { + pub operator: Operator, // NonASAP(NonASAPOp) | ASAP(ASAPOp) + pub result_kind: OperatorResultKind, // Relation | InstantVector | RangeVector | State | Scalar + pub schema: Schema, // output schema, derived at construction + pub guarantee: Option, // None until accuracy assessment establishes one + pub timing: Option, // None until a lifecycle assignment is applied +} +``` + +- `operator` is the operation. A front-end DAG contains only `Operator::NonASAP` nodes; + `OperatorNode::expect_non_asap()` relies on that. +- `result_kind` is the output category, derived from the operator and its inputs. Matching + column schemas do not make categories interchangeable (a range vector is not an instant + vector). +- `schema` is derived by `OperatorNode::new(operator)`; it fails when the schema cannot be + derived (a column reference out of range, a reserved ASAP operator). ASAP planning may + retain a more specific schema through `OperatorNode::with_schema`. +- `guarantee` is `None` until accuracy assessment establishes one; `None` never means exact. +- `timing` is `None` in every front-end DAG and every candidate. It is written by + `ir::timing::apply_lifecycle_timings` (see the Post-ASAP IR document); export rejects an + untimed node. + +`OperatorNode::children()` returns the operator's inputs in field order followed by the +operator nodes its scalar expressions read (see "Scalar expressions"). Every DAG traversal — +`map_children`, `reachable`, `contains_asap`, CSE, export — follows that same list. +`OperatorNode::validate_structure()` checks every operator's input contract, scalar typing +against the owning operator's input schema, and that each retained schema agrees with the +derived one. + +## Schema + +One `Schema` type (`crates/types/src/pre_asap/schema.rs`) describes every edge, whether it +carries rows or summary state: + +```rust +pub struct Schema { + pub fields: Vec, // positional; every ColumnId indexes into this + pub time_index: Option, // the time axis, if any (PromQL leaves always have one) + pub unique_keys: Vec>, + pub closed: bool, // true: these are all the columns; false: open (schemaless) superset +} + +pub struct Field { + pub name: String, + pub dtype: FieldDataType, // Plain(DataType) | ExactAggregate(..) | Sketch(..) | Sample(..) | Wavelet(..) | StatModel(..) + pub nullable: bool, + pub table: Option, // SQL table/alias qualifier; None for PromQL labels +} +``` + +A pre-ASAP field is always `FieldDataType::Plain(DataType)`. The other variants carry summary +state and only appear below an ASAP operator; a scalar expression that reads such a field is a +typing error (`ScalarExpr::scalar_type`), because state must be read out before a value can use +it. Column references are positional `ColumnId`s (indexes into the input schema), never names. + +`Schema::has_unique_key()` is the legality gate CSE uses: a non-ASAP producer is only shared +across consumers when its row identity is provable. + +## Scalar expressions + +Value computation lives in `ScalarExpr` (`crates/types/src/ir/scalar.rs`), owned **by value** +by an operator field: `Scan.predicates`, `Filter.pred`, `Join.pred`, `Project.cols[i].expr`, +`Aggregate.having`, `Sort.keys[i].expr`, `SQLWindowFunc.args`/`order_by`, `PromqlRelabel.value`, +`Values.rows`, and `QueryRoot::Scalar` and `PromqlVectorFromScalar`. A scalar expression never +produces a table and is never a node of the DAG; it is evaluated against the input schema of +the operator that owns it. + +Variants: `Column(ColumnId)`, `Literal(ScalarValue)`, `Negative` (unary minus), `Compare`, +`BoolAnd` / `BoolOr` (flat conjunction/disjunction), `Not`, `IsNull` / `IsNotNull`, `Cast` +(with `try_cast`), `InList`, `FunctionCall { name, args }`, `Arithmetic`, `Case`, +`CurrentTimestamp` (SQL `NOW()`), `EvalTimestamp` (PromQL `time()`), and four +**plan-reading** variants that reference an operator node: + +| Variant | Meaning | +|---|---| +| `PromqlScalarFromVector(Rc)` | PromQL `scalar(v)`: the single sample of an instant vector, NaN otherwise | +| `ScalarSubquery(Rc)` | Uncorrelated SQL scalar subquery: one column; zero rows is NULL, more than one row is an error | +| `Exists { subquery, negated }` | SQL `[NOT] EXISTS (subquery)` | +| `InSubquery { expr, subquery, negated }` | SQL `expr [NOT] IN (subquery)` over a one-column relation | + +These are the **only** operator references inside a scalar tree. `ScalarExpr::operator_refs()` +lists them, `NonASAPOp::children()` appends them after the operator's own inputs, and +canonicalization lowers the three SQL subquery forms to joins (see below), so a canonical SQL +DAG contains none of them. `PromqlScalarFromVector` survives canonicalization: its referenced +vector is a real plan dependency, exported as a `ScalarRef` edge. + +`Compare`, `Arithmetic` and `Negative` carry an `ExprSemantics` (`Sql` or `Promql`): both +languages use `Float64`, so the result type alone does not preserve NaN, ordering or error +rules, and the executing engine needs to know which language's rules apply. + +Wrapper types: `Predicate(ScalarExpr)`, `ProjectItem { alias, expr }`, +`SortKey { expr, ascending, nulls_first }`. + +## How a front end produces the DAG + +A front end never constructs `OperatorNode`s directly. It builds a name-based tree in +`crates/frontend-common` — `UnresolvedOp` / `UnresolvedScalar`, a mirror of `NonASAPOp` / +`ScalarExpr` in which every column reference is a `ColumnRef` and a PromQL `Scan` has no schema +yet — and calls `asap_frontend_common::resolve_root`, which does three things in order: + +1. **Resolution** — a bottom-up walk that binds every `ColumnRef` to a positional `ColumnId` + against the derived schema of the already-resolved child. A schemaless (PromQL) leaf gets + its binding schema from `SchemaResolver`, built from the names the query references. + `Join` / `SetOp` sides and the operators referenced from scalar positions are each bound as + a root in their own scope; a `BinaryOp` side additionally inherits the label names its + enclosing scope references. +2. **Schema derivation** — each `OperatorNode::new` derives the node's output schema and + result kind from the operator and its children. +3. **Canonicalization** — `asap_types::ir::canonicalize::canonicalize` erases structural + differences between semantically identical queries: it promotes an additive + `Limit { Sort { Aggregate } }` ranking to the `AggIntent::TopK` heavy-hitter shape, and + lowers `EXISTS` / `NOT EXISTS` / `IN (subquery)` predicates to `Join { Semi | Anti }` and a + scalar subquery to a `Join { Cross }` plus column reference. The pass is idempotent and + keeps the pointer identity of every untouched sub-DAG. + +The result is `Rc`. `lower_promql_workload`, `lower_sql` / `lower_sql_dialect` / +`lower_sql_batch` and `lower_metricsql` all return it. + +Workload search then runs structural CSE (`asap_types::ir::cse::share_common_sub_dags`) once +across every root: bottom-up hash-consing where the structural hash is only a filter and the +typed `PartialEq` decides sharing, following scalar references like any other input, and +gated by `Schema::has_unique_key()` for non-ASAP producers. ## Fields and column references @@ -31,27 +167,28 @@ to one source language. - [`Aggregate`](#aggregate) — collapses input rows into fewer output rows via a reduction and aggregate intents. **[Time-related nodes](#time-related-nodes)** -- [`TimeRange`](#timerange) — a range-vector lookback over the time axis (PromQL `[5m]`). +- [`TimeRange`](#timerange) — temporal selection over a time-series input (PromQL instant lookback or `[5m]` range selector). - [`TimeShift`](#timeshift) — shifts *when* a selector is evaluated (PromQL `offset`/`@`). - [`PromqlSubquery`](#promqlsubquery) — re-evaluates an instant-vector expression over a range at a given step. **[Relational nodes](#relational-nodes)** — common to both SQL and PromQL - [`Scan`](#scan) — identifies the logical data source. +- [`Values`](#values) — SQL `VALUES` rows, or the one empty row of a `SELECT` without `FROM`. - [`Filter`](#filter) — restricts rows using a predicate. - [`Project`](#project) — column projection (SQL `SELECT` list). -- [`BinaryOp`](#binaryop) — arithmetic / comparison / boolean composition of two inputs. +- [`BinaryOp`](#binaryop) — arithmetic / comparison / set composition of two inputs. - [`Sort`](#sort) — generic (non-heavy-hitter) order-by, optionally per-group. -- [`Limit`](#limit) — caps the row count, with an offset. +- [`Limit`](#limit) — caps the row count, with an offset, optionally per-group. - [`Dedup`](#dedup) — row-level deduplication. - [`Join`](#join) — logical join of two inputs. - [`SetOp`](#setop) — SQL's typed set operations (`UNION`/`INTERSECT`/`EXCEPT`). - [`Concat`](#concat) — exact, untyped `UNION ALL` of union-compatible branches. -**[PromQL-specific nodes](#promql-specific-nodes)** -- [`PromqlScalarBridge`](#promqlscalarbridge) — a scalar sub-expression at an operator-DAG position. -- [`EvalTimestamp`](#evaltimestamp) — the query evaluation time as a scalar (PromQL `time()`). +**[Scalar-position nodes](#scalar-position-nodes)** +- `QueryRoot::Scalar` — a standalone scalar expression, without an operator node. - [`PromqlVectorFromScalar`](#promqlvectorfromscalar) — promotes a scalar to a label-less instant vector. -- [`PromqlScalarFromVector`](#promqlscalarfromvector) — collapses a single-series vector to a scalar. + +**[PromQL-specific nodes](#promql-specific-nodes)** - [`PromqlRelabel`](#promqlrelabel) — per-series label rewrite (PromQL `label_replace`/`label_join`). - [`PromqlInfoEnrich`](#promqlinfoenrich) — left-join label enrichment from an info metric. - [`PromqlSeriesSample`](#promqlseriessample) — keeps a subset of whole series, not a reduction. @@ -59,6 +196,9 @@ to one source language. **[SQL-specific nodes](#sql-specific-nodes)** - [`SQLWindowFunc`](#sqlwindowfunc) — SQL analytic window function (`OVER (...)`). +PromQL `time()` and `scalar(v)` are scalar expressions (`ScalarExpr::EvalTimestamp`, +`ScalarExpr::PromqlScalarFromVector`), not nodes. + ## Aggregation-related nodes ### Aggregate @@ -93,7 +233,7 @@ list of aggregate intents (`measures`). value is still recomputed by the agg intent, e.g. `Rate`), for a computation with no `by(...)` clause to attach to. `PerEntity` is different from `by` for all columns, because in PromQL, it is schemaless and you don't know all columns beforehand. E.g. PromQL `rate(http_requests_total[5m])`, which has one rate value - per input series: + per input series: ```text Aggregate( @@ -101,7 +241,7 @@ list of aggregate intents (`measures`). measures = [Rate], output_names = [], having = None, - child = TimeRange(range = 5m, child = Scan("http_requests_total")) + child = TimeRange(range = 5m, kind = Range, child = Scan("http_requests_total")) ) ``` @@ -208,7 +348,7 @@ Example for `filters`: `count(CASE WHEN p THEN x END)` (`p`, plus `x IS NOT NULL` when `x` is nullable), and from `count(expr)` over any other nullable `expr` (`expr IS NOT NULL`), because canonical `Count` counts rows and never consults its argument. A filtered measure has no summary binding yet: - `asap-aware-mapping` keeps such an `Aggregate` as `KeepPreAsap`, and canonicalization does + `asap-aware-mapping` retains such an `Aggregate` as an ordinary exact sub-DAG, and canonicalization does not promote a filtered count ranking to a heavy-hitter `TopK`. Example for `having`: @@ -232,8 +372,8 @@ Example for `having`: **Rules/Invariants**: A filtering predicate will be passed to at the lowest node (closer to the leaves) in the AST/DAG that can express it — `Scan.predicates`, then `Aggregate.having`, then `Filter` as the fallback — so its constraint is visible at - the node it actually applies to, not behind an opaque wrapper, once pre-ASAP IR translates - to post-ASAP IR with summary binding. The upper nodes (closer to the root) in the AST/DAG can still have a `Filter` node with the same condition. This intentional duplication is for Summary related translation and optimizations. + the node it actually applies to, not behind an opaque wrapper, once summary binding reads it. + The upper nodes (closer to the root) in the AST/DAG can still have a `Filter` node with the same condition. This intentional duplication is for Summary related translation and optimizations. For example, `SELECT srcip, COUNT(*) AS cnt FROM packets GROUP BY srcip HAVING COUNT(*) > 10` pins `cnt > 10` to the lowest node that can express it, `Aggregate.having`: @@ -266,7 +406,7 @@ Example for `having`: Both are valid at once, and neither is derived from the other: `having` is the canonical spot a summary-aware pass reads to decide whether `Aggregate` can bind to a summary, while the outer `Filter` is what a plain logical evaluator runs without knowing `having` exists. The duplication is forward-looking groundwork for - once HAVING-aware summary binding (pre-ASAP-IR to post-ASAP-IR translation) lands. + once HAVING-aware summary binding lands. Neither direction of that push-down is enforced yet: the SQL front end doesn't populate `having` from a real `HAVING` clause (#201), and canonicalization doesn't fold an existing @@ -278,14 +418,21 @@ Example for `having`: ### TimeRange -Represents a range of time. Kept different from `Filter` to treat time as an explicit concern. +Temporal selection over a time-series input. Kept different from `Filter` to treat time as an +explicit concern. `kind` records which samples a PromQL selector reads: + +- `TimeRangeKind::Instant` — an instant selector: `range` is the lookback horizon and the + latest eligible sample per series is selected (the planner injects the declared + `data_ingestion_interval` around a bare selector). +- `TimeRangeKind::Range` — a range selector (`m[5m]`): every sample in the window. ```promql rate(http_requests_total[5m]) ``` **Fields:** -- `range` — how far back to look (the PromQL `[5m]` duration). +- `range` — how far back to look (the PromQL `[5m]` duration, or the instant lookback). +- `kind` — `Instant` or `Range`. - `child` — the input the range applies to. ### TimeShift @@ -334,9 +481,19 @@ the same logical data domain. **Fields:** - `source` — the logical data source (a table name or PromQL metric selector). - `predicates` — row-level filters pushed all the way down to this scan (Rules/Invariants - rule 1); enforced structurally at lowering time — a `Filter` directly over a `Scan` never - survives. -- `schema` — the binding schema every positional column reference in the DAG resolves against. + rule 1): PromQL label matchers and pushed-down `WHERE` conjuncts. +- `schema` — the binding schema every positional column reference in the tree resolves against. + A catalog-backed SQL leaf carries its catalog schema; a PromQL leaf carries the usage-derived + schema `SchemaResolver` built from the labels the query references. + +### Values + +SQL `VALUES` rows, or the one empty row of a `SELECT` without `FROM` +(`SELECT 1 + 1`). Row expressions have no input-column scope. + +**Fields:** +- `rows` — one `Vec` per row. +- `schema` — the output schema of the rows. ### Filter @@ -370,6 +527,10 @@ that's neither a base scan column nor an aggregate output: SELECT * FROM (SELECT srcip, bytes_in + bytes_out AS total FROM packets) t WHERE total > 500 ``` +A `Filter` whose predicate contains `EXISTS` / `NOT EXISTS` / `IN (subquery)` does not +survive canonicalization: the conjunct becomes a `Join { Semi | Anti }` under the remaining +predicate. + **Fields:** - `pred` — the row-level predicate to apply. - `child` — the input being filtered. @@ -390,18 +551,21 @@ SELECT srcip, dstip FROM packets ### BinaryOp -Arithmetic / comparison / boolean composition. PromQL binary operators between two vectors, -a vector and a scalar, or two scalars. +Arithmetic / comparison / set composition of two operands. PromQL binary operators between two vectors, +two vectors. Mixed vector/scalar arithmetic uses `Project`; non-bool comparison uses `Filter`. Standalone scalar expressions are `QueryRoot::Scalar`. ```promql up > 1 ``` **Fields:** -- `op` — the arithmetic/comparison/boolean operator. +- `operator` — a `BinaryOperator { kind, vector_match, checked_relative_division, checked_finite_division }`: + - `kind` — `BinaryOpKind::Arithmetic(..)`, `Compare(..)` or `Set(..)` (PromQL `and`/`or`/`unless`). + - `vector_match` — PromQL vector-matching modifiers (`on`/`ignoring`, `group_left`/`group_right`); `None` outside PromQL and the only supported value today. + - `checked_relative_division` / `checked_finite_division` — typed division guards set by summary planning, never by a front end (see [physical-plan integration](../design_docs/architecture/physical-plan-integration.md#conditional-temporal-average-lowering)). +- `return_bool` — the PromQL `bool` modifier: a comparison returns `0`/`1` instead of filtering. Valid only for comparison operators. - `lhs` — the left operand. - `rhs` — the right operand. -- `vector_match` — PromQL vector-matching modifiers (`on`/`ignoring`, `group_left`/`group_right`); `None` outside PromQL. ### Sort @@ -413,7 +577,7 @@ sort_desc(up) ``` **Fields:** -- `keys` — the ordering columns/expressions and direction. +- `keys` — the ordering expressions and direction (`SortKey`). - `partition_by` — grouping keys that make the ordering per-group instead of global; empty = a single global order. - `child` — the input being ordered. @@ -427,8 +591,9 @@ topk(3, up) ``` **Fields:** -- `n` — the maximum number of rows to keep. +- `n` — the maximum number of rows to keep; `None` is offset-only. - `offset` — how many leading rows to skip first. +- `partition_by` — applies the limit per group (PromQL `topk by (..)`); empty = global. - `child` — the input being capped. ### Dedup @@ -447,14 +612,16 @@ SELECT DISTINCT srcip, dstip FROM packets ### Join -Logical join; the physical strategy (hash/merge/broadcast) is picked in the post-ASAP IR. SQL `JOIN`. +Logical join; the physical strategy (hash/merge/broadcast) is picked downstream of the planner. SQL `JOIN`, +and the shape canonicalization lowers subqueries to. ```sql SELECT u.prefix FROM bgp_updates u JOIN bgp_rib_state r ON u.prefix = r.prefix ``` **Fields:** -- `kind` — the join type (inner/left/right/full/semi/anti). +- `kind` — the join type (`Inner`/`Left`/`Right`/`Full`/`Cross`/`Semi`/`Anti`). A semi/anti join + outputs the left input's columns alone, but its predicate resolves against `left ++ right`. - `pred` — the join condition. - `left` — the left input. - `right` — the right input. @@ -480,7 +647,7 @@ SELECT srcip FROM packets UNION ALL SELECT dstip FROM packets never dedup. Used when a single `Aggregate` can't express the shape — the canonical case is PromQL `histogram_quantiles` (one branch per φ, each its own `HistogramQuantile` reduction relabeled with its `le` value) — and SQL `ROLLUP`/`CUBE`/`GROUPING SETS` (one branch per -grouping level). +grouping level). The output schema is the first child's. ```promql histogram_quantiles(rate(http_request_duration_seconds_bucket[5m]), "le", 0.5, 0.9) @@ -488,39 +655,32 @@ histogram_quantiles(rate(http_request_duration_seconds_bucket[5m]), "le", 0.5, 0 **Fields:** - `children` — the union-compatible branches to concatenate; must be non-empty. +- `discriminator_unique_key` — an optional caller-proven compound unique key + `(discriminator, inner_key)` over the output; nothing verifies the claim. -## PromQL-specific nodes - -### PromqlScalarBridge +## Scalar-position nodes -A scalar sub-expression (issue #220: in practice always `Literal(ScalarValue::Float64(_))` — -a PromQL number literal, or a folded constant scalar expression) sitting at an **operator-DAG -position** — a `BinaryOp` operand for ` op ` thresholds and unit conversions, -a `PromqlVectorFromScalar` child, or a whole query's root. This wrapper is what marks the -position; it no longer duplicates `Literal`'s value the way the old `PromqlScalar(f64)` variant -did. - -```promql -up > 1 -``` +### Scalar query roots -**Fields:** a single unnamed child `QueryExpr` — the wrapped scalar sub-expression. +`QueryRoot` distinguishes an operator result from an owned `ScalarExpr`. It is +an API root discriminator, not an operator. `2`, `time()`, and +`scalar(sum(up)) + 1` therefore introduce no constant-wrapper nodes. -### EvalTimestamp +Use `lower_promql_query_workload` for mixed scalar/vector workloads. The +operator-only convenience API rejects standalone scalar roots. `ParsedWorkload` +retains each scalar's workload index; `PlanOutput::roots()` returns all results +in workload order. Scalar plan reads remain exact and retain their operator +references; summary selection currently operates on operator roots. -The query **evaluation timestamp** as Unix seconds — PromQL `time()` — and the implicit -input of the no-argument calendar functions (`hour()`, `day_of_week()`, ...). It is the -instant or range-step at which the expression is evaluated, not inherently the current -wall-clock time. The Prometheus instant-query HTTP API separately defaults an omitted -`time` request parameter to the server's current time. - -```promql -time() -``` +`up * 2` projects the sample expression while retaining time and full series +identity, removing the metric name. `up > 0` and `0 < up` filter the vector and +retain its sample and name. `up > bool 0` projects a zero-or-one `Case`. +Open label schemas acquire a full runtime series-identity field before this +lowering. The runtime must populate that field with all labels. ### PromqlVectorFromScalar -The scalar→instant-vector bridge — PromQL `vector(s)`. Promotes a scalar-typed child to a +The scalar→instant-vector bridge — PromQL `vector(s)`. Promotes a scalar expression to a single label-less series carrying that value at every step, e.g. for dead-man's-switch patterns (`up or vector(0)`). @@ -528,18 +688,9 @@ patterns (`up or vector(0)`). vector(1) ``` -**Fields:** a single unnamed child `QueryExpr` — the scalar-typed expression being promoted to a vector. - -### PromqlScalarFromVector +**Fields:** a single unnamed `ScalarExpr` — the scalar being promoted to a vector. -The instant-vector→scalar bridge — PromQL `scalar(v)`. Collapses a single-element vector to -its value (NaN at runtime if the input isn't exactly one series). - -```promql -scalar(up) -``` - -**Fields:** a single unnamed child `QueryExpr` — the single-series vector being collapsed to a scalar. +## PromQL-specific nodes ### PromqlRelabel @@ -600,5 +751,6 @@ SELECT srcip, LAG(time) OVER (PARTITION BY srcip ORDER BY time) FROM packets - `args` — the function's operand expressions; empty for rank-only functions. - `partition_by` — grouping keys the window is computed within. - `order_by` — the ordering the window function reads. +- `frame` — the optional window frame. - `output_name` — the name of the new output column. - `child` — the input the window function is computed over. diff --git a/docs/develop_docs/target-candidate-api-migration.md b/docs/develop_docs/target-candidate-api-migration.md index a5b081a61..19e4a13b7 100644 --- a/docs/develop_docs/target-candidate-api-migration.md +++ b/docs/develop_docs/target-candidate-api-migration.md @@ -13,7 +13,7 @@ are unchanged. #453 separately defines the integration API surface. | `MaterializeSummaryMaintenanceLifecycleError` | `SummaryMaintenanceLifecycleAssemblyError` | Failure assembling a DAG or deriving maintenance decisions | | Error variant `Materialize` | `AssembleDAG` | Wrap an underlying `RealizationError` from DAG assembly | | Internal `materialize_inner` / `materialize_residual` | `assemble_target` / `assemble_residual` | Assemble selected nodes, not runtime materialized views | -| Internal assembly cache `materialized` | `assembled_nodes` | Preserve shared `Rc` identity | +| Internal assembly cache `materialized` | `assembled_nodes` | Preserve shared node identity (now `Rc`, see below) | Update imports and calls together; old public names are not retained as aliases. Downstream Rust integrations using these symbols must migrate. No serialized @@ -26,3 +26,26 @@ counterpart) are prerequisites, not additional changes here. The workflow remains one selection call per workload followed by one assembly call per query root. `SummaryMaintenanceLifecyclePlan` contains the assembled Post-ASAP DAG root plus maintenance decisions; it is not an executable plan. + +## Later: unified operator IR (operator flattening) + +The pre-ASAP and post-ASAP trees became one IR in `asap_types::ir`. Every +node is an `Rc` whose `operator` is `Operator::NonASAP(NonASAPOp)` +or `Operator::ASAP(ASAPOp)`. Old public names are not kept as aliases. + +| Old | New | +|---|---| +| `Rc` (pre-ASAP) | `Rc` holding `Operator::NonASAP(NonASAPOp)` | +| `Rc` / `SummaryExpr` (post-ASAP) | The same `Rc`; summary steps are `Operator::ASAP(ASAPOp)` | +| `SummaryExpr::KeepPreAsap(q)` | The non-ASAP sub-DAG itself; `retain_exact` only adds an exact `guarantee` | +| `SummaryExpr::ValueOperation { .. }` over an evaluation | An ordinary `NonASAPOp` (`Project`, `Filter`, `Sort`, `Limit`, `Aggregate`) reading an ASAP node; `FinalizeExactAccumulator`, `MaintainPopulation`, `EvaluatePopulation` are `ASAPOp` variants | +| `Replacement::Summary(..)` / `Replacement::Rewrite(..)` | `Replacement::SubDAG(Rc)`; `is_logical_rewrite` tells them apart | +| `SummaryFamilyType` | `FieldDataType` (its non-`Plain` variants) | +| Timing stored on post-ASAP nodes | `OperatorNode::timing`, `None` until `ir::timing::apply_lifecycle_timings` writes it from a `LifecycleAssignment` | +| `UnresolvedQueryExpr` + `asap_types::pre_asap::resolve_root` | `UnresolvedOp` / `UnresolvedScalar` + `asap_frontend_common::resolve_root` | +| `pre_asap::canonicalize`, `pre_asap::cse::share_common_sub_dags` | `ir::canonicalize::canonicalize`, `ir::cse::share_common_sub_dags` | +| `asap_types::post_asap::compile_post_asap_dag` (wire version 5, `Fallback`/`Binary`/`Value` payloads) | `asap_types::ir::physical_export::compile_physical_asap_dag` (`PhysicalASAPDAG`, wire version `PHYSICAL_ASAP_DAG_WIRE_VERSION`: one node per operator, `Operator` payloads, `ScalarRef` edges); input must be timed | +| Exported schema JSON `columns` | `fields` | + +Field and schema details: [Pre-ASAP IR](pre-asap-ir.md) and +[Post-ASAP IR](../design_docs/concepts/post-asap-ir.md). diff --git a/docs/user_guide_docs/run-a-query.md b/docs/user_guide_docs/run-a-query.md index 3f5825c29..2f7325cc3 100644 --- a/docs/user_guide_docs/run-a-query.md +++ b/docs/user_guide_docs/run-a-query.md @@ -109,8 +109,10 @@ whether to select them using its own evidence. Planner's automatic `global_selection` skips them; their presence alone does not show that they meet the requested target. -Each input line is followed by its debug IR or an `ERR:` message. Post-ASAP -output may contain summary state, readouts or exact `KeepPreAsap` work. An +Each input line is followed by its debug IR or an `ERR:` message. Pre-ASAP and +Post-ASAP output use the same node format: Post-ASAP output adds summary nodes +(state, readouts) and keeps the original exact operators wherever no summary +replaces them. An approximate target permits approximation; it does not guarantee a legal or certified sketch. The tool prints plans, not query results. diff --git a/tools/dag-viewer/README.md b/tools/dag-viewer/README.md index b90aae272..8c55aef56 100644 --- a/tools/dag-viewer/README.md +++ b/tools/dag-viewer/README.md @@ -69,7 +69,8 @@ cargo run -p asap-devtools --bin dag_export -- \ Load the JSON with the page's file picker. `--planner-cost-json` is a complete physical-evidence document: an immutable `evidence_version`, calibration, and -target records containing the exact target `QueryExpr` and comparison scope. +target records containing the exact target node (a serialized pre-ASAP +`OperatorNode`) and comparison scope. Each exact replacement candidate owns its complete logical-node `PhysicalNodeEvidence`; summary candidates additionally own their bound `PhysicalDAG`. Candidate-local evidence prevents statistics for one physical @@ -129,7 +130,7 @@ a selected replacement directly contains: { "decision": { "id": 7, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, "cost": 1.14001088, @@ -147,8 +148,13 @@ The exporter assigns `workload_node_id`; union rendering reads that mapping directly. Node boxes use concrete IR fields: aggregate measures/grouping, sort keys, -filter predicates, projections, sources, summary families, and readout -queries. Category icons are deliberately omitted so they cannot be confused +filter predicates, projections, sources, summary families, and evaluation +queries. A node's `kind` is the operator variant name (`Operator::kind_name`): +a `NonASAPOp` such as `Aggregate` or `Values`, or an `ASAPOp` such as +`SummaryAgg` or `EvaluatePopulation`. `node-style.js` maps each kind to a color +category. Scalar expressions are not nodes; an operator a scalar expression +reads (`scalar(v)`, `EXISTS (subquery)`) is a child node, shown in `detail` +as `{"scalar_ref": }`. Schemas list their entries under `fields`. Category icons are deliberately omitted so they cannot be confused with IR text. ### Cost/benefit annotations (issue #286) diff --git a/tools/dag-viewer/dag.example.json b/tools/dag-viewer/dag.example.json index 85c0e97b6..c489f7b78 100644 --- a/tools/dag-viewer/dag.example.json +++ b/tools/dag-viewer/dag.example.json @@ -13,33 +13,43 @@ "predicates": [], "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -56,33 +66,43 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -93,7 +113,7 @@ }, "children": [], "workload_node_id": 0, - "hash": 2606922452740434172 + "hash": 5992903821157128693 }, { "id": 1, @@ -110,7 +130,7 @@ } ], "output_names": [ - "count(*)" + "count(Int64(1))" ], "reduction": { "Reduce": [ @@ -120,16 +140,20 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "int64", - "name": "count(*)", + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", "nullable": false, "table": null } @@ -145,11 +169,11 @@ 0 ], "workload_node_id": 1, - "hash": 12396162568747900801, + "hash": 2322418053818133469, "notes": [ { "kind": "SketchApproximation", - "reason": "count realizes as a Cms sketch \u2014 one of summary_candidates' alternatives for this intent (asap_aware_mapping::replacement::implementations_for_with); count realizes as a CountSketch sketch \u2014 one of summary_candidates' alternatives for this intent (asap_aware_mapping::replacement::implementations_for_with); count realizes as a shared HydraCms structure over Cms serving every subpopulation of this grouped aggregate, instead of one Cms instance per distinct `by` key \u2014 legal because this aggregate has a non-empty subpopulation concept and Cms has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's; count realizes as a shared HydraCountSketch structure over CountSketch serving every subpopulation of this grouped aggregate, instead of one CountSketch instance per distinct `by` key \u2014 legal because this aggregate has a non-empty subpopulation concept and CountSketch has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's" + "reason": "count realizes as a Cms sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a CountSketch sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a shared HydraCms structure over Cms serving every subpopulation of this grouped aggregate, instead of one Cms instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and Cms has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's; count realizes as a shared HydraCountSketch structure over CountSketch serving every subpopulation of this grouped aggregate, instead of one CountSketch instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and CountSketch has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's" } ] }, @@ -166,7 +190,7 @@ } }, { - "alias": null, + "alias": "count(*)", "expr": { "Column": 1 } @@ -176,15 +200,19 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": null }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "count(*)", "nullable": false, "table": null @@ -201,7 +229,7 @@ 1 ], "workload_node_id": 2, - "hash": 17063396757587155626 + "hash": 13155751879062808849 } ], "root": 2 @@ -210,10 +238,10 @@ { "decision_id": 0, "target_pre_id": 1, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, - "cost": 1.14001088, + "cost": null, "before": { "nodes": [ { @@ -224,33 +252,43 @@ "predicates": [], "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -267,33 +305,43 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -303,7 +351,7 @@ "unique_keys": [] }, "children": [], - "hash": 2606922452740434172 + "hash": 5992903821157128693 }, { "id": 1, @@ -320,7 +368,7 @@ } ], "output_names": [ - "count(*)" + "count(Int64(1))" ], "reduction": { "Reduce": [ @@ -330,16 +378,20 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "int64", - "name": "count(*)", + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", "nullable": false, "table": null } @@ -354,7 +406,7 @@ "children": [ 0 ], - "hash": 12396162568747900801 + "hash": 2322418053818133469 } ], "root": 1 @@ -365,202 +417,311 @@ "nodes": [ { "id": 0, - "kind": "KeepPreAsap", - "label": "KeepPreAsap(Scan)", + "kind": "Scan", + "label": "Scan(metrics)", "detail": { - "pre_asap_sub_dag": { - "nodes": [ - { - "children": [], - "detail": { - "predicates": [], - "schema": { - "closed": true, - "columns": [ - { - "dtype": "timestamp", - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "float64", - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "int64", - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - }, - "source": { - "Table": { - "table_ref": "metrics" - } - } + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" }, - "hash": 2606922452740434172, - "id": 0, - "kind": "Scan", - "label": "Scan(metrics)", - "schema": { - "closed": true, - "columns": [ - { - "dtype": "timestamp", - "name": "ts", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "service", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "utf8", - "name": "region", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "float64", - "name": "latency", - "nullable": false, - "table": "metrics" - }, - { - "dtype": "int64", - "name": "bytes", - "nullable": false, - "table": "metrics" - } - ], - "time_index": 0, - "unique_keys": [] - } + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" } ], - "root": 0 + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } } }, - "children": [] + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 6315306962080483567 }, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(Sketch(Cms))", "detail": { - "col": "SampleValue", "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + }, + "weight_domain": { + "kind": "non_negative", + "proof": "unit_count" + } + }, "reduction": { "Reduce": [ 1 ] } }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "Cms", + "category": "Frequency", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, "children": [ 0 - ] + ], + "hash": 7319371269782491912 }, { "id": 2, "kind": "SummaryEstimate", "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", "detail": { + "guarantee": { + "bound": { + "op": "constant", + "value": 0.009993683192864136 + }, + "failure_probability": { + "op": "constant", + "value": 0.006737946999085467 + }, + "metric": "frequency", + "provenance": [ + { + "algorithm": "Cms", + "contract": "count_min_l1_markov_v1", + "kind": "sketch_evaluation", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + }, + "query": "PointCount { key: SampleValue, value: None }" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "approximate_aggregate" + }, + "rule": "exact_input" + }, + { + "kind": "accuracy_target", + "target": { + "Epsilon": 0.01 + } + } + ] + }, "query": "PointCount { key: SampleValue, value: None }" }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, "children": [ 1 - ] + ], + "hash": 11635781505417422576 } ], "root": 2 } }, "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } ], @@ -571,36 +732,61 @@ "kind": "Scan", "label": "Scan(metrics)", "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, "predicates": [], "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -617,33 +803,43 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "timestamp", + "dtype": { + "Plain": "timestamp" + }, "name": "ts", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": "metrics" }, { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "region", "nullable": false, "table": "metrics" }, { - "dtype": "float64", + "dtype": { + "Plain": "float64" + }, "name": "latency", "nullable": false, "table": "metrics" }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "bytes", "nullable": false, "table": "metrics" @@ -654,77 +850,28 @@ }, "children": [], "workload_node_id": 0, - "hash": 2606922452740434172, + "hash": 6315306962080483567, "decision": { "id": 0, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, - "cost": 1.14001088, + "cost": null, "role": "replacement_region", "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } }, @@ -733,9 +880,20 @@ "kind": "SummaryAgg", "label": "SummaryAgg(Sketch(Cms))", "detail": { - "col": "SampleValue", "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + }, + "weight_domain": { + "kind": "non_negative", + "proof": "unit_count" + } + }, "reduction": { "Reduce": [ 1 @@ -743,94 +901,70 @@ } }, "schema": { + "closed": true, "fields": [ { - "dtype": "Plain(Utf8)", + "dtype": { + "Plain": "utf8" + }, "name": "service", - "nullable": false + "nullable": false, + "table": "metrics" }, { - "dtype": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", - "name": "count(*)", - "nullable": false + "dtype": { + "Sketch": [ + { + "algorithm": "Cms", + "category": "Frequency", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null } ], - "time_index": null + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] }, "children": [ 0 ], "workload_node_id": 1, + "hash": 7319371269782491912, "decision": { "id": 0, - "strategy": "SketchAlgorithmStrategy", + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, - "cost": 1.14001088, + "cost": null, "role": "replacement_region", "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } }, @@ -839,102 +973,6987 @@ "kind": "SummaryEstimate", "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", "detail": { + "guarantee": { + "bound": { + "op": "constant", + "value": 0.009993683192864136 + }, + "failure_probability": { + "op": "constant", + "value": 0.006737946999085467 + }, + "metric": "frequency", + "provenance": [ + { + "algorithm": "Cms", + "contract": "count_min_l1_markov_v1", + "kind": "sketch_evaluation", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + }, + "query": "PointCount { key: SampleValue, value: None }" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "approximate_aggregate" + }, + "rule": "exact_input" + }, + { + "kind": "accuracy_target", + "target": { + "Epsilon": 0.01 + } + } + ] + }, "query": "PointCount { key: SampleValue, value: None }" }, "schema": { + "closed": true, "fields": [ { - "dtype": "Plain(Utf8)", + "dtype": { + "Plain": "utf8" + }, "name": "service", - "nullable": false + "nullable": false, + "table": "metrics" }, { - "dtype": "Plain(Int64)", + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 1 + ], + "workload_node_id": 2, + "hash": 11635781505417422576, + "decision": { + "id": 0, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 3, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "count(*)", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" + }, "name": "count(*)", - "nullable": false + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2 + ], + "workload_node_id": 3, + "hash": 13155751879062808849 + } + ], + "root": 3 + }, + "workload_cost": { + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "name": "q2", + "source": "SELECT service, AVG(latency) FROM metrics GROUP BY service", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 0, + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "col": 3, + "kind": "avg" + } + ], + "output_names": [ + "avg(metrics.latency)" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "workload_node_id": 3, + "hash": 15432500422583384542 + }, + { + "id": 2, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 1 + ], + "workload_node_id": 4, + "hash": 17963877225883903375 + } + ], + "root": 2 + }, + "replacements": [ + { + "decision_id": 1, + "target_pre_id": 1, + "strategy": "SemanticEquivalentRewriteStrategy", + "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "col": 3, + "kind": "avg" + } + ], + "output_names": [ + "avg(metrics.latency)" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "hash": 15432500422583384542 + } + ], + "root": 1 + }, + "after": { + "kind": "Rewrite", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "col": 3, + "kind": "sum" + } + ], + "output_names": [], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "hash": 310390602443712772 + }, + { + "id": 2, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "avg(metrics.latency)", + "expr": { + "Cast": { + "expr": { + "Column": 1 + }, + "to": "float64", + "try_cast": false + } + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 1 + ], + "hash": 18059109450340713504 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": "Exact", + "kind": "count" + } + ], + "output_names": [], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "hash": 10926873078186554083 + }, + { + "id": 4, + "kind": "BinaryOp", + "label": "BinaryOp(/)", + "detail": { + "checked_finite_division": false, + "checked_relative_division": false, + "op": "/", + "return_bool": false, + "vector_match": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2, + 3 + ], + "hash": 1089002885905118593 + } + ], + "root": 4 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + ], + "post_dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 0, + "hash": 6315306962080483567, + "decision": { + "id": 7, + "strategy": "ASAPStrategies", + "rationale": "Sum { col: Some(3) } realizes as an exact Sum accumulator", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 1, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Sum))", + "detail": { + "family": "ExactAggregate(Sum, Sum)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Sum)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "exact_sum" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": { + "Qualified": { + "name": "latency", + "table": "metrics" + } + } + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Sum", + "Sum" + ] + }, + "name": "sum", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "workload_node_id": 4, + "hash": 2701552184495960072, + "decision": { + "id": 7, + "strategy": "ASAPStrategies", + "rationale": "Sum { col: Some(3) } realizes as an exact Sum accumulator", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 2, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "avg(metrics.latency)", + "expr": { + "Cast": { + "expr": { + "Column": 1 + }, + "to": "float64", + "try_cast": false + } + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 1 + ], + "workload_node_id": 5, + "hash": 18059109450340713504, + "decision": { + "id": 1, + "strategy": "SemanticEquivalentRewriteStrategy", + "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 3, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Count))", + "detail": { + "family": "ExactAggregate(Count, Count)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Count): row count is independent of input values" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "ExactAggregate": [ + "Count", + "Count" + ] + }, + "name": "count", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 0 + ], + "workload_node_id": 6, + "hash": 14727181282387726522, + "decision": { + "id": 8, + "strategy": "ASAPStrategies", + "rationale": "count realizes as an exact Count accumulator", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 4, + "kind": "BinaryOp", + "label": "BinaryOp(/)", + "detail": { + "checked_finite_division": false, + "checked_relative_division": false, + "op": "/", + "return_bool": false, + "vector_match": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2, + 3 + ], + "workload_node_id": 7, + "hash": 1089002885905118593, + "decision": { + "id": 1, + "strategy": "SemanticEquivalentRewriteStrategy", + "rationale": "Composes compatible nested aggregates using their declared algebraic intent while preserving the output schema.", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 5, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": null, + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "avg(metrics.latency)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 4 + ], + "workload_node_id": 8, + "hash": 17963877225883903375 + } + ], + "root": 5 + }, + "workload_cost": { + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "name": "q3", + "source": "topk(5, rate(http_requests_total[5m]))", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 5, + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "workload_node_id": 6, + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "workload_node_id": 7, + "hash": 977274539382465184 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "k": 5, + "kind": "top_k" + } + ], + "output_names": [], + "reduction": { + "Reduce": [] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_5", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [ + 2 + ], + "workload_node_id": 8, + "hash": 13389420092441124844 + } + ], + "root": 3 + }, + "replacements": [ + { + "decision_id": 3, + "target_pre_id": 2, + "strategy": "ASAPStrategies", + "rationale": "Rate realizes as an exact Rate accumulator", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 977274539382465184 + } + ], + "root": 2 + }, + "after": { + "kind": "Summary", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 4252745894123182058 + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 1949434180762498567 + } + ], + "root": 2 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + }, + { + "decision_id": 2, + "target_pre_id": 3, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 977274539382465184 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "k": 5, + "kind": "top_k" + } + ], + "output_names": [], + "reduction": { + "Reduce": [] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_5", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [ + 2 + ], + "hash": 13389420092441124844 + } + ], + "root": 3 + }, + "after": { + "kind": "Summary", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 4252745894123182058 + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 1949434180762498567 + }, + { + "id": 3, + "kind": "FinalizeExactAccumulator", + "label": "FinalizeExactAccumulator", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "timing": "query_time" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 2 + ], + "hash": 4765736981108370595 + }, + { + "id": 4, + "kind": "Sort", + "label": "Sort(1 keys)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 3 + ], + "hash": 1968578139481232113 + }, + { + "id": 5, + "kind": "Limit", + "label": "Limit(5)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "n": 5, + "offset": 0, + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 4 + ], + "hash": 18045468768442785975 + } + ], + "root": 5 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + ], + "post_dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 9, + "hash": 15884523639471678680, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "workload_node_id": 10, + "hash": 4252745894123182058, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "workload_node_id": 11, + "hash": 1949434180762498567, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 3, + "kind": "FinalizeExactAccumulator", + "label": "FinalizeExactAccumulator", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "timing": "query_time" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 2 + ], + "workload_node_id": 12, + "hash": 4765736981108370595, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 4, + "kind": "Sort", + "label": "Sort(1 keys)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 3 + ], + "workload_node_id": 13, + "hash": 1968578139481232113, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 5, + "kind": "Limit", + "label": "Limit(5)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "n": 5, + "offset": 0, + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 4 + ], + "workload_node_id": 14, + "hash": 18045468768442785975, + "decision": { + "id": 2, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + } + ], + "root": 5 + }, + "workload_cost": { + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "name": "q4", + "source": "topk(10, rate(http_requests_total[5m]))", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 5, + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "workload_node_id": 6, + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "workload_node_id": 7, + "hash": 977274539382465184 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "k": 10, + "kind": "top_k" + } + ], + "output_names": [], + "reduction": { + "Reduce": [] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [ + 2 + ], + "workload_node_id": 9, + "hash": 10517606883559557536 + } + ], + "root": 3 + }, + "replacements": [ + { + "decision_id": 3, + "target_pre_id": 2, + "strategy": "ASAPStrategies", + "rationale": "Rate realizes as an exact Rate accumulator", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 977274539382465184 + } + ], + "root": 2 + }, + "after": { + "kind": "Summary", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 4252745894123182058 + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 1949434180762498567 + } + ], + "root": 2 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + }, + { + "decision_id": 4, + "target_pre_id": 3, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 5927480761285910816 + }, + { + "id": 2, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "kind": "rate" + } + ], + "output_names": [ + "" + ], + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 977274539382465184 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "k": 10, + "kind": "top_k" + } + ], + "output_names": [], + "reduction": { + "Reduce": [] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "topk_10", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [ + 2 + ], + "hash": 10517606883559557536 + } + ], + "root": 3 + }, + "after": { + "kind": "Summary", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 15884523639471678680 + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "hash": 4252745894123182058 + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "hash": 1949434180762498567 + }, + { + "id": 3, + "kind": "FinalizeExactAccumulator", + "label": "FinalizeExactAccumulator", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "timing": "query_time" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 2 + ], + "hash": 4765736981108370595 + }, + { + "id": 4, + "kind": "Sort", + "label": "Sort(1 keys)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 3 + ], + "hash": 1968578139481232113 + }, + { + "id": 5, + "kind": "Limit", + "label": "Limit(10)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "n": 10, + "offset": 0, + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 4 + ], + "hash": 9012020501007212672 + } + ], + "root": 5 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + ], + "post_dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(http_requests_total)", + "detail": { + "predicates": [], + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "TimeSeries": { + "metric": "http_requests_total" + } + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 9, + "hash": 15884523639471678680, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 1, + "kind": "TimeRange", + "label": "TimeRange(Range, 300s)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "range", + "range": { + "nanos": 0, + "secs": 300 + } + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0 + ], + "workload_node_id": 10, + "hash": 4252745894123182058, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 2, + "kind": "SummaryAgg", + "label": "SummaryAgg(ExactAggregate(Rate))", + "detail": { + "family": "ExactAggregate(Rate, Rate)", + "grouping": "PerSubpopulationInstance", + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "input": { + "item": null, + "weight": { + "Column": "SampleValue" + }, + "weight_domain": { + "kind": "unknown_or_signed" + } + }, + "reduction": "PerEntity" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "ExactAggregate": [ + "Rate", + "Rate" + ] + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 1 + ], + "workload_node_id": 11, + "hash": 1949434180762498567, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 3, + "kind": "FinalizeExactAccumulator", + "label": "FinalizeExactAccumulator", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "timing": "query_time" + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 2 + ], + "workload_node_id": 12, + "hash": 4765736981108370595, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 4, + "kind": "Sort", + "label": "Sort(1 keys)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "keys": [ + { + "ascending": false, + "expr": { + "Column": 1 + }, + "nulls_first": false + } + ], + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 3 + ], + "workload_node_id": 13, + "hash": 1968578139481232113, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 5, + "kind": "Limit", + "label": "Limit(10)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "ExactAggregate(Rate)" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "counter_rate" + }, + "rule": "exact_input" + } + ] + }, + "n": 10, + "offset": 0, + "partition_by": [] + }, + "schema": { + "closed": false, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "value", + "nullable": false, + "table": null + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 4 + ], + "workload_node_id": 15, + "hash": 9012020501007212672, + "decision": { + "id": 4, + "strategy": "ASAPStrategies", + "rationale": "select exact Top-K from independently maintained temporal values", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + } + ], + "root": 5 + }, + "workload_cost": { + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "name": "q6", + "source": "SELECT metrics.service, COUNT(*) FROM metrics JOIN hosts ON metrics.service = hosts.service GROUP BY metrics.service", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 0, + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Scan", + "label": "Scan(hosts)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "hosts" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 10, + "hash": 14183779228549732123 + }, + { + "id": 2, + "kind": "Join", + "label": "Join(Inner)", + "detail": { + "kind": "Inner", + "pred": { + "Compare": { + "left": { + "Column": 1 + }, + "op": "Eq", + "right": { + "Column": 5 + }, + "semantics": "sql" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0, + 1 + ], + "workload_node_id": 11, + "hash": 530354748869277082 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "kind": "count" + } + ], + "output_names": [ + "count(Int64(1))" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2 + ], + "workload_node_id": 12, + "hash": 7814897957356983635, + "notes": [ + { + "kind": "SketchApproximation", + "reason": "count realizes as a Cms sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a CountSketch sketch — one of summary_candidates' candidates for this intent (asap_aware_mapping::replacement::realizations_for_intent); count realizes as a shared HydraCms structure over Cms serving every subpopulation of this grouped aggregate, instead of one Cms instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and Cms has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's; count realizes as a shared HydraCountSketch structure over CountSketch serving every subpopulation of this grouped aggregate, instead of one CountSketch instance per distinct `by` key — legal because this aggregate has a non-empty subpopulation concept and CountSketch has a modeled Hydra variant (asap_types::post_asap::hydra_kind_for); whether it's *worth* the shared/independent trade-off for the actual subpopulation cardinality is a CostModel's call, not this strategy's" + } + ] + }, + { + "id": 4, + "kind": "Project", + "label": "Project(2 cols)", + "detail": { + "cols": [ + { + "alias": null, + "expr": { + "Column": 0 + } + }, + { + "alias": "count(*)", + "expr": { + "Column": 1 + } + } + ], + "qualifier": null + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": null + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(*)", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 3 + ], + "workload_node_id": 13, + "hash": 1112846222258997472 + } + ], + "root": 4 + }, + "replacements": [ + { + "decision_id": 6, + "target_pre_id": 3, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "before": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Scan", + "label": "Scan(hosts)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "hosts" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [], + "hash": 14183779228549732123 + }, + { + "id": 2, + "kind": "Join", + "label": "Join(Inner)", + "detail": { + "kind": "Inner", + "pred": { + "Compare": { + "left": { + "Column": 1 + }, + "op": "Eq", + "right": { + "Column": 5 + }, + "semantics": "sql" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0, + 1 + ], + "hash": 530354748869277082 + }, + { + "id": 3, + "kind": "Aggregate", + "label": "Aggregate(1 measures)", + "detail": { + "having": null, + "measures": [ + { + "accuracy": { + "Epsilon": 0.01 + }, + "kind": "count" + } + ], + "output_names": [ + "count(Int64(1))" + ], + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2 + ], + "hash": 7814897957356983635 + } + ], + "root": 3 + }, + "after": { + "kind": "Summary", + "dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "hash": 5992903821157128693 + }, + { + "id": 1, + "kind": "Scan", + "label": "Scan(hosts)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "hosts" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [], + "hash": 14183779228549732123 + }, + { + "id": 2, + "kind": "Join", + "label": "Join(Inner)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "Inner", + "pred": { + "Compare": { + "left": { + "Column": 1 + }, + "op": "Eq", + "right": { + "Column": 5 + }, + "semantics": "sql" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [ + 0, + 1 + ], + "hash": 5789239540888802011 + }, + { + "id": 3, + "kind": "SummaryAgg", + "label": "SummaryAgg(Sketch(Cms))", + "detail": { + "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + }, + "weight_domain": { + "kind": "non_negative", + "proof": "unit_count" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "Cms", + "category": "Frequency", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 2 + ], + "hash": 10642821447658622813 + }, + { + "id": 4, + "kind": "SummaryEstimate", + "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", + "detail": { + "guarantee": { + "bound": { + "op": "constant", + "value": 0.009993683192864136 + }, + "failure_probability": { + "op": "constant", + "value": 0.006737946999085467 + }, + "metric": "frequency", + "provenance": [ + { + "algorithm": "Cms", + "contract": "count_min_l1_markov_v1", + "kind": "sketch_evaluation", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + }, + "query": "PointCount { key: SampleValue, value: None }" + }, + { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" + }, + { + "kind": "composition_step", + "operator": { + "op": "approximate_aggregate" + }, + "rule": "exact_input" + }, + { + "kind": "accuracy_target", + "target": { + "Epsilon": 0.01 + } + } + ] + }, + "query": "PointCount { key: SampleValue, value: None }" + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 3 + ], + "hash": 3897106577570430774 + } + ], + "root": 4 + } + }, + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + ], + "post_dag": { + "nodes": [ + { + "id": 0, + "kind": "Scan", + "label": "Scan(metrics)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "metrics" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + } + ], + "time_index": 0, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 16, + "hash": 5992903821157128693, + "decision": { + "id": 6, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 1, + "kind": "Scan", + "label": "Scan(hosts)", + "detail": { + "predicates": [], + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "source": { + "Table": { + "table_ref": "hosts" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" + } + ], + "time_index": null, + "unique_keys": [] + }, + "children": [], + "workload_node_id": 17, + "hash": 14183779228549732123, + "decision": { + "id": 6, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 2, + "kind": "Join", + "label": "Join(Inner)", + "detail": { + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "kind": "Inner", + "pred": { + "Compare": { + "left": { + "Column": 1 + }, + "op": "Eq", + "right": { + "Column": 5 + }, + "semantics": "sql" + } + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "timestamp" + }, + "name": "ts", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "float64" + }, + "name": "latency", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "bytes", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "hosts" + }, + { + "dtype": { + "Plain": "utf8" + }, + "name": "region", + "nullable": false, + "table": "hosts" } ], - "time_index": null + "time_index": 0, + "unique_keys": [] }, "children": [ + 0, 1 ], - "workload_node_id": 2, + "workload_node_id": 18, + "hash": 5789239540888802011, "decision": { - "id": 0, - "strategy": "SketchAlgorithmStrategy", + "id": 6, + "strategy": "ASAPStrategies", "rationale": "count realizes as a Cms sketch", "rank": 0, - "cost": 1.14001088, - "role": "replacement_root", + "cost": null, + "role": "replacement_region", "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + } + } + }, + { + "id": 3, + "kind": "SummaryAgg", + "label": "SummaryAgg(Sketch(Cms))", + "detail": { + "family": "Sketch(SketchKind { category: Frequency, algorithm: Cms, params: Cms { width: 272, depth: 5 } }, PerSubpopulationInstance)", + "grouping": "PerSubpopulationInstance", + "input": { + "item": { + "Column": "SampleValue" + }, + "weight": { + "Constant": 1.0 + }, + "weight_domain": { + "kind": "non_negative", + "proof": "unit_count" + } + }, + "reduction": { + "Reduce": [ + 1 + ] + } + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Sketch": [ + { + "algorithm": "Cms", + "category": "Frequency", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + } + }, + "PerSubpopulationInstance" + ] }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 ] + ] + }, + "children": [ + 2 + ], + "workload_node_id": 19, + "hash": 10642821447658622813, + "decision": { + "id": 6, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "role": "replacement_region", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "benefit": { + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" + "source": "Unavailable" + } + } + }, + { + "id": 4, + "kind": "SummaryEstimate", + "label": "SummaryEstimate(PointCount { key: SampleValue, value: None })", + "detail": { + "guarantee": { + "bound": { + "op": "constant", + "value": 0.009993683192864136 }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ + "failure_probability": { + "op": "constant", + "value": 0.006737946999085467 + }, + "metric": "frequency", + "provenance": [ + { + "algorithm": "Cms", + "contract": "count_min_l1_markov_v1", + "kind": "sketch_evaluation", + "params": { + "Cms": { + "depth": 5, + "width": 272 + } + }, + "query": "PointCount { key: SampleValue, value: None }" + }, { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" + "guarantee": { + "bound": { + "op": "zero" + }, + "failure_probability": { + "op": "zero" + }, + "metric": "absolute_value", + "provenance": [ + { + "kind": "exact", + "reason": "RetainedExact" + } + ] + }, + "input_index": 0, + "kind": "child_guarantee" }, { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" + "kind": "composition_step", + "operator": { + "op": "approximate_aggregate" + }, + "rule": "exact_input" }, { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" + "kind": "accuracy_target", + "target": { + "Epsilon": 0.01 + } } ] }, + "query": "PointCount { key: SampleValue, value: None }" + }, + "schema": { + "closed": true, + "fields": [ + { + "dtype": { + "Plain": "utf8" + }, + "name": "service", + "nullable": false, + "table": "metrics" + }, + { + "dtype": { + "Plain": "int64" + }, + "name": "count(Int64(1))", + "nullable": false, + "table": null + } + ], + "time_index": null, + "unique_keys": [ + [ + 0 + ] + ] + }, + "children": [ + 3 + ], + "workload_node_id": 20, + "hash": 3897106577570430774, + "decision": { + "id": 6, + "strategy": "ASAPStrategies", + "rationale": "count realizes as a Cms sketch", + "rank": 0, + "cost": null, + "role": "replacement_root", + "baseline_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, + "selected_cost": { + "value": null, + "unit": "CostUnits", + "source": "Unavailable" + }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } }, { - "id": 3, + "id": 5, "kind": "Project", "label": "Project(2 cols)", "detail": { @@ -946,7 +7965,7 @@ } }, { - "alias": null, + "alias": "count(*)", "expr": { "Column": 1 } @@ -956,15 +7975,19 @@ }, "schema": { "closed": true, - "columns": [ + "fields": [ { - "dtype": "utf8", + "dtype": { + "Plain": "utf8" + }, "name": "service", "nullable": false, "table": null }, { - "dtype": "int64", + "dtype": { + "Plain": "int64" + }, "name": "count(*)", "nullable": false, "table": null @@ -978,146 +8001,48 @@ ] }, "children": [ - 2 + 4 ], - "workload_node_id": 3, - "hash": 17063396757587155626 + "workload_node_id": 21, + "hash": 1112846222258997472 } ], - "root": 3 + "root": 5 }, "workload_cost": { "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } } ], "workload_cost": { "baseline_cost": { - "value": 104.0032, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 40000000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 3200000.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 640000000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "selected_cost": { - "value": 1.14001088, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "delta": 102.86318912, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1", - "inputs": [ - { - "name": "estimated_cpu_ops", - "value": 500000000.0, - "unit": "operations" - }, - { - "name": "estimated_peak_memory", - "value": 10880.0, - "unit": "bytes" - }, - { - "name": "estimated_scan", - "value": 6400000000.0, - "unit": "bytes" - } - ] + "source": "Unavailable" }, "benefit": { - "value": 102.86318912, + "value": null, "unit": "CostUnits", - "source": "Modeled", - "baseline": { - "kind": "PreAsapRecomputation" - }, - "benefit_ratio": 0.9890386941940248, - "model_version": "analytical-cost-v1+example-calibration-v1", - "evidence_version": "example-evidence-v1" + "source": "Unavailable" } } } diff --git a/tools/dag-viewer/generate-sample.sh b/tools/dag-viewer/generate-sample.sh index e916cba5f..15662d30b 100755 --- a/tools/dag-viewer/generate-sample.sh +++ b/tools/dag-viewer/generate-sample.sh @@ -6,12 +6,14 @@ set -euo pipefail cd "$(dirname "${BASH_SOURCE[0]}")/../.." # --epsilon asks for an approximate accuracy target instead of the default -# Exact, so SketchAlgorithmStrategy actually has a sketch alternative to +# Exact, so ASAPStrategies actually has a sketch alternative to # report — without it, no query below would ever pick up a `notes` badge # (see crates/devtools/src/bin/dag_export.rs's own `--epsilon` doc comment). cargo run -p asap-devtools --bin dag_export -- \ --post-asap --progress \ --epsilon 0.01 \ + --data-ingestion-interval-ms 15000 \ + --default-cost \ --sql "SELECT service, COUNT(*) FROM metrics GROUP BY service" --name q1 \ --sql "SELECT service, AVG(latency) FROM metrics GROUP BY service" --name q2 \ --promql "topk(5, rate(http_requests_total[5m]))" --name q3 \ diff --git a/tools/dag-viewer/node-style.js b/tools/dag-viewer/node-style.js index a957817c2..fee49586e 100644 --- a/tools/dag-viewer/node-style.js +++ b/tools/dag-viewer/node-style.js @@ -1,20 +1,17 @@ -// Logical QueryExpr/SummaryExpr kinds exported by +// Operator kinds (`Operator::kind_name`) exported by // crates/types/src/dag_export.rs. Categories describe the visible logical DAG // shape. They do not model hidden physical inputs: for example, // PromqlInfoEnrich is a one-child enrichment here even if physical costing // later accounts for an auxiliary source scan. const KIND_CATEGORY_JSON = `{ "Scan": "data", - "PromqlScalarBridge": "data", - "EvalTimestamp": "data", - "CurrentTimestamp": "data", + "Values": "data", "Filter": "filter", "PromqlSeriesSample": "sample", "Project": "derive", "PromqlRelabel": "derive", "PromqlInfoEnrich": "derive", "PromqlVectorFromScalar": "derive", - "PromqlScalarFromVector": "derive", "BinaryOp": "derive", "Aggregate": "aggregate", "TimeRange": "window", @@ -22,21 +19,21 @@ const KIND_CATEGORY_JSON = `{ "TimeShift": "window", "SQLWindowFunc": "window", "Join": "join", - "RelationalJoin": "join", "Dedup": "set", "SetOp": "set", "Concat": "combine", "Sort": "sort", "Limit": "sort", - "KeepPreAsap": "summary", "SummaryAgg": "summary", "SummaryJoin": "summary", "SummarySubtract": "summary", - "SummaryBinaryOp": "summary", - "ValueOperation": "summary", "SummaryDelete": "summary", "SummaryEstimate": "summary", - "SummaryMerge": "summary" + "SummaryMerge": "summary", + "FinalizeExactAccumulator": "summary", + "MaintainPopulation": "summary", + "EvaluatePopulation": "summary", + "Extension": "summary" }`; const KIND_CATEGORY = Object.freeze(JSON.parse(KIND_CATEGORY_JSON)); @@ -44,7 +41,7 @@ const KIND_CATEGORY = Object.freeze(JSON.parse(KIND_CATEGORY_JSON)); const CATEGORIES = { data: { label: 'Data', - description: 'Scan, PromqlScalarBridge, EvalTimestamp, CurrentTimestamp — leaves that introduce a value', + description: 'Scan and Values — data sources', light: { bg: '#eef5fd', border: '#0369a1' }, dark: { bg: '#0c2438', border: '#38bdf8' }, }, @@ -62,7 +59,7 @@ const CATEGORIES = { }, derive: { label: 'Derive', - description: 'Project, PromqlRelabel, PromqlInfoEnrich, PromqlVectorFromScalar, PromqlScalarFromVector, BinaryOp — transforms or enriches columns on otherwise-unchanged rows', + description: 'Project, PromqlRelabel, PromqlInfoEnrich, PromqlVectorFromScalar, BinaryOp — transforms or enriches columns on otherwise-unchanged rows', light: { bg: '#f5f0fd', border: '#6d28d9' }, dark: { bg: '#241a3d', border: '#a78bfa' }, }, @@ -102,10 +99,10 @@ const CATEGORIES = { light: { bg: '#eef4fd', border: '#1d4ed8' }, dark: { bg: '#12233d', border: '#60a5fa' }, }, - // Post-ASAP nodes use a neutral palette; KeepPreAsap has a muted override. + // ASAP operators use a neutral palette. summary: { label: 'Summary', - description: 'KeepPreAsap, SummaryBinaryOp, ValueOperation, SummaryAgg, SummaryJoin, SummarySubtract, SummaryDelete, SummaryEstimate, SummaryMerge — post-ASAP materialized structures', + description: 'SummaryAgg, SummaryEstimate, FinalizeExactAccumulator, MaintainPopulation, EvaluatePopulation, SummaryJoin, SummarySubtract, SummaryDelete, SummaryMerge, Extension — summary state and its evaluations', light: { bg: '#f1f2f4', border: '#4b5563' }, dark: { bg: '#20242b', border: '#9ca3af' }, }, diff --git a/tools/dag-viewer/post_asap_fixture.json b/tools/dag-viewer/post_asap_fixture.json index ea2de185f..e616e80e2 100644 --- a/tools/dag-viewer/post_asap_fixture.json +++ b/tools/dag-viewer/post_asap_fixture.json @@ -14,9 +14,9 @@ }, "post_dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}], "root": 0}}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(Kll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "col": {"Column": 6}, "reduction": {"Reduce": [1]}, "grouping": "PerSubpopulationInstance"}, "children": [0], "origin_pre_id": 1 }, - { "id": 2, "kind": "KeepPreAsap", "label": "KeepPreAsap(Project)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Project", "label": "Project(2 cols)", "detail": {}, "children": []}], "root": 0}}, "children": [1] } + {"id": 2, "kind": "Project", "label": "Project(2 cols)", "detail": {}, "children": [1]} ], "root": 2 }, @@ -39,7 +39,7 @@ "kind": "Summary", "dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {"pre_asap_sub_dag": {"nodes": [{"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": [], "hash": 111}], "root": 0}}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan(netflow_table)", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(Kll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "col": {"Column": 6}, "reduction": {"Reduce": [1]}, "grouping": "PerSubpopulationInstance"}, "children": [0], "origin_pre_id": 1 } ], "root": 1 @@ -64,7 +64,7 @@ "kind": "Summary", "dag": { "nodes": [ - { "id": 0, "kind": "KeepPreAsap", "label": "KeepPreAsap(Scan)", "detail": {}, "children": [] }, + {"id": 0, "kind": "Scan", "label": "Scan", "detail": {}, "children": []}, { "id": 1, "kind": "SummaryAgg", "label": "SummaryAgg(HydraKll)", "detail": {"family": {"Sketch": ["Kll", {"k": 200}]}, "grouping": {"SharedMultiSubpopulation": {"params": {}}}}, "children": [0], "origin_pre_id": 1 } ], "root": 1 diff --git a/tools/dag-viewer/render.py b/tools/dag-viewer/render.py index fbdaca97a..e9e101b0a 100755 --- a/tools/dag-viewer/render.py +++ b/tools/dag-viewer/render.py @@ -17,7 +17,7 @@ This does not add anything index.html doesn't already do — it shares viewer.js and node-style.js with it verbatim (see viewer.js's header comment) and only differs in packaging: one query's worth of exported -`QueryExpr` detail *is* its plan (see the side panel on node click), and +`OperatorNode` detail *is* its plan (see the side panel on node click), and shared-hash highlighting *is* what this repo has for CSE today — both a hash-based proxy, not real CSE output; see README.md's "Shared-sub-DAG highlighting is a proxy" section. Structured cost/benefit annotations @@ -72,7 +72,7 @@ def _compact(value: object) -> str: if not isinstance(value, dict): return str(value) - # Common serde enum/newtype shapes in QueryExpr detail. + # Common serde enum/newtype shapes in OperatorNode detail. if set(value) == {"Column"}: return f"col[{_compact(value['Column'])}]" if set(value) == {"Table"} and isinstance(value["Table"], dict): @@ -219,17 +219,11 @@ def _semantic_label(node: dict, input_schema: object = None) -> str: lines.append(f"query: {_compact(detail.get('query'))}") elif kind == "SummaryDelete": lines.append(f"key: {_compact(detail.get('key'))}") - elif kind == "KeepPreAsap": - nested = detail.get("pre_asap_sub_dag") - nested_nodes = nested.get("nodes", []) if isinstance(nested, dict) else [] - nested_root = nested.get("root") if isinstance(nested, dict) else None - root = next((item for item in nested_nodes if item.get("id") == nested_root), None) - lines.append(f"unchanged: {root.get('kind', 'pre-ASAP sub-DAG') if root else 'pre-ASAP sub-DAG'}") else: # Less common variants still show their own scalar IR fields. Avoid - # schema/sub-DAG blobs, which belong in the click-to-inspect panel. + # the schema blob, which belongs in the click-to-inspect panel. for key, value in detail.items(): - if key not in {"schema", "pre_asap_sub_dag"} and value not in (None, [], {}): + if key != "schema" and value not in (None, [], {}): lines.append(f"{key}: {_compact(value)}") return _bounded_lines(lines) @@ -251,8 +245,6 @@ def prepare_dag(dag: object) -> None: else None ) node["label"] = _semantic_label(node, input_schema) - nested = (node.get("detail") or {}).get("pre_asap_sub_dag") - prepare_dag(nested) for query in prepared.get("queries", []): prepare_dag(query.get("dag")) diff --git a/tools/dag-viewer/viewer.js b/tools/dag-viewer/viewer.js index 6e4c696ee..36649c03a 100644 --- a/tools/dag-viewer/viewer.js +++ b/tools/dag-viewer/viewer.js @@ -18,7 +18,7 @@ cytoscape.use(window.cytoscapeDagre); // --post-asap whole-query merged post-ASAP DAG (same flattened // `{nodes, root}` shape as `dag`, but nodes may be post-ASAP-only kinds // like "SummaryAgg" mixed in, and any such node has no `hash` — there's no -// corresponding QueryExpr to hash) — left `undefined` when absent (omitted +// corresponding OperatorNode to hash) — left `undefined` when absent (omitted // whenever --post-asap wasn't set, or this query had zero replacements), // unlike `replacements` which always defaults to an array. `workload_cost` // is the optional per-query `NamedDAG.workload_cost` (issue #286), also @@ -290,22 +290,6 @@ function buildCyStyle() { selector: 'node[category = "unknown"]', style: { 'border-style': 'dashed', 'border-width': 3 }, }, - { - // KeepPreAsap (post-ASAP lane only) is post-ASAP-only - // as a *kind*, but represents literally unchanged pre-ASAP content — - // override the 'summary' category's color/icon with the same neutral - // panel/muted/dashed treatment the rest of the chrome uses for "nothing - // to see here", so a glance at the After lane separates "the planner - // did something" (solid, colored) from "left alone" (dashed, muted). - // See node-style.js's CATEGORIES.summary comment for the category-level - // color choice this overrides. - selector: 'node[kind = "KeepPreAsap"]', - style: { - 'background-color': panelColor, - 'border-color': borderColor, - 'border-style': 'dashed', - }, - }, { selector: 'node.root', style: { 'border-width': 2.5 }, @@ -663,10 +647,7 @@ function laneElements(laneId, laneLabel, dag, query, stage, laneCost) { // exactly the plain IR label. label: node.label + nodeCostBadgeSuffix(node), node, - // Flat (not nested under `node`) so buildCyStyle's - // `node[kind = "KeepPreAsap"]` selector can actually match it — - // cytoscape selectors can't reach into a data field that's itself an - // object. + // Cytoscape selectors read flat data fields. kind: node.kind, category: categoryOf(node.kind), root: node.id === dag.root, @@ -1172,10 +1153,6 @@ function renderLegend() { Shared workload nodeExplicitly identified by the exporter as shared across selected queries`); rows.push(`
Query root${escapeHtml(ROOT_BADGE.description)}
`); - const panelBg = getComputedStyle(document.documentElement).getPropertyValue('--panel2').trim() || '#f0f2f5'; - const mutedColor = getComputedStyle(document.documentElement).getPropertyValue('--muted').trim() || '#6b7280'; - rows.push(`
- Pass-through (KeepPreAsap)Unchanged pre-ASAP sub-DAG carried into the Summary DAG as-is
`); legendList.innerHTML = rows.join(''); }