From 6adea64b0dd2b294f710e5770feeb814f3efbdb9 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:59:27 +0000 Subject: [PATCH 01/18] feat(ir): define joint summary observation coverage --- crates/types/src/ir/mod.rs | 2 + crates/types/src/ir/node.rs | 35 +++++++ crates/types/src/ir/summary_coverage.rs | 125 ++++++++++++++++++++++++ crates/types/tests/summary_coverage.rs | 121 +++++++++++++++++++++++ docs/develop_docs/summary-coverage.md | 35 +++++++ 5 files changed, 318 insertions(+) create mode 100644 crates/types/src/ir/summary_coverage.rs create mode 100644 crates/types/tests/summary_coverage.rs create mode 100644 docs/develop_docs/summary-coverage.md diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index e42ad375d..4bc2b9f26 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -19,3 +19,5 @@ pub use scalar::{ExprSemantics, Predicate, ProjectItem, ScalarExpr, SortKey}; pub mod canonicalize; pub mod cse; pub mod flat; +/// Semantic observation coverage, separate from field layout and physical timing. +pub mod summary_coverage; diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index a2c6c789c..e78d801a9 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -99,6 +99,8 @@ pub struct OperatorNode { pub schema: Schema, pub guarantee: Option, pub timing: Option, + #[serde(default)] + pub summary_coverage: Option, } impl OperatorNode { @@ -121,6 +123,7 @@ impl OperatorNode { schema, guarantee: None, timing: None, + summary_coverage: None, } } @@ -141,6 +144,33 @@ impl OperatorNode { self } + /// Attach caller-established observation coverage; unknown coverage remains None. + pub fn with_summary_coverage( + mut self, + coverage: super::summary_coverage::SummaryCoverage, + ) -> Result { + coverage + .validate() + .map_err(|error| SchemaDerivationError::InvalidScalarSignature(error.to_string()))?; + if self.result_kind != OperatorResultKind::State { + return Err(SchemaDerivationError::InvalidScalarSignature( + "summary coverage requires state output".into(), + )); + } + if let Some(ASAPOp::SummaryAgg { + input, reduction, .. + }) = self.asap() + { + if *input != coverage.input || *reduction != coverage.grouping { + return Err(SchemaDerivationError::InvalidScalarSignature( + "coverage input/grouping disagrees with summary producer".into(), + )); + } + } + self.summary_coverage = Some(coverage); + Ok(self) + } + pub fn non_asap(&self) -> Option<&NonASAPOp> { match &self.operator { Operator::NonASAP(op) => Some(op), @@ -290,6 +320,11 @@ impl OperatorNode { "invalid time or identity column in schema".into(), )); } + if let Some(coverage) = &node.summary_coverage { + (*node.as_ref()) + .clone() + .with_summary_coverage(coverage.clone())?; + } node.operator.validate_inputs()?; if node.result_kind != node.operator.output_kind() { return Err(SchemaDerivationError::InvalidScalarSignature( diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/summary_coverage.rs new file mode 100644 index 000000000..e60ef3072 --- /dev/null +++ b/crates/types/src/ir/summary_coverage.rs @@ -0,0 +1,125 @@ +//! Joint time/population coverage for summary composition, independent of schema. +//! Equality predicates are a deliberately narrow proof vocabulary. Unsupported +//! predicates cannot be declared disjoint merely by giving them different names. +use super::operator_properties::Reduction; +use crate::post_asap::SummaryUpdate; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; +use thiserror::Error; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SummaryCoverage { + pub source: String, + pub revision: String, + pub input: SummaryUpdate, + pub grouping: Reduction, + pub multiplicity: ObservationMultiplicity, + /// Union of joint regions; never the Cartesian product of independent bounds. + pub regions: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub enum ObservationMultiplicity { + OncePerObservation, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CoverageRegion { + /// Half-open bounds on one canonical time axis, in milliseconds. + pub start_ms: i64, + pub end_ms: i64, + /// Conjunction of non-null equality predicates; empty means unrestricted. + pub population: BTreeMap, +} + +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum CoverageError { + #[error("coverage requires explicit source and revision identity")] + MissingIdentity, + #[error("coverage interval must have start < end")] + InvalidInterval, + #[error("population dimension names cannot be empty")] + InvalidPopulation, + #[error("summary input identity, grouping or multiplicity differs")] + IncompatibleInput, + #[error("coverage overlap is not proven absent")] + PossibleOverlap, + #[error("coverage merge requires at least one input")] + EmptyMerge, +} + +impl SummaryCoverage { + pub fn validate(&self) -> Result<(), CoverageError> { + if self.source.is_empty() || self.revision.is_empty() { + return Err(CoverageError::MissingIdentity); + } + for (index, region) in self.regions.iter().enumerate() { + if region.start_ms >= region.end_ms { + return Err(CoverageError::InvalidInterval); + } + if region.population.keys().any(String::is_empty) { + return Err(CoverageError::InvalidPopulation); + } + if self.regions[..index] + .iter() + .any(|other| region.may_overlap(other)) + { + return Err(CoverageError::PossibleOverlap); + } + } + Ok(()) + } + + /// Compose once-per-observation summaries only when their joint regions are + /// provably disjoint. Family merge capability and accuracy are separate checks. + pub fn merge_disjoint(inputs: &[Self]) -> Result { + let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; + let mut merged = first.clone(); + merged.regions.clear(); + for input in inputs { + input.validate()?; + if input.source != first.source + || input.revision != first.revision + || input.input != first.input + || input.grouping != first.grouping + || input.multiplicity != first.multiplicity + { + return Err(CoverageError::IncompatibleInput); + } + merged.regions.extend(input.regions.iter().cloned()); + } + merged.validate()?; + // Coalesce adjacent intervals only for identical population predicates. + merged.regions.sort_by(|a, b| { + a.population + .cmp(&b.population) + .then(a.start_ms.cmp(&b.start_ms)) + }); + let mut normalized: Vec = Vec::new(); + for region in merged.regions { + if let Some(last) = normalized.last_mut() { + if last.population == region.population && last.end_ms == region.start_ms { + last.end_ms = region.end_ms; + continue; + } + } + normalized.push(region); + } + merged.regions = normalized; + Ok(merged) + } +} +impl CoverageRegion { + fn may_overlap(&self, other: &Self) -> bool { + self.start_ms < other.end_ms + && other.start_ms < self.end_ms + && !self.population.iter().any(|(dimension, value)| { + other + .population + .get(dimension) + .is_some_and(|other| other != value) + }) + } +} diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs new file mode 100644 index 000000000..246d6b0c8 --- /dev/null +++ b/crates/types/tests/summary_coverage.rs @@ -0,0 +1,121 @@ +//! Coverage composition preserves gaps and rejects duplicate observations. +use asap_types::{ + ir::operator_properties::Reduction, ir::summary_coverage::*, post_asap::SummaryUpdate, + pre_asap::ColumnRef, +}; +fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverage { + SummaryCoverage { + source: "flows".into(), + revision: "snapshot-1".into(), + input: SummaryUpdate::column(ColumnRef::Named("latency".into())), + grouping: Reduction::by(vec![0]), + multiplicity: ObservationMultiplicity::OncePerObservation, + regions: vec![CoverageRegion { + start_ms: start, + end_ms: end, + population: population + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + }], + } +} +/// Adjacent panes coalesce; gaps remain disconnected rather than becoming a hull. +#[test] +fn time_union_preserves_gaps() { + let merged = + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); + assert_eq!(merged.regions[0].end_ms, 2); + assert_eq!(merged.regions.len(), 1); + let gapped = + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); + assert_eq!(gapped.regions.len(), 2); +} +/// Population partitions can overlap in time without sharing observations. +#[test] +fn population_and_joint_union() { + let merged = SummaryCoverage::merge_disjoint(&[ + coverage(0, 2, &[("region", "us")]), + coverage(0, 2, &[("region", "eu")]), + ]) + .unwrap(); + assert_eq!(merged.regions.len(), 2); + let joint = SummaryCoverage::merge_disjoint(&[ + coverage(0, 1, &[("region", "us")]), + coverage(1, 2, &[("region", "eu")]), + ]) + .unwrap(); + assert_eq!(joint.regions.len(), 2); + let decoded: SummaryCoverage = + serde_json::from_str(&serde_json::to_string(&joint).unwrap()).unwrap(); + assert_eq!(decoded, joint); +} +/// Intersecting predicates and windows cannot authorize once-per-observation merge. +#[test] +fn overlap_and_identity_fail_closed() { + assert_eq!( + SummaryCoverage::merge_disjoint(&[coverage(0, 2, &[]), coverage(1, 3, &[])]), + Err(CoverageError::PossibleOverlap) + ); + assert_eq!( + SummaryCoverage::merge_disjoint(&[ + coverage(0, 2, &[("region", "us")]), + coverage(0, 2, &[("tier", "premium")]) + ]), + Err(CoverageError::PossibleOverlap) + ); + let mut other = coverage(1, 2, &[]); + other.revision = "snapshot-2".into(); + assert_eq!( + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), other]), + Err(CoverageError::IncompatibleInput) + ); + assert_eq!( + coverage(2, 1, &[]).validate(), + Err(CoverageError::InvalidInterval) + ); +} + +/// Coverage is logical state metadata, and input rewrites invalidate its proof. +#[test] +fn node_coverage_is_checked_and_rewrites_clear_it() { + use asap_types::{ + ir::operator_properties::Source, + ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}, + post_asap::{SketchAlgorithm, SketchKind, SketchParams}, + pre_asap::{DataType, Field, FieldDataType, Schema}, + }; + let raw = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { + source: Source::Table { + table_ref: "flows".into(), + }, + predicates: vec![], + schema: Schema::new(vec![Field::plain("latency", DataType::Float64, false)]), + })) + .unwrap(); + let mut declared = coverage(0, 1, &[]); + declared.grouping = Reduction::by(vec![]); + assert!((*raw) + .clone() + .with_summary_coverage(declared.clone()) + .is_err()); + let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { + child: raw, + family: FieldDataType::Sketch( + SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), + Default::default(), + ), + input: declared.input.clone(), + reduction: declared.grouping.clone(), + grouping: Default::default(), + filter: None, + })) + .unwrap(); + let state = state.with_summary_coverage(declared.clone()).unwrap(); + assert!(state.summary_coverage.is_some()); + let mut bad = declared; + bad.input = SummaryUpdate::column(ColumnRef::Named("other".into())); + assert!(state.clone().with_summary_coverage(bad).is_err()); + let rebuilt = state.map_children(Clone::clone).unwrap(); + assert!(rebuilt.summary_coverage.is_none()); +} diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md new file mode 100644 index 000000000..2653d0426 --- /dev/null +++ b/docs/develop_docs/summary-coverage.md @@ -0,0 +1,35 @@ +# Summary coverage contract + +Schema describes field layout; summary coverage describes eligible observations. +`OperatorNode.summary_coverage` is optional logical metadata. `None` means unknown, +not unrestricted coverage. Rewriting inputs clears it along with other assessed +metadata. `with_summary_coverage` validates declared coverage and checks state kind +and SummaryAgg input/grouping agreement. Provenance is supplied by a trusted +composition rule/catalog; this API does not infer predicates from arbitrary SQL. + +`SummaryCoverage` records source and revision identity, update expression, +grouping, once-per-observation multiplicity and a union of joint `CoverageRegion`s. +Each region pairs half-open time bounds in milliseconds with a conjunction of +non-null equality predicates over canonical population dimensions. Source identity +must include the time axis and observation-identity namespace. Revision identifies +the input snapshot/update contract used to construct the state. + +`merge_disjoint` requires equal input identities and provably disjoint joint +regions. Adjacent intervals coalesce only with identical population predicates; +gaps remain separate. Conflicting equality predicates on the same dimension prove +population disjointness. Independent predicates do not: region=US can overlap +tier=premium. Different source/revision/input/grouping contracts fail. + +US×[0,1) merged with EU×[1,2) remains two regions, not +{US,EU}×[0,2). This avoids inventing missing cross-population/time coverage. +Empty regions describe empty observation coverage. Empty merge input is invalid. + +This contract supports conservative once-per-observation composition. Arbitrary +predicates, null predicates, unbounded time coverage, idempotent set-union algebra, +coverage inference and full requested-window containment need explicit extensions. +It never labels unsupported/unknown predicates disjoint. It provides no runtime +merge capability, accuracy certificate, storage policy or execution timing. + +The following merge PR must require known coverage, derive the output union and +validate it rather than treating matching schemas as sufficient authorization. +Logical transport and CSE must preserve and compare coverage metadata. From d97fe2b5d6da683d32681cf016dd623926aa26ff Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:32:07 +0000 Subject: [PATCH 02/18] refactor(ir): name represented observations ObservationExtent --- crates/types/src/ir/mod.rs | 2 +- crates/types/src/ir/node.rs | 14 +++---- ...mary_coverage.rs => observation_extent.rs} | 30 ++++++------- crates/types/src/post_asap/mod.rs | 2 +- crates/types/src/post_asap/summary_window.rs | 18 ++++---- ...mary_coverage.rs => observation_extent.rs} | 42 +++++++++---------- ...mary-coverage.md => observation-extent.md} | 18 ++++++-- 7 files changed, 68 insertions(+), 58 deletions(-) rename crates/types/src/ir/{summary_coverage.rs => observation_extent.rs} (84%) rename crates/types/tests/{summary_coverage.rs => observation_extent.rs} (74%) rename docs/develop_docs/{summary-coverage.md => observation-extent.md} (70%) diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 4bc2b9f26..2783fc7f9 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -20,4 +20,4 @@ pub mod canonicalize; pub mod cse; pub mod flat; /// Semantic observation coverage, separate from field layout and physical timing. -pub mod summary_coverage; +pub mod observation_extent; diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index e78d801a9..5ed2a6c1f 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -100,7 +100,7 @@ pub struct OperatorNode { pub guarantee: Option, pub timing: Option, #[serde(default)] - pub summary_coverage: Option, + pub observation_extent: Option, } impl OperatorNode { @@ -123,7 +123,7 @@ impl OperatorNode { schema, guarantee: None, timing: None, - summary_coverage: None, + observation_extent: None, } } @@ -145,9 +145,9 @@ impl OperatorNode { } /// Attach caller-established observation coverage; unknown coverage remains None. - pub fn with_summary_coverage( + pub fn with_observation_extent( mut self, - coverage: super::summary_coverage::SummaryCoverage, + coverage: super::observation_extent::ObservationExtent, ) -> Result { coverage .validate() @@ -167,7 +167,7 @@ impl OperatorNode { )); } } - self.summary_coverage = Some(coverage); + self.observation_extent = Some(coverage); Ok(self) } @@ -320,10 +320,10 @@ impl OperatorNode { "invalid time or identity column in schema".into(), )); } - if let Some(coverage) = &node.summary_coverage { + if let Some(coverage) = &node.observation_extent { (*node.as_ref()) .clone() - .with_summary_coverage(coverage.clone())?; + .with_observation_extent(coverage.clone())?; } node.operator.validate_inputs()?; if node.result_kind != node.operator.output_kind() { diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/observation_extent.rs similarity index 84% rename from crates/types/src/ir/summary_coverage.rs rename to crates/types/src/ir/observation_extent.rs index e60ef3072..0d7e2ad9d 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/observation_extent.rs @@ -9,14 +9,14 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct SummaryCoverage { +pub struct ObservationExtent { pub source: String, pub revision: String, pub input: SummaryUpdate, pub grouping: Reduction, pub multiplicity: ObservationMultiplicity, /// Union of joint regions; never the Cartesian product of independent bounds. - pub regions: Vec, + pub regions: Vec, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] @@ -26,7 +26,7 @@ pub enum ObservationMultiplicity { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct CoverageRegion { +pub struct ExtentRegion { /// Half-open bounds on one canonical time axis, in milliseconds. pub start_ms: i64, pub end_ms: i64, @@ -35,7 +35,7 @@ pub struct CoverageRegion { } #[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum CoverageError { +pub enum ExtentError { #[error("coverage requires explicit source and revision identity")] MissingIdentity, #[error("coverage interval must have start < end")] @@ -50,23 +50,23 @@ pub enum CoverageError { EmptyMerge, } -impl SummaryCoverage { - pub fn validate(&self) -> Result<(), CoverageError> { +impl ObservationExtent { + pub fn validate(&self) -> Result<(), ExtentError> { if self.source.is_empty() || self.revision.is_empty() { - return Err(CoverageError::MissingIdentity); + return Err(ExtentError::MissingIdentity); } for (index, region) in self.regions.iter().enumerate() { if region.start_ms >= region.end_ms { - return Err(CoverageError::InvalidInterval); + return Err(ExtentError::InvalidInterval); } if region.population.keys().any(String::is_empty) { - return Err(CoverageError::InvalidPopulation); + return Err(ExtentError::InvalidPopulation); } if self.regions[..index] .iter() .any(|other| region.may_overlap(other)) { - return Err(CoverageError::PossibleOverlap); + return Err(ExtentError::PossibleOverlap); } } Ok(()) @@ -74,8 +74,8 @@ impl SummaryCoverage { /// Compose once-per-observation summaries only when their joint regions are /// provably disjoint. Family merge capability and accuracy are separate checks. - pub fn merge_disjoint(inputs: &[Self]) -> Result { - let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; + pub fn merge_disjoint(inputs: &[Self]) -> Result { + let first = inputs.first().ok_or(ExtentError::EmptyMerge)?; let mut merged = first.clone(); merged.regions.clear(); for input in inputs { @@ -86,7 +86,7 @@ impl SummaryCoverage { || input.grouping != first.grouping || input.multiplicity != first.multiplicity { - return Err(CoverageError::IncompatibleInput); + return Err(ExtentError::IncompatibleInput); } merged.regions.extend(input.regions.iter().cloned()); } @@ -97,7 +97,7 @@ impl SummaryCoverage { .cmp(&b.population) .then(a.start_ms.cmp(&b.start_ms)) }); - let mut normalized: Vec = Vec::new(); + let mut normalized: Vec = Vec::new(); for region in merged.regions { if let Some(last) = normalized.last_mut() { if last.population == region.population && last.end_ms == region.start_ms { @@ -111,7 +111,7 @@ impl SummaryCoverage { Ok(merged) } } -impl CoverageRegion { +impl ExtentRegion { fn may_overlap(&self, other: &Self) -> bool { self.start_ms < other.end_ms && other.start_ms < self.end_ms diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index f71aaca0f..c0028bad6 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -77,6 +77,6 @@ pub use summary_maintenance_lifecycle::{ SummaryMaintenanceLifecycleGuarantee, }; pub use summary_window::{ - plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, SummaryWindowFramework, + plan_pane_phase, validate_pane_coverage, PaneExtentError, PaneLayout, SummaryWindowFramework, WindowEdgeCoverage, }; diff --git a/crates/types/src/post_asap/summary_window.rs b/crates/types/src/post_asap/summary_window.rs index 0e344c1ed..440a79344 100644 --- a/crates/types/src/post_asap/summary_window.rs +++ b/crates/types/src/post_asap/summary_window.rs @@ -47,7 +47,7 @@ pub enum WindowEdgeCoverage { } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum PaneCoverageError { +pub enum PaneExtentError { ZeroPaneWidth, UnknownPaneOrigin, UnknownEvaluationPhase, @@ -64,9 +64,9 @@ pub fn validate_pane_coverage( layout: &PaneLayout, evaluation_time_ms: Option, edge_coverage: &WindowEdgeCoverage, -) -> Result<(), PaneCoverageError> { +) -> Result<(), PaneExtentError> { if layout.pane_width_ms == 0 { - return Err(PaneCoverageError::ZeroPaneWidth); + return Err(PaneExtentError::ZeroPaneWidth); } if matches!( edge_coverage, @@ -76,15 +76,15 @@ pub fn validate_pane_coverage( } let origin = layout .pane_origin_ms - .ok_or(PaneCoverageError::UnknownPaneOrigin)?; - let evaluation = evaluation_time_ms.ok_or(PaneCoverageError::UnknownEvaluationPhase)?; + .ok_or(PaneExtentError::UnknownPaneOrigin)?; + let evaluation = evaluation_time_ms.ok_or(PaneExtentError::UnknownEvaluationPhase)?; let width = layout.pane_width_ms as i64; let pane_phase_ms = origin.rem_euclid(width) as u64; let query_phase_ms = evaluation.rem_euclid(width) as u64; if pane_phase_ms == query_phase_ms { Ok(()) } else { - Err(PaneCoverageError::PhaseMismatch { + Err(PaneExtentError::PhaseMismatch { pane_phase_ms, query_phase_ms, }) @@ -98,9 +98,9 @@ pub fn validate_pane_coverage( pub fn plan_pane_phase( demand: &RepeatedDemand, pane_width_ms: u64, -) -> Result { +) -> Result { if pane_width_ms == 0 { - return Err(PaneCoverageError::ZeroPaneWidth); + return Err(PaneExtentError::ZeroPaneWidth); } let phase = match demand { RepeatedDemand::FixedIntervalAt { @@ -154,7 +154,7 @@ mod tests { }; assert_eq!( validate_pane_coverage(&layout, Some(56_000), &WindowEdgeCoverage::PaneAligned), - Err(PaneCoverageError::PhaseMismatch { + Err(PaneExtentError::PhaseMismatch { pane_phase_ms: 26_000, query_phase_ms: 56_000, }) diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/observation_extent.rs similarity index 74% rename from crates/types/tests/summary_coverage.rs rename to crates/types/tests/observation_extent.rs index 246d6b0c8..9947052f1 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/observation_extent.rs @@ -1,16 +1,16 @@ //! Coverage composition preserves gaps and rejects duplicate observations. use asap_types::{ - ir::operator_properties::Reduction, ir::summary_coverage::*, post_asap::SummaryUpdate, + ir::observation_extent::*, ir::operator_properties::Reduction, post_asap::SummaryUpdate, pre_asap::ColumnRef, }; -fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverage { - SummaryCoverage { +fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> ObservationExtent { + ObservationExtent { source: "flows".into(), revision: "snapshot-1".into(), input: SummaryUpdate::column(ColumnRef::Named("latency".into())), grouping: Reduction::by(vec![0]), multiplicity: ObservationMultiplicity::OncePerObservation, - regions: vec![CoverageRegion { + regions: vec![ExtentRegion { start_ms: start, end_ms: end, population: population @@ -24,29 +24,29 @@ fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverag #[test] fn time_union_preserves_gaps() { let merged = - SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); + ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); assert_eq!(merged.regions[0].end_ms, 2); assert_eq!(merged.regions.len(), 1); let gapped = - SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); + ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); assert_eq!(gapped.regions.len(), 2); } /// Population partitions can overlap in time without sharing observations. #[test] fn population_and_joint_union() { - let merged = SummaryCoverage::merge_disjoint(&[ + let merged = ObservationExtent::merge_disjoint(&[ coverage(0, 2, &[("region", "us")]), coverage(0, 2, &[("region", "eu")]), ]) .unwrap(); assert_eq!(merged.regions.len(), 2); - let joint = SummaryCoverage::merge_disjoint(&[ + let joint = ObservationExtent::merge_disjoint(&[ coverage(0, 1, &[("region", "us")]), coverage(1, 2, &[("region", "eu")]), ]) .unwrap(); assert_eq!(joint.regions.len(), 2); - let decoded: SummaryCoverage = + let decoded: ObservationExtent = serde_json::from_str(&serde_json::to_string(&joint).unwrap()).unwrap(); assert_eq!(decoded, joint); } @@ -54,25 +54,25 @@ fn population_and_joint_union() { #[test] fn overlap_and_identity_fail_closed() { assert_eq!( - SummaryCoverage::merge_disjoint(&[coverage(0, 2, &[]), coverage(1, 3, &[])]), - Err(CoverageError::PossibleOverlap) + ObservationExtent::merge_disjoint(&[coverage(0, 2, &[]), coverage(1, 3, &[])]), + Err(ExtentError::PossibleOverlap) ); assert_eq!( - SummaryCoverage::merge_disjoint(&[ + ObservationExtent::merge_disjoint(&[ coverage(0, 2, &[("region", "us")]), coverage(0, 2, &[("tier", "premium")]) ]), - Err(CoverageError::PossibleOverlap) + Err(ExtentError::PossibleOverlap) ); let mut other = coverage(1, 2, &[]); other.revision = "snapshot-2".into(); assert_eq!( - SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), other]), - Err(CoverageError::IncompatibleInput) + ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), other]), + Err(ExtentError::IncompatibleInput) ); assert_eq!( coverage(2, 1, &[]).validate(), - Err(CoverageError::InvalidInterval) + Err(ExtentError::InvalidInterval) ); } @@ -97,7 +97,7 @@ fn node_coverage_is_checked_and_rewrites_clear_it() { declared.grouping = Reduction::by(vec![]); assert!((*raw) .clone() - .with_summary_coverage(declared.clone()) + .with_observation_extent(declared.clone()) .is_err()); let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: raw, @@ -111,11 +111,11 @@ fn node_coverage_is_checked_and_rewrites_clear_it() { filter: None, })) .unwrap(); - let state = state.with_summary_coverage(declared.clone()).unwrap(); - assert!(state.summary_coverage.is_some()); + let state = state.with_observation_extent(declared.clone()).unwrap(); + assert!(state.observation_extent.is_some()); let mut bad = declared; bad.input = SummaryUpdate::column(ColumnRef::Named("other".into())); - assert!(state.clone().with_summary_coverage(bad).is_err()); + assert!(state.clone().with_observation_extent(bad).is_err()); let rebuilt = state.map_children(Clone::clone).unwrap(); - assert!(rebuilt.summary_coverage.is_none()); + assert!(rebuilt.observation_extent.is_none()); } diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/observation-extent.md similarity index 70% rename from docs/develop_docs/summary-coverage.md rename to docs/develop_docs/observation-extent.md index 2653d0426..9a8406f3d 100644 --- a/docs/develop_docs/summary-coverage.md +++ b/docs/develop_docs/observation-extent.md @@ -1,14 +1,14 @@ # Summary coverage contract Schema describes field layout; summary coverage describes eligible observations. -`OperatorNode.summary_coverage` is optional logical metadata. `None` means unknown, +`OperatorNode.observation_extent` is optional logical metadata. `None` means unknown, not unrestricted coverage. Rewriting inputs clears it along with other assessed -metadata. `with_summary_coverage` validates declared coverage and checks state kind +metadata. `with_observation_extent` validates declared coverage and checks state kind and SummaryAgg input/grouping agreement. Provenance is supplied by a trusted composition rule/catalog; this API does not infer predicates from arbitrary SQL. -`SummaryCoverage` records source and revision identity, update expression, -grouping, once-per-observation multiplicity and a union of joint `CoverageRegion`s. +`ObservationExtent` records source and revision identity, update expression, +grouping, once-per-observation multiplicity and a union of joint `ExtentRegion`s. Each region pairs half-open time bounds in milliseconds with a conjunction of non-null equality predicates over canonical population dimensions. Source identity must include the time axis and observation-identity namespace. Revision identifies @@ -33,3 +33,13 @@ merge capability, accuracy certificate, storage policy or execution timing. The following merge PR must require known coverage, derive the output union and validate it rather than treating matching schemas as sufficient authorization. Logical transport and CSE must preserve and compare coverage metadata. + +## Why extent + +`ObservationExtent` describes the declared set of source observations represented +by a state. The name borrows the set meaning of "extent" from object databases; +it is a project-specific term, not an ODMG class extent implementation. +`None` means unknown extent; an empty region list means a known empty extent. +Disjoint union preserves gaps and joint population/time relationships. +The later query-relative coverage check asks whether this extent satisfies a +requested population/window. Declaring an extent does not prove that check. From 1369bcdbad6c43262f758e59de5139b14cd84c02 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:56:48 +0000 Subject: [PATCH 03/18] refactor(ir): rename observation extent to SummaryCoverage and trim its fields Name the metadata coverage to match #560 and the docs; drop the single-variant multiplicity and deployment-specific revision; rename grouping to reduction to match SummaryAgg; report failures through SchemaDerivationError::Coverage; revert the unrelated PaneCoverageError rename. Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/error.rs | 2 + crates/types/src/ir/mod.rs | 2 +- crates/types/src/ir/node.rs | 31 ++++------ ...ervation_extent.rs => summary_coverage.rs} | 57 +++++++++---------- crates/types/src/post_asap/mod.rs | 2 +- crates/types/src/post_asap/summary_window.rs | 18 +++--- ...ervation_extent.rs => summary_coverage.rs} | 55 ++++++++---------- docs/develop_docs/observation-extent.md | 45 --------------- docs/develop_docs/summary-coverage.md | 32 +++++++++++ 9 files changed, 110 insertions(+), 134 deletions(-) rename crates/types/src/ir/{observation_extent.rs => summary_coverage.rs} (70%) rename crates/types/tests/{observation_extent.rs => summary_coverage.rs} (66%) delete mode 100644 docs/develop_docs/observation-extent.md create mode 100644 docs/develop_docs/summary-coverage.md diff --git a/crates/types/src/ir/error.rs b/crates/types/src/ir/error.rs index 4b6849965..a0f9e2271 100644 --- a/crates/types/src/ir/error.rs +++ b/crates/types/src/ir/error.rs @@ -16,4 +16,6 @@ pub enum SchemaDerivationError { EmptyConcat, #[error("invalid per-series sample column: {0}")] InvalidSampleColumn(String), + #[error("invalid summary coverage: {0}")] + Coverage(#[from] super::summary_coverage::CoverageError), } diff --git a/crates/types/src/ir/mod.rs b/crates/types/src/ir/mod.rs index 2783fc7f9..4bc2b9f26 100644 --- a/crates/types/src/ir/mod.rs +++ b/crates/types/src/ir/mod.rs @@ -20,4 +20,4 @@ pub mod canonicalize; pub mod cse; pub mod flat; /// Semantic observation coverage, separate from field layout and physical timing. -pub mod observation_extent; +pub mod summary_coverage; diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index 5ed2a6c1f..269b245a9 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize}; use super::asap::ASAPOp; use super::non_asap::NonASAPOp; +use super::summary_coverage::{CoverageError, SummaryCoverage}; use crate::ir::SchemaDerivationError; use crate::post_asap::execution_data_state::ExecutionTiming; use crate::post_asap::guarantee::ResultGuarantee; @@ -100,7 +101,7 @@ pub struct OperatorNode { pub guarantee: Option, pub timing: Option, #[serde(default)] - pub observation_extent: Option, + pub coverage: Option, } impl OperatorNode { @@ -123,7 +124,7 @@ impl OperatorNode { schema, guarantee: None, timing: None, - observation_extent: None, + coverage: None, } } @@ -145,29 +146,23 @@ impl OperatorNode { } /// Attach caller-established observation coverage; unknown coverage remains None. - pub fn with_observation_extent( + pub fn with_coverage( mut self, - coverage: super::observation_extent::ObservationExtent, + coverage: SummaryCoverage, ) -> Result { - coverage - .validate() - .map_err(|error| SchemaDerivationError::InvalidScalarSignature(error.to_string()))?; + coverage.validate()?; if self.result_kind != OperatorResultKind::State { - return Err(SchemaDerivationError::InvalidScalarSignature( - "summary coverage requires state output".into(), - )); + return Err(CoverageError::NotState.into()); } if let Some(ASAPOp::SummaryAgg { input, reduction, .. }) = self.asap() { - if *input != coverage.input || *reduction != coverage.grouping { - return Err(SchemaDerivationError::InvalidScalarSignature( - "coverage input/grouping disagrees with summary producer".into(), - )); + if *input != coverage.input || *reduction != coverage.reduction { + return Err(CoverageError::ProducerMismatch.into()); } } - self.observation_extent = Some(coverage); + self.coverage = Some(coverage); Ok(self) } @@ -320,10 +315,8 @@ impl OperatorNode { "invalid time or identity column in schema".into(), )); } - if let Some(coverage) = &node.observation_extent { - (*node.as_ref()) - .clone() - .with_observation_extent(coverage.clone())?; + if let Some(coverage) = &node.coverage { + (*node.as_ref()).clone().with_coverage(coverage.clone())?; } node.operator.validate_inputs()?; if node.result_kind != node.operator.output_kind() { diff --git a/crates/types/src/ir/observation_extent.rs b/crates/types/src/ir/summary_coverage.rs similarity index 70% rename from crates/types/src/ir/observation_extent.rs rename to crates/types/src/ir/summary_coverage.rs index 0d7e2ad9d..b761480ea 100644 --- a/crates/types/src/ir/observation_extent.rs +++ b/crates/types/src/ir/summary_coverage.rs @@ -9,24 +9,20 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ObservationExtent { +pub struct SummaryCoverage { + /// Observation stream identity, including its time axis. pub source: String, - pub revision: String, + /// Must equal the producing `SummaryAgg.input`. pub input: SummaryUpdate, - pub grouping: Reduction, - pub multiplicity: ObservationMultiplicity, + /// Must equal the producing `SummaryAgg.reduction`. + pub reduction: Reduction, /// Union of joint regions; never the Cartesian product of independent bounds. - pub regions: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] -pub enum ObservationMultiplicity { - OncePerObservation, + pub regions: Vec, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] -pub struct ExtentRegion { +pub struct CoverageRegion { /// Half-open bounds on one canonical time axis, in milliseconds. pub start_ms: i64, pub end_ms: i64, @@ -35,58 +31,61 @@ pub struct ExtentRegion { } #[derive(Debug, Clone, PartialEq, Eq, Error)] -pub enum ExtentError { - #[error("coverage requires explicit source and revision identity")] +pub enum CoverageError { + #[error("coverage requires an explicit source identity")] MissingIdentity, #[error("coverage interval must have start < end")] InvalidInterval, #[error("population dimension names cannot be empty")] InvalidPopulation, - #[error("summary input identity, grouping or multiplicity differs")] + #[error("summary source, input or reduction differs")] IncompatibleInput, #[error("coverage overlap is not proven absent")] PossibleOverlap, #[error("coverage merge requires at least one input")] EmptyMerge, + #[error("summary coverage requires state output")] + NotState, + #[error("coverage input/reduction disagrees with summary producer")] + ProducerMismatch, } -impl ObservationExtent { - pub fn validate(&self) -> Result<(), ExtentError> { - if self.source.is_empty() || self.revision.is_empty() { - return Err(ExtentError::MissingIdentity); +impl SummaryCoverage { + pub fn validate(&self) -> Result<(), CoverageError> { + if self.source.is_empty() { + return Err(CoverageError::MissingIdentity); } for (index, region) in self.regions.iter().enumerate() { if region.start_ms >= region.end_ms { - return Err(ExtentError::InvalidInterval); + return Err(CoverageError::InvalidInterval); } if region.population.keys().any(String::is_empty) { - return Err(ExtentError::InvalidPopulation); + return Err(CoverageError::InvalidPopulation); } if self.regions[..index] .iter() .any(|other| region.may_overlap(other)) { - return Err(ExtentError::PossibleOverlap); + return Err(CoverageError::PossibleOverlap); } } Ok(()) } + /// Every observation in a region is assumed to contribute once to the state. /// Compose once-per-observation summaries only when their joint regions are /// provably disjoint. Family merge capability and accuracy are separate checks. - pub fn merge_disjoint(inputs: &[Self]) -> Result { - let first = inputs.first().ok_or(ExtentError::EmptyMerge)?; + pub fn merge_disjoint(inputs: &[Self]) -> Result { + let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; let mut merged = first.clone(); merged.regions.clear(); for input in inputs { input.validate()?; if input.source != first.source - || input.revision != first.revision || input.input != first.input - || input.grouping != first.grouping - || input.multiplicity != first.multiplicity + || input.reduction != first.reduction { - return Err(ExtentError::IncompatibleInput); + return Err(CoverageError::IncompatibleInput); } merged.regions.extend(input.regions.iter().cloned()); } @@ -97,7 +96,7 @@ impl ObservationExtent { .cmp(&b.population) .then(a.start_ms.cmp(&b.start_ms)) }); - let mut normalized: Vec = Vec::new(); + let mut normalized: Vec = Vec::new(); for region in merged.regions { if let Some(last) = normalized.last_mut() { if last.population == region.population && last.end_ms == region.start_ms { @@ -111,7 +110,7 @@ impl ObservationExtent { Ok(merged) } } -impl ExtentRegion { +impl CoverageRegion { fn may_overlap(&self, other: &Self) -> bool { self.start_ms < other.end_ms && other.start_ms < self.end_ms diff --git a/crates/types/src/post_asap/mod.rs b/crates/types/src/post_asap/mod.rs index c0028bad6..f71aaca0f 100644 --- a/crates/types/src/post_asap/mod.rs +++ b/crates/types/src/post_asap/mod.rs @@ -77,6 +77,6 @@ pub use summary_maintenance_lifecycle::{ SummaryMaintenanceLifecycleGuarantee, }; pub use summary_window::{ - plan_pane_phase, validate_pane_coverage, PaneExtentError, PaneLayout, SummaryWindowFramework, + plan_pane_phase, validate_pane_coverage, PaneCoverageError, PaneLayout, SummaryWindowFramework, WindowEdgeCoverage, }; diff --git a/crates/types/src/post_asap/summary_window.rs b/crates/types/src/post_asap/summary_window.rs index 440a79344..0e344c1ed 100644 --- a/crates/types/src/post_asap/summary_window.rs +++ b/crates/types/src/post_asap/summary_window.rs @@ -47,7 +47,7 @@ pub enum WindowEdgeCoverage { } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub enum PaneExtentError { +pub enum PaneCoverageError { ZeroPaneWidth, UnknownPaneOrigin, UnknownEvaluationPhase, @@ -64,9 +64,9 @@ pub fn validate_pane_coverage( layout: &PaneLayout, evaluation_time_ms: Option, edge_coverage: &WindowEdgeCoverage, -) -> Result<(), PaneExtentError> { +) -> Result<(), PaneCoverageError> { if layout.pane_width_ms == 0 { - return Err(PaneExtentError::ZeroPaneWidth); + return Err(PaneCoverageError::ZeroPaneWidth); } if matches!( edge_coverage, @@ -76,15 +76,15 @@ pub fn validate_pane_coverage( } let origin = layout .pane_origin_ms - .ok_or(PaneExtentError::UnknownPaneOrigin)?; - let evaluation = evaluation_time_ms.ok_or(PaneExtentError::UnknownEvaluationPhase)?; + .ok_or(PaneCoverageError::UnknownPaneOrigin)?; + let evaluation = evaluation_time_ms.ok_or(PaneCoverageError::UnknownEvaluationPhase)?; let width = layout.pane_width_ms as i64; let pane_phase_ms = origin.rem_euclid(width) as u64; let query_phase_ms = evaluation.rem_euclid(width) as u64; if pane_phase_ms == query_phase_ms { Ok(()) } else { - Err(PaneExtentError::PhaseMismatch { + Err(PaneCoverageError::PhaseMismatch { pane_phase_ms, query_phase_ms, }) @@ -98,9 +98,9 @@ pub fn validate_pane_coverage( pub fn plan_pane_phase( demand: &RepeatedDemand, pane_width_ms: u64, -) -> Result { +) -> Result { if pane_width_ms == 0 { - return Err(PaneExtentError::ZeroPaneWidth); + return Err(PaneCoverageError::ZeroPaneWidth); } let phase = match demand { RepeatedDemand::FixedIntervalAt { @@ -154,7 +154,7 @@ mod tests { }; assert_eq!( validate_pane_coverage(&layout, Some(56_000), &WindowEdgeCoverage::PaneAligned), - Err(PaneExtentError::PhaseMismatch { + Err(PaneCoverageError::PhaseMismatch { pane_phase_ms: 26_000, query_phase_ms: 56_000, }) diff --git a/crates/types/tests/observation_extent.rs b/crates/types/tests/summary_coverage.rs similarity index 66% rename from crates/types/tests/observation_extent.rs rename to crates/types/tests/summary_coverage.rs index 9947052f1..c36ed0ecb 100644 --- a/crates/types/tests/observation_extent.rs +++ b/crates/types/tests/summary_coverage.rs @@ -1,16 +1,14 @@ //! Coverage composition preserves gaps and rejects duplicate observations. use asap_types::{ - ir::observation_extent::*, ir::operator_properties::Reduction, post_asap::SummaryUpdate, + ir::operator_properties::Reduction, ir::summary_coverage::*, post_asap::SummaryUpdate, pre_asap::ColumnRef, }; -fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> ObservationExtent { - ObservationExtent { +fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverage { + SummaryCoverage { source: "flows".into(), - revision: "snapshot-1".into(), input: SummaryUpdate::column(ColumnRef::Named("latency".into())), - grouping: Reduction::by(vec![0]), - multiplicity: ObservationMultiplicity::OncePerObservation, - regions: vec![ExtentRegion { + reduction: Reduction::by(vec![0]), + regions: vec![CoverageRegion { start_ms: start, end_ms: end, population: population @@ -24,29 +22,29 @@ fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> ObservationExt #[test] fn time_union_preserves_gaps() { let merged = - ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); assert_eq!(merged.regions[0].end_ms, 2); assert_eq!(merged.regions.len(), 1); let gapped = - ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); assert_eq!(gapped.regions.len(), 2); } /// Population partitions can overlap in time without sharing observations. #[test] fn population_and_joint_union() { - let merged = ObservationExtent::merge_disjoint(&[ + let merged = SummaryCoverage::merge_disjoint(&[ coverage(0, 2, &[("region", "us")]), coverage(0, 2, &[("region", "eu")]), ]) .unwrap(); assert_eq!(merged.regions.len(), 2); - let joint = ObservationExtent::merge_disjoint(&[ + let joint = SummaryCoverage::merge_disjoint(&[ coverage(0, 1, &[("region", "us")]), coverage(1, 2, &[("region", "eu")]), ]) .unwrap(); assert_eq!(joint.regions.len(), 2); - let decoded: ObservationExtent = + let decoded: SummaryCoverage = serde_json::from_str(&serde_json::to_string(&joint).unwrap()).unwrap(); assert_eq!(decoded, joint); } @@ -54,25 +52,25 @@ fn population_and_joint_union() { #[test] fn overlap_and_identity_fail_closed() { assert_eq!( - ObservationExtent::merge_disjoint(&[coverage(0, 2, &[]), coverage(1, 3, &[])]), - Err(ExtentError::PossibleOverlap) + SummaryCoverage::merge_disjoint(&[coverage(0, 2, &[]), coverage(1, 3, &[])]), + Err(CoverageError::PossibleOverlap) ); assert_eq!( - ObservationExtent::merge_disjoint(&[ + SummaryCoverage::merge_disjoint(&[ coverage(0, 2, &[("region", "us")]), coverage(0, 2, &[("tier", "premium")]) ]), - Err(ExtentError::PossibleOverlap) + Err(CoverageError::PossibleOverlap) ); let mut other = coverage(1, 2, &[]); - other.revision = "snapshot-2".into(); + other.source = "other-flows".into(); assert_eq!( - ObservationExtent::merge_disjoint(&[coverage(0, 1, &[]), other]), - Err(ExtentError::IncompatibleInput) + SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), other]), + Err(CoverageError::IncompatibleInput) ); assert_eq!( coverage(2, 1, &[]).validate(), - Err(ExtentError::InvalidInterval) + Err(CoverageError::InvalidInterval) ); } @@ -94,11 +92,8 @@ fn node_coverage_is_checked_and_rewrites_clear_it() { })) .unwrap(); let mut declared = coverage(0, 1, &[]); - declared.grouping = Reduction::by(vec![]); - assert!((*raw) - .clone() - .with_observation_extent(declared.clone()) - .is_err()); + declared.reduction = Reduction::by(vec![]); + assert!((*raw).clone().with_coverage(declared.clone()).is_err()); let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: raw, family: FieldDataType::Sketch( @@ -106,16 +101,16 @@ fn node_coverage_is_checked_and_rewrites_clear_it() { Default::default(), ), input: declared.input.clone(), - reduction: declared.grouping.clone(), + reduction: declared.reduction.clone(), grouping: Default::default(), filter: None, })) .unwrap(); - let state = state.with_observation_extent(declared.clone()).unwrap(); - assert!(state.observation_extent.is_some()); + let state = state.with_coverage(declared.clone()).unwrap(); + assert!(state.coverage.is_some()); let mut bad = declared; bad.input = SummaryUpdate::column(ColumnRef::Named("other".into())); - assert!(state.clone().with_observation_extent(bad).is_err()); + assert!(state.clone().with_coverage(bad).is_err()); let rebuilt = state.map_children(Clone::clone).unwrap(); - assert!(rebuilt.observation_extent.is_none()); + assert!(rebuilt.coverage.is_none()); } diff --git a/docs/develop_docs/observation-extent.md b/docs/develop_docs/observation-extent.md deleted file mode 100644 index 9a8406f3d..000000000 --- a/docs/develop_docs/observation-extent.md +++ /dev/null @@ -1,45 +0,0 @@ -# Summary coverage contract - -Schema describes field layout; summary coverage describes eligible observations. -`OperatorNode.observation_extent` is optional logical metadata. `None` means unknown, -not unrestricted coverage. Rewriting inputs clears it along with other assessed -metadata. `with_observation_extent` validates declared coverage and checks state kind -and SummaryAgg input/grouping agreement. Provenance is supplied by a trusted -composition rule/catalog; this API does not infer predicates from arbitrary SQL. - -`ObservationExtent` records source and revision identity, update expression, -grouping, once-per-observation multiplicity and a union of joint `ExtentRegion`s. -Each region pairs half-open time bounds in milliseconds with a conjunction of -non-null equality predicates over canonical population dimensions. Source identity -must include the time axis and observation-identity namespace. Revision identifies -the input snapshot/update contract used to construct the state. - -`merge_disjoint` requires equal input identities and provably disjoint joint -regions. Adjacent intervals coalesce only with identical population predicates; -gaps remain separate. Conflicting equality predicates on the same dimension prove -population disjointness. Independent predicates do not: region=US can overlap -tier=premium. Different source/revision/input/grouping contracts fail. - -US×[0,1) merged with EU×[1,2) remains two regions, not -{US,EU}×[0,2). This avoids inventing missing cross-population/time coverage. -Empty regions describe empty observation coverage. Empty merge input is invalid. - -This contract supports conservative once-per-observation composition. Arbitrary -predicates, null predicates, unbounded time coverage, idempotent set-union algebra, -coverage inference and full requested-window containment need explicit extensions. -It never labels unsupported/unknown predicates disjoint. It provides no runtime -merge capability, accuracy certificate, storage policy or execution timing. - -The following merge PR must require known coverage, derive the output union and -validate it rather than treating matching schemas as sufficient authorization. -Logical transport and CSE must preserve and compare coverage metadata. - -## Why extent - -`ObservationExtent` describes the declared set of source observations represented -by a state. The name borrows the set meaning of "extent" from object databases; -it is a project-specific term, not an ODMG class extent implementation. -`None` means unknown extent; an empty region list means a known empty extent. -Disjoint union preserves gaps and joint population/time relationships. -The later query-relative coverage check asks whether this extent satisfies a -requested population/window. Declaring an extent does not prove that check. diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md new file mode 100644 index 000000000..3f41e9b83 --- /dev/null +++ b/docs/develop_docs/summary-coverage.md @@ -0,0 +1,32 @@ +# Summary coverage contract + +`Schema` describes an edge's field layout and committed state type. It does not +say which observations a summary state was built from: a KLL over `[0,1)` and a +KLL over `[1,3)` have equal schemas. `OperatorNode.coverage` records that +separately, as optional logical metadata. `None` means unknown, not unrestricted. +Rewriting a node's inputs clears it along with other assessed metadata. + +`SummaryCoverage` records: + +- `source`: the observation stream, including its time axis. +- `input`, `reduction`: must equal the producing `SummaryAgg`'s fields of the same + name. `with_coverage` checks this and requires state output. +- `regions`: a union of `CoverageRegion`s. Each pairs half-open time bounds in + milliseconds with a conjunction of non-null equality predicates over population + dimensions; an empty predicate map means all observations of the source. + +Every observation in a region contributes once to the state. Declarations come +from trusted composition rules or catalogs; nothing is inferred from SQL. + +`merge_disjoint` requires equal source/input/reduction and provably disjoint +regions. Adjacent intervals coalesce only with identical population predicates; +gaps remain separate. Conflicting values for the same dimension prove disjointness. +Different dimensions do not: `region=us` can overlap `tier=premium`. + +`us×[0,1)` merged with `eu×[1,2)` remains two regions, not `{us,eu}×[0,2)`. +Empty `regions` is known empty coverage. Empty merge input is invalid. + +Not covered: arbitrary or null predicates, unbounded time, idempotent set-union +families, and checking that coverage contains a requested query window or +population. The contract provides no runtime merge capability, accuracy +certificate, storage policy or execution timing. From b9767e04c2dc5dc3204844ac891c531d2c432f26 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:10:07 +0000 Subject: [PATCH 04/18] docs(ir): describe coverage source as any observation data source Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/summary_coverage.rs | 3 ++- docs/develop_docs/summary-coverage.md | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/summary_coverage.rs index b761480ea..77ce54187 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/summary_coverage.rs @@ -10,7 +10,8 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryCoverage { - /// Observation stream identity, including its time axis. + /// Observation data source identity: any table or stream, not necessarily + /// time series. Region time bounds refer to its time column. pub source: String, /// Must equal the producing `SummaryAgg.input`. pub input: SummaryUpdate, diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md index 3f41e9b83..3f452f7b2 100644 --- a/docs/develop_docs/summary-coverage.md +++ b/docs/develop_docs/summary-coverage.md @@ -8,7 +8,8 @@ Rewriting a node's inputs clears it along with other assessed metadata. `SummaryCoverage` records: -- `source`: the observation stream, including its time axis. +- `source`: the observation data source. It can be any tabular data, not + necessarily a time series; region time bounds refer to its time column. - `input`, `reduction`: must equal the producing `SummaryAgg`'s fields of the same name. `with_coverage` checks this and requires state output. - `regions`: a union of `CoverageRegion`s. Each pairs half-open time bounds in From 743b177da69921ba0abfc5d0ded0c595c37b6c7b Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:19:01 +0000 Subject: [PATCH 05/18] feat(ir): require coverage on summary nodes; allow regions without time bounds validate_structure rejects a SummaryAgg without coverage (CoverageError::Missing). CoverageRegion time bounds become optional so tabular sources without a time column can declare coverage. Population stays trusted; #570 tracks checking it. Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/node.rs | 16 ++++++-- crates/types/src/ir/summary_coverage.rs | 35 ++++++++++++------ crates/types/tests/schema_rebuilding.rs | 47 +++++++++++++++++++----- crates/types/tests/structure_contract.rs | 39 +++++++++++++++----- crates/types/tests/summary_coverage.rs | 39 +++++++++++++++++--- docs/develop_docs/summary-coverage.md | 43 ++++++++++++++++------ 6 files changed, 170 insertions(+), 49 deletions(-) diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index 269b245a9..37975cef6 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -145,7 +145,8 @@ impl OperatorNode { self } - /// Attach caller-established observation coverage; unknown coverage remains None. + /// Attach caller-established coverage. Required on summary nodes; see + /// [`Self::requires_coverage`]. pub fn with_coverage( mut self, coverage: SummaryCoverage, @@ -166,6 +167,11 @@ impl OperatorNode { Ok(self) } + /// Summary nodes whose state can be composed must declare coverage. + pub fn requires_coverage(&self) -> bool { + matches!(self.asap(), Some(ASAPOp::SummaryAgg { .. })) + } + pub fn non_asap(&self) -> Option<&NonASAPOp> { match &self.operator { Operator::NonASAP(op) => Some(op), @@ -315,8 +321,12 @@ impl OperatorNode { "invalid time or identity column in schema".into(), )); } - if let Some(coverage) = &node.coverage { - (*node.as_ref()).clone().with_coverage(coverage.clone())?; + match &node.coverage { + Some(coverage) => { + (*node.as_ref()).clone().with_coverage(coverage.clone())?; + } + None if node.requires_coverage() => return Err(CoverageError::Missing.into()), + None => {} } node.operator.validate_inputs()?; if node.result_kind != node.operator.output_kind() { diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/summary_coverage.rs index 77ce54187..5056ef3f2 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/summary_coverage.rs @@ -5,6 +5,7 @@ use super::operator_properties::Reduction; use crate::post_asap::SummaryUpdate; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; +use std::ops::Range; use thiserror::Error; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -24,9 +25,9 @@ pub struct SummaryCoverage { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CoverageRegion { - /// Half-open bounds on one canonical time axis, in milliseconds. - pub start_ms: i64, - pub end_ms: i64, + /// Half-open bounds on the source's time column, in milliseconds. `None` + /// means no time restriction, e.g. a source without a time column. + pub time_ms: Option>, /// Conjunction of non-null equality predicates; empty means unrestricted. pub population: BTreeMap, } @@ -49,6 +50,8 @@ pub enum CoverageError { NotState, #[error("coverage input/reduction disagrees with summary producer")] ProducerMismatch, + #[error("summary node requires coverage")] + Missing, } impl SummaryCoverage { @@ -57,7 +60,7 @@ impl SummaryCoverage { return Err(CoverageError::MissingIdentity); } for (index, region) in self.regions.iter().enumerate() { - if region.start_ms >= region.end_ms { + if region.time_ms.as_ref().is_some_and(Range::is_empty) { return Err(CoverageError::InvalidInterval); } if region.population.keys().any(String::is_empty) { @@ -93,16 +96,21 @@ impl SummaryCoverage { merged.validate()?; // Coalesce adjacent intervals only for identical population predicates. merged.regions.sort_by(|a, b| { - a.population - .cmp(&b.population) - .then(a.start_ms.cmp(&b.start_ms)) + a.population.cmp(&b.population).then( + a.time_ms + .as_ref() + .map(|t| t.start) + .cmp(&b.time_ms.as_ref().map(|t| t.start)), + ) }); let mut normalized: Vec = Vec::new(); for region in merged.regions { if let Some(last) = normalized.last_mut() { - if last.population == region.population && last.end_ms == region.start_ms { - last.end_ms = region.end_ms; - continue; + if let (Some(last_time), Some(time)) = (&mut last.time_ms, ®ion.time_ms) { + if last.population == region.population && last_time.end == time.start { + last_time.end = time.end; + continue; + } } } normalized.push(region); @@ -113,8 +121,11 @@ impl SummaryCoverage { } impl CoverageRegion { fn may_overlap(&self, other: &Self) -> bool { - self.start_ms < other.end_ms - && other.start_ms < self.end_ms + let time_overlaps = match (&self.time_ms, &other.time_ms) { + (Some(a), Some(b)) => a.start < b.end && b.start < a.end, + _ => true, + }; + time_overlaps && !self.population.iter().any(|(dimension, value)| { other .population diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index 1f1b57dbc..ca30d39e5 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -1,3 +1,4 @@ +use asap_types::ir::summary_coverage::{CoverageRegion, SummaryCoverage}; use asap_types::ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}; use asap_types::post_asap::{ ExactKind, ExactParams, ExecutionTiming, GroupingStrategy, ResultGuarantee, SummaryUpdate, @@ -7,6 +8,27 @@ use asap_types::pre_asap::{ }; use std::rc::Rc; +fn coverage() -> SummaryCoverage { + SummaryCoverage { + source: "t".into(), + input: SummaryUpdate::column(ColumnRef::Named("value".into())), + reduction: Reduction::by(vec![0]), + regions: vec![CoverageRegion { + time_ms: None, + population: Default::default(), + }], + } +} +/// Rewrites clear coverage; a rewriter must declare it again for summary nodes. +fn redeclare(node: OperatorNode) -> Rc { + let node = Rc::new(node); + if !node.requires_coverage() { + return node; + } + assert!(node.validate_structure().is_err()); + Rc::new((*node).clone().with_coverage(coverage()).unwrap()) +} + fn scan(key_type: DataType, name: &str) -> Rc { OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { source: Source::Table { @@ -40,7 +62,12 @@ fn aggregate(child: Rc, asap: bool) -> Rc { having: None, }) }; - OperatorNode::new_shared(operator).unwrap() + let node = OperatorNode::new(operator).unwrap(); + Rc::new(if asap { + node.with_coverage(coverage()).unwrap() + } else { + node + }) } /// Rewrites follow changed input types and inherited names for either category. @@ -50,7 +77,7 @@ fn rebuilding_rederives_schema_for_both_categories() { let original = aggregate(scan(DataType::Int64, "key"), asap); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); - let rebuilt = Rc::new(original.map_children(|_| replacement.clone()).unwrap()); + let rebuilt = redeclare(original.map_children(|_| replacement.clone()).unwrap()); assert_eq!(rebuilt.schema, rebuilt.operator.output_schema().unwrap()); rebuilt.validate_structure().unwrap(); } @@ -64,14 +91,14 @@ fn rebuilding_preserves_only_explicit_naming_overrides() { let mut schema = original.schema.clone(); schema.fields[0].name = "alias".into(); schema.fields[0].table = Some("result".into()); - let original = Rc::new( - OperatorNode::with_schema(original.operator.clone(), schema) - .with_guarantee(Some(ResultGuarantee::exact("fixture"))) - .with_timing(Some(ExecutionTiming::QueryTime)), - ); + let mut renamed = OperatorNode::with_schema(original.operator.clone(), schema) + .with_guarantee(Some(ResultGuarantee::exact("fixture"))) + .with_timing(Some(ExecutionTiming::QueryTime)); + renamed.coverage = original.coverage.clone(); + let original = Rc::new(renamed); original.validate_structure().unwrap(); let replacement = scan(DataType::Utf8, "new_key"); - let rebuilt = Rc::new(original.map_children(|_| replacement.clone()).unwrap()); + let rebuilt = redeclare(original.map_children(|_| replacement.clone()).unwrap()); assert_eq!(rebuilt.schema.fields[0].name, "alias"); assert_eq!(rebuilt.schema.fields[0].table.as_deref(), Some("result")); assert_eq!( @@ -110,7 +137,9 @@ fn validation_rejects_structural_overrides_for_both_categories() { schema.fields.pop(); invalid.push(schema); for schema in invalid { - let forged = Rc::new(OperatorNode::with_schema(original.operator.clone(), schema)); + let mut forged = OperatorNode::with_schema(original.operator.clone(), schema); + forged.coverage = original.coverage.clone(); + let forged = Rc::new(forged); assert!( forged.validate_structure().is_err(), "accepted structural override: {:?}", diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index d4e164f71..af21337a8 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -13,6 +13,21 @@ fn scan() -> Rc { })) .unwrap() } +/// Tabular coverage for a whole-table summary of column `x`. +fn whole_table() -> asap_types::ir::summary_coverage::SummaryCoverage { + use asap_types::ir::summary_coverage::{CoverageRegion, SummaryCoverage}; + SummaryCoverage { + source: "t".into(), + input: asap_types::post_asap::SummaryUpdate::column( + asap_types::pre_asap::ColumnRef::Named("x".into()), + ), + reduction: asap_types::pre_asap::Reduction::by(vec![]), + regions: vec![CoverageRegion { + time_ms: None, + population: Default::default(), + }], + } +} /// Resolved filters cannot hide invalid scalar types or out-of-scope columns. #[test] fn invalid_predicates_are_rejected() { @@ -103,6 +118,8 @@ fn state_evaluations_and_passthrough_keep_their_contracts() { grouping: GroupingStrategy::default(), filter: None, })) + .unwrap() + .with_coverage(whole_table()) .unwrap(), ); state.validate_structure().unwrap(); @@ -173,15 +190,19 @@ fn shared_construction_derives_both_operator_categories() { use asap_types::pre_asap::{ColumnRef, FieldDataType, Reduction}; let input = scan(); - let state = OperatorNode::new_shared(Operator::ASAP(ASAPOp::SummaryAgg { - child: input.clone(), - family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), - input: SummaryUpdate::column(ColumnRef::Named("x".into())), - reduction: Reduction::by(vec![]), - grouping: GroupingStrategy::default(), - filter: None, - })) - .unwrap(); + let state = Rc::new( + OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { + child: input.clone(), + family: FieldDataType::ExactAggregate(ExactKind::Sum, ExactParams::Sum), + input: SummaryUpdate::column(ColumnRef::Named("x".into())), + reduction: Reduction::by(vec![]), + grouping: GroupingStrategy::default(), + filter: None, + })) + .unwrap() + .with_coverage(whole_table()) + .unwrap(), + ); assert_eq!(state.result_kind, OperatorResultKind::State); assert!(!state.schema.fields.last().unwrap().is_plain()); assert!(state.guarantee.is_none()); diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs index c36ed0ecb..e3059677e 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/summary_coverage.rs @@ -9,8 +9,7 @@ fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverag input: SummaryUpdate::column(ColumnRef::Named("latency".into())), reduction: Reduction::by(vec![0]), regions: vec![CoverageRegion { - start_ms: start, - end_ms: end, + time_ms: Some(start..end), population: population .iter() .map(|(k, v)| (k.to_string(), v.to_string())) @@ -23,7 +22,7 @@ fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverag fn time_union_preserves_gaps() { let merged = SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(1, 2, &[])]).unwrap(); - assert_eq!(merged.regions[0].end_ms, 2); + assert_eq!(merged.regions[0].time_ms, Some(0..2)); assert_eq!(merged.regions.len(), 1); let gapped = SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), coverage(2, 3, &[])]).unwrap(); @@ -76,7 +75,7 @@ fn overlap_and_identity_fail_closed() { /// Coverage is logical state metadata, and input rewrites invalidate its proof. #[test] -fn node_coverage_is_checked_and_rewrites_clear_it() { +fn node_coverage_is_required_checked_and_cleared_by_rewrites() { use asap_types::{ ir::operator_properties::Source, ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}, @@ -106,11 +105,41 @@ fn node_coverage_is_checked_and_rewrites_clear_it() { filter: None, })) .unwrap(); + // Summary nodes cannot validate without coverage. + assert!(matches!( + std::rc::Rc::new(state.clone()).validate_structure(), + Err(asap_types::ir::SchemaDerivationError::Coverage( + CoverageError::Missing + )) + )); let state = state.with_coverage(declared.clone()).unwrap(); - assert!(state.coverage.is_some()); + std::rc::Rc::new(state.clone()) + .validate_structure() + .unwrap(); let mut bad = declared; bad.input = SummaryUpdate::column(ColumnRef::Named("other".into())); assert!(state.clone().with_coverage(bad).is_err()); let rebuilt = state.map_children(Clone::clone).unwrap(); assert!(rebuilt.coverage.is_none()); } + +/// Sources without a time column declare no time bounds; such a region overlaps +/// any region it is not population-disjoint from. +#[test] +fn regions_without_time_bounds() { + let mut tabular = coverage(0, 1, &[("region", "us")]); + tabular.regions[0].time_ms = None; + let mut other = coverage(0, 1, &[("region", "eu")]); + other.regions[0].time_ms = None; + assert_eq!( + SummaryCoverage::merge_disjoint(&[tabular.clone(), other]) + .unwrap() + .regions + .len(), + 2 + ); + assert_eq!( + SummaryCoverage::merge_disjoint(&[tabular, coverage(5, 6, &[("region", "us")])]), + Err(CoverageError::PossibleOverlap) + ); +} diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md index 3f452f7b2..d43a52aed 100644 --- a/docs/develop_docs/summary-coverage.md +++ b/docs/develop_docs/summary-coverage.md @@ -3,8 +3,10 @@ `Schema` describes an edge's field layout and committed state type. It does not say which observations a summary state was built from: a KLL over `[0,1)` and a KLL over `[1,3)` have equal schemas. `OperatorNode.coverage` records that -separately, as optional logical metadata. `None` means unknown, not unrestricted. -Rewriting a node's inputs clears it along with other assessed metadata. +separately. It is required on summary nodes (`SummaryAgg`, and `SummaryMerge`, +which derives it): `validate_structure` rejects them with `CoverageError::Missing` +when it is `None`. Other nodes leave it `None`. Rewriting a node's inputs clears +it, so a rewriter must declare it again with `with_coverage`. `SummaryCoverage` records: @@ -12,22 +14,41 @@ Rewriting a node's inputs clears it along with other assessed metadata. necessarily a time series; region time bounds refer to its time column. - `input`, `reduction`: must equal the producing `SummaryAgg`'s fields of the same name. `with_coverage` checks this and requires state output. -- `regions`: a union of `CoverageRegion`s. Each pairs half-open time bounds in - milliseconds with a conjunction of non-null equality predicates over population +- `regions`: a union of `CoverageRegion`s. Each pairs optional half-open time + bounds in milliseconds (`None`: no time restriction, e.g. a source without a + time column) with a conjunction of non-null equality predicates over population dimensions; an empty predicate map means all observations of the source. -Every observation in a region contributes once to the state. Declarations come -from trusted composition rules or catalogs; nothing is inferred from SQL. +Every observation in a region contributes once to the state. + +## Trusted declarations + +Coverage is declared by the composition rule or catalog that built the subtree. +Only `input` and `reduction` are checked against the producer. Population and +time bounds are trusted: population is not compared with `SummaryAgg.filter`, +`Filter` nodes or `Scan.predicates`, and `TimeRange` is relative, so absolute +bounds cannot be checked. A wrong declaration therefore passes: + +```text +A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong +B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) +merge_disjoint(A, B) is accepted, and every US observation is counted twice. +``` + +Issue #570 tracks checking population against the subtree's predicates. + +## Merging `merge_disjoint` requires equal source/input/reduction and provably disjoint regions. Adjacent intervals coalesce only with identical population predicates; gaps remain separate. Conflicting values for the same dimension prove disjointness. -Different dimensions do not: `region=us` can overlap `tier=premium`. +Different dimensions do not: `region=us` can overlap `tier=premium`. A region +without time bounds overlaps every region it is not population-disjoint from. `us×[0,1)` merged with `eu×[1,2)` remains two regions, not `{us,eu}×[0,2)`. Empty `regions` is known empty coverage. Empty merge input is invalid. -Not covered: arbitrary or null predicates, unbounded time, idempotent set-union -families, and checking that coverage contains a requested query window or -population. The contract provides no runtime merge capability, accuracy -certificate, storage policy or execution timing. +Not covered: arbitrary or null predicates, idempotent set-union families, and +checking that coverage contains a requested query window or population. The +contract provides no runtime merge capability, accuracy certificate, storage +policy or execution timing. From 6d48e07df3054cce2dabd421c47349711f9aa167 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:29:17 +0000 Subject: [PATCH 06/18] docs: explain the summary coverage problem with examples Co-Authored-By: Claude Opus 5.5 --- docs/develop_docs/summary-coverage.md | 198 ++++++++++++++++++++------ 1 file changed, 157 insertions(+), 41 deletions(-) diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md index d43a52aed..1ad87c570 100644 --- a/docs/develop_docs/summary-coverage.md +++ b/docs/develop_docs/summary-coverage.md @@ -1,54 +1,170 @@ # Summary coverage contract -`Schema` describes an edge's field layout and committed state type. It does not -say which observations a summary state was built from: a KLL over `[0,1)` and a -KLL over `[1,3)` have equal schemas. `OperatorNode.coverage` records that -separately. It is required on summary nodes (`SummaryAgg`, and `SummaryMerge`, -which derives it): `validate_structure` rejects them with `CoverageError::Missing` -when it is `None`. Other nodes leave it `None`. Rewriting a node's inputs clears -it, so a rewriter must declare it again with `with_coverage`. - -`SummaryCoverage` records: - -- `source`: the observation data source. It can be any tabular data, not - necessarily a time series; region time bounds refer to its time column. -- `input`, `reduction`: must equal the producing `SummaryAgg`'s fields of the same - name. `with_coverage` checks this and requires state output. -- `regions`: a union of `CoverageRegion`s. Each pairs optional half-open time - bounds in milliseconds (`None`: no time restriction, e.g. a source without a - time column) with a conjunction of non-null equality predicates over population - dimensions; an empty predicate map means all observations of the source. - -Every observation in a region contributes once to the state. +## Problem: a schema says what a summary *is*, not what it *summarizes* -## Trusted declarations +Every ASAP edge has one `Schema`. For a summary edge it records the field +layout and the committed state type: + +```rust +pub struct Schema { + pub fields: Vec, // e.g. job: Plain(Utf8), state: Sketch(KLL{k=200}, PerSubpopulationInstance) + pub time_index: Option, // position of a timestamp column, not a time range + pub unique_keys: Vec>, + pub closed: bool, +} +``` -Coverage is declared by the composition rule or catalog that built the subtree. -Only `input` and `reduction` are checked against the producer. Population and -time bounds are trusted: population is not compared with `SummaryAgg.filter`, -`Filter` nodes or `Scan.predicates`, and `TimeRange` is relative, so absolute -bounds cannot be checked. A wrong declaration therefore passes: +Nothing in it says **which time range** or **which population (label values)** +the state was built from. Filters, group keys and windows are deliberately not +`Schema` or `Field` members. This becomes a gap once the planner combines +existing summary states (`SummaryMerge`, reuse of ingested panes, sub-DAG +sharing). The producer no longer shows where a state came from, so only the +schema is left to compare. Every example below uses two states with +**exactly equal schemas**: ```text -A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong -B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) -merge_disjoint(A, B) is accepted, and every US observation is counted twice. +Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}, PerSubpopulationInstance)), result_kind = State ``` -Issue #570 tracks checking population against the subtree's predicates. +### Example 1: time. Equal schemas, different answers + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| latency, `[00:00, 00:01)` | latency, `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | +| latency, `[00:00, 00:02)` | latency, `[00:01, 00:03)` | **wrong**: every observation in `[00:01, 00:02)` is counted twice, which skews the quantile and doubles counts or frequencies | +| latency, `[00:00, 00:01)` | latency, `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`; **wrong** if used for the continuous window `[00:00, 00:03)` | + +`time_index` is a column position. A KLL state has no timestamp column, so +`time_index` is `None` in all three rows and the schema cannot tell them apart. + +### Example 2: population (label values). Equal schemas, different answers + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| `region='us'` | `region='eu'` | correct: p99 for `us ∪ eu` within each `job` | +| `region='us'` | `tier='premium'` | **wrong**: premium US requests are counted in both inputs | +| `region='us'` | `region='us'` | **wrong**: everything is counted twice | + +`region` is a filter label, not an output column, so it never appears in the +schema. The `job` field only says the state is grouped by job. It does not say +which jobs or which rows contributed. + +### Example 3: time and population together + +A = `us × [0,1)` and B = `eu × [1,2)`. The merged state covers exactly those two +blocks. Storing a time range and a label set separately would give +`{us,eu} × [0,2)`. That claims EU data for `[0,1)` and US data for `[1,2)` +that was never read. Time and population must stay **paired per region**. + +### Example 4: answering a query from a stored state + +Query: `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) GROUP BY job`. +A stored state with the matching schema could hold US data for 10:00–10:05, EU +data, or US data for only 10:00–10:03. All three have the same schema. The +schema confirms that the state *type* fits, not that the *contents* fit. + +**Conclusion.** Schema equality is necessary but not sufficient for composing +or reusing summaries. Without time and population metadata, the planner must +either refuse every composition or accept silent double counting and missing +data. + +## The contract + +`Schema` stays the layout contract and does **not** describe coverage. Coverage +is a separate field on the node, next to `schema`: + +```text +OperatorNode +├── schema: Schema what each output row looks like +└── coverage: Option which observations the state holds +``` + +Coverage cannot live inside `Schema`. `SummaryMerge` requires equal input +schemas, and the inputs of a useful merge (`[0,1)` + `[1,2)`) always have +different coverage. + +```rust +pub struct SummaryCoverage { + pub source: String, // observation data source; any tabular data, not necessarily time series + pub input: SummaryUpdate, // must equal the producing SummaryAgg.input + pub reduction: Reduction, // must equal the producing SummaryAgg.reduction + pub regions: Vec, // union of time × population blocks +} +pub struct CoverageRegion { + pub time_ms: Option>, // half-open, on the source's time column; None = no time restriction + pub population: BTreeMap, // label = value AND …; empty = all observations +} +``` + +Rules: + +- Coverage is **required on summary nodes**. `validate_structure` rejects a + `SummaryAgg` or `SummaryMerge` whose coverage is `None` with + `CoverageError::Missing`. Other nodes leave it `None`. The field is an + `Option` only because all operators share `OperatorNode`. +- `with_coverage` validates the declaration, requires `State` output + (`NotState`), and checks `input`/`reduction` against a `SummaryAgg` producer + (`ProducerMismatch`). `validate_structure` re-checks it. +- `SummaryMerge` derives its coverage from its inputs. `validate_structure` + rejects a retained value that differs from that union. +- Rewriting a node's inputs clears its coverage. The rewriter must declare it + again with `with_coverage`. +- Every observation in a region contributes once to the state. `regions = []` + means known empty coverage. +- `time_ms: None` is for sources without a time column. Such a region overlaps + every region it is not population-disjoint from. ## Merging -`merge_disjoint` requires equal source/input/reduction and provably disjoint -regions. Adjacent intervals coalesce only with identical population predicates; -gaps remain separate. Conflicting values for the same dimension prove disjointness. -Different dimensions do not: `region=us` can overlap `tier=premium`. A region -without time bounds overlaps every region it is not population-disjoint from. +`SummaryCoverage::merge_disjoint` requires equal `source`/`input`/`reduction` +and provably disjoint regions. Two regions are disjoint when their time ranges +do not intersect, or when they give different values for the same population +label. Different labels prove nothing. The examples above come out as: + +| Case | Result | +|---|---| +| `[0,1)` + `[1,2)`, same population | accepted, coalesced to one region `[0,2)` | +| `[0,1)` + `[2,3)` | accepted, **two** regions (gap kept) | +| `[0,2)` + `[1,3)` | `PossibleOverlap` | +| `region=us` + `region=eu`, same time | accepted, two regions | +| `region=us` + `tier=premium` | `PossibleOverlap` | +| `us×[0,1)` + `eu×[1,2)` | accepted, two regions, never widened to `{us,eu}×[0,2)` | +| no time bounds + any region of the same population | `PossibleOverlap` | +| different source / input / reduction | `IncompatibleInput` | + +Adjacent intervals coalesce only when their population maps are identical. +Merging an empty input list fails with `EmptyMerge`. + +## Trusted declarations + +Coverage is declared by the composition rule or catalog that built the +subtree. Nothing is inferred from SQL. Only `input` and `reduction` are checked +against the producer. Population is not compared with `SummaryAgg.filter`, +`Filter` nodes or `Scan.predicates`. Time bounds cannot be checked, because +`TimeRange` stores a relative duration. So a wrong declaration passes: + +```text +A = SummaryAgg(filter: region='us', input: latency, reduction: by job) + declared coverage: {region: eu} × [0,1) ← wrong; the state holds US data +B = SummaryAgg(filter: region='us', input: latency, reduction: by job) + declared coverage: {region: us} × [0,1) + +merge_disjoint(A, B) → accepted ("eu" ≠ "us" proves disjoint) +actual merged state → every US observation in [0,1) counted twice +a query for region='eu' could also be answered from A, which holds no EU data +``` + +Issue #570 tracks the check. The declared population must exactly equal the +`column = literal` predicates collected between the `SummaryAgg` and its +`Scan`, and any other predicate shape fails closed. It starts strict about +which operators may sit on that path (only `Filter` and `TimeRange`), because +`Project` or `Join` can rename columns or change rows. -`us×[0,1)` merged with `eu×[1,2)` remains two regions, not `{us,eu}×[0,2)`. -Empty `regions` is known empty coverage. Empty merge input is invalid. +## Not covered -Not covered: arbitrary or null predicates, idempotent set-union families, and -checking that coverage contains a requested query window or population. The -contract provides no runtime merge capability, accuracy certificate, storage -policy or execution timing. +- Checking that coverage *contains* a requested query window or population + (Example 4). That is a later query-relative check, which uses this data. +- Predicates beyond non-null equality conjunctions; idempotent set-union + families. +- Runtime merge kernels, accuracy certificates, storage policy or execution + timing. From c57497fff3b76eb66caaeae762f5785fe801659f Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:32:07 +0000 Subject: [PATCH 07/18] refactor(ir): keep only time and population in SummaryCoverage Given required coverage on summary nodes, input and reduction duplicated the producing SummaryAgg fields; drop them along with ProducerMismatch. Type source as Source, matching Scan. Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/node.rs | 8 ----- crates/types/src/ir/summary_coverage.rs | 34 ++++++-------------- crates/types/tests/schema_rebuilding.rs | 6 ++-- crates/types/tests/structure_contract.rs | 10 +++--- crates/types/tests/summary_coverage.rs | 41 +++++++++++++----------- docs/develop_docs/summary-coverage.md | 24 +++++++------- 6 files changed, 53 insertions(+), 70 deletions(-) diff --git a/crates/types/src/ir/node.rs b/crates/types/src/ir/node.rs index 37975cef6..7534353b0 100644 --- a/crates/types/src/ir/node.rs +++ b/crates/types/src/ir/node.rs @@ -155,14 +155,6 @@ impl OperatorNode { if self.result_kind != OperatorResultKind::State { return Err(CoverageError::NotState.into()); } - if let Some(ASAPOp::SummaryAgg { - input, reduction, .. - }) = self.asap() - { - if *input != coverage.input || *reduction != coverage.reduction { - return Err(CoverageError::ProducerMismatch.into()); - } - } self.coverage = Some(coverage); Ok(self) } diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/summary_coverage.rs index 5056ef3f2..a95896eb8 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/summary_coverage.rs @@ -1,8 +1,7 @@ //! Joint time/population coverage for summary composition, independent of schema. //! Equality predicates are a deliberately narrow proof vocabulary. Unsupported //! predicates cannot be declared disjoint merely by giving them different names. -use super::operator_properties::Reduction; -use crate::post_asap::SummaryUpdate; +use crate::pre_asap::Source; use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; use std::ops::Range; @@ -11,13 +10,9 @@ use thiserror::Error; #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct SummaryCoverage { - /// Observation data source identity: any table or stream, not necessarily - /// time series. Region time bounds refer to its time column. - pub source: String, - /// Must equal the producing `SummaryAgg.input`. - pub input: SummaryUpdate, - /// Must equal the producing `SummaryAgg.reduction`. - pub reduction: Reduction, + /// Observation data source, as named by `Scan`: a table or a time series. + /// Region time bounds refer to its time column. + pub source: Source, /// Union of joint regions; never the Cartesian product of independent bounds. pub regions: Vec, } @@ -34,31 +29,24 @@ pub struct CoverageRegion { #[derive(Debug, Clone, PartialEq, Eq, Error)] pub enum CoverageError { - #[error("coverage requires an explicit source identity")] - MissingIdentity, #[error("coverage interval must have start < end")] InvalidInterval, #[error("population dimension names cannot be empty")] InvalidPopulation, - #[error("summary source, input or reduction differs")] - IncompatibleInput, + #[error("summary coverage sources differ")] + SourceMismatch, #[error("coverage overlap is not proven absent")] PossibleOverlap, #[error("coverage merge requires at least one input")] EmptyMerge, #[error("summary coverage requires state output")] NotState, - #[error("coverage input/reduction disagrees with summary producer")] - ProducerMismatch, #[error("summary node requires coverage")] Missing, } impl SummaryCoverage { pub fn validate(&self) -> Result<(), CoverageError> { - if self.source.is_empty() { - return Err(CoverageError::MissingIdentity); - } for (index, region) in self.regions.iter().enumerate() { if region.time_ms.as_ref().is_some_and(Range::is_empty) { return Err(CoverageError::InvalidInterval); @@ -78,18 +66,16 @@ impl SummaryCoverage { /// Every observation in a region is assumed to contribute once to the state. /// Compose once-per-observation summaries only when their joint regions are - /// provably disjoint. Family merge capability and accuracy are separate checks. + /// provably disjoint. Update/reduction compatibility, family merge capability + /// and accuracy are checked by `SummaryMerge`, not here. pub fn merge_disjoint(inputs: &[Self]) -> Result { let first = inputs.first().ok_or(CoverageError::EmptyMerge)?; let mut merged = first.clone(); merged.regions.clear(); for input in inputs { input.validate()?; - if input.source != first.source - || input.input != first.input - || input.reduction != first.reduction - { - return Err(CoverageError::IncompatibleInput); + if input.source != first.source { + return Err(CoverageError::SourceMismatch); } merged.regions.extend(input.regions.iter().cloned()); } diff --git a/crates/types/tests/schema_rebuilding.rs b/crates/types/tests/schema_rebuilding.rs index ca30d39e5..fc7a3f145 100644 --- a/crates/types/tests/schema_rebuilding.rs +++ b/crates/types/tests/schema_rebuilding.rs @@ -10,9 +10,9 @@ use std::rc::Rc; fn coverage() -> SummaryCoverage { SummaryCoverage { - source: "t".into(), - input: SummaryUpdate::column(ColumnRef::Named("value".into())), - reduction: Reduction::by(vec![0]), + source: Source::Table { + table_ref: "t".into(), + }, regions: vec![CoverageRegion { time_ms: None, population: Default::default(), diff --git a/crates/types/tests/structure_contract.rs b/crates/types/tests/structure_contract.rs index af21337a8..d7c144f4e 100644 --- a/crates/types/tests/structure_contract.rs +++ b/crates/types/tests/structure_contract.rs @@ -13,15 +13,13 @@ fn scan() -> Rc { })) .unwrap() } -/// Tabular coverage for a whole-table summary of column `x`. +/// Tabular coverage for a whole-table summary. fn whole_table() -> asap_types::ir::summary_coverage::SummaryCoverage { use asap_types::ir::summary_coverage::{CoverageRegion, SummaryCoverage}; SummaryCoverage { - source: "t".into(), - input: asap_types::post_asap::SummaryUpdate::column( - asap_types::pre_asap::ColumnRef::Named("x".into()), - ), - reduction: asap_types::pre_asap::Reduction::by(vec![]), + source: Source::Table { + table_ref: "t".into(), + }, regions: vec![CoverageRegion { time_ms: None, population: Default::default(), diff --git a/crates/types/tests/summary_coverage.rs b/crates/types/tests/summary_coverage.rs index e3059677e..3b99e9bba 100644 --- a/crates/types/tests/summary_coverage.rs +++ b/crates/types/tests/summary_coverage.rs @@ -1,13 +1,18 @@ //! Coverage composition preserves gaps and rejects duplicate observations. use asap_types::{ - ir::operator_properties::Reduction, ir::summary_coverage::*, post_asap::SummaryUpdate, - pre_asap::ColumnRef, + ir::operator_properties::Reduction, + ir::summary_coverage::*, + post_asap::SummaryUpdate, + pre_asap::{ColumnRef, Source}, }; +fn table(name: &str) -> Source { + Source::Table { + table_ref: name.into(), + } +} fn coverage(start: i64, end: i64, population: &[(&str, &str)]) -> SummaryCoverage { SummaryCoverage { - source: "flows".into(), - input: SummaryUpdate::column(ColumnRef::Named("latency".into())), - reduction: Reduction::by(vec![0]), + source: table("flows"), regions: vec![CoverageRegion { time_ms: Some(start..end), population: population @@ -61,11 +66,18 @@ fn overlap_and_identity_fail_closed() { ]), Err(CoverageError::PossibleOverlap) ); + assert_eq!( + SummaryCoverage::merge_disjoint(&[ + coverage(0, 2, &[("region", "us")]), + coverage(0, 2, &[("region", "us")]) + ]), + Err(CoverageError::PossibleOverlap) + ); let mut other = coverage(1, 2, &[]); - other.source = "other-flows".into(); + other.source = table("other-flows"); assert_eq!( SummaryCoverage::merge_disjoint(&[coverage(0, 1, &[]), other]), - Err(CoverageError::IncompatibleInput) + Err(CoverageError::SourceMismatch) ); assert_eq!( coverage(2, 1, &[]).validate(), @@ -77,21 +89,17 @@ fn overlap_and_identity_fail_closed() { #[test] fn node_coverage_is_required_checked_and_cleared_by_rewrites() { use asap_types::{ - ir::operator_properties::Source, ir::{ASAPOp, NonASAPOp, Operator, OperatorNode}, post_asap::{SketchAlgorithm, SketchKind, SketchParams}, pre_asap::{DataType, Field, FieldDataType, Schema}, }; let raw = OperatorNode::new_shared(Operator::NonASAP(NonASAPOp::Scan { - source: Source::Table { - table_ref: "flows".into(), - }, + source: table("flows"), predicates: vec![], schema: Schema::new(vec![Field::plain("latency", DataType::Float64, false)]), })) .unwrap(); - let mut declared = coverage(0, 1, &[]); - declared.reduction = Reduction::by(vec![]); + let declared = coverage(0, 1, &[]); assert!((*raw).clone().with_coverage(declared.clone()).is_err()); let state = OperatorNode::new(Operator::ASAP(ASAPOp::SummaryAgg { child: raw, @@ -99,8 +107,8 @@ fn node_coverage_is_required_checked_and_cleared_by_rewrites() { SketchKind::new(SketchAlgorithm::Kll, SketchParams::Kll { k: 200 }), Default::default(), ), - input: declared.input.clone(), - reduction: declared.reduction.clone(), + input: SummaryUpdate::column(ColumnRef::Named("latency".into())), + reduction: Reduction::by(vec![]), grouping: Default::default(), filter: None, })) @@ -116,9 +124,6 @@ fn node_coverage_is_required_checked_and_cleared_by_rewrites() { std::rc::Rc::new(state.clone()) .validate_structure() .unwrap(); - let mut bad = declared; - bad.input = SummaryUpdate::column(ColumnRef::Named("other".into())); - assert!(state.clone().with_coverage(bad).is_err()); let rebuilt = state.map_children(Clone::clone).unwrap(); assert!(rebuilt.coverage.is_none()); } diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md index 1ad87c570..0de7d30fc 100644 --- a/docs/develop_docs/summary-coverage.md +++ b/docs/develop_docs/summary-coverage.md @@ -85,9 +85,7 @@ different coverage. ```rust pub struct SummaryCoverage { - pub source: String, // observation data source; any tabular data, not necessarily time series - pub input: SummaryUpdate, // must equal the producing SummaryAgg.input - pub reduction: Reduction, // must equal the producing SummaryAgg.reduction + pub source: Source, // as named by Scan: Table { table_ref } or TimeSeries { metric } pub regions: Vec, // union of time × population blocks } pub struct CoverageRegion { @@ -102,9 +100,12 @@ Rules: `SummaryAgg` or `SummaryMerge` whose coverage is `None` with `CoverageError::Missing`. Other nodes leave it `None`. The field is an `Option` only because all operators share `OperatorNode`. -- `with_coverage` validates the declaration, requires `State` output - (`NotState`), and checks `input`/`reduction` against a `SummaryAgg` producer - (`ProducerMismatch`). `validate_structure` re-checks it. +- Coverage holds only what the operator does not already record. What is fed + into the state and how it is grouped stay on `SummaryAgg.input` and + `SummaryAgg.reduction`; `SummaryMerge` compares those on its inputs. `source` + uses the same `Source` type as `Scan`, so equal sources compare equal. +- `with_coverage` validates the declaration and requires `State` output + (`NotState`). `validate_structure` re-checks it. - `SummaryMerge` derives its coverage from its inputs. `validate_structure` rejects a retained value that differs from that union. - Rewriting a node's inputs clears its coverage. The rewriter must declare it @@ -116,8 +117,9 @@ Rules: ## Merging -`SummaryCoverage::merge_disjoint` requires equal `source`/`input`/`reduction` -and provably disjoint regions. Two regions are disjoint when their time ranges +`SummaryCoverage::merge_disjoint` requires equal `source` and provably +disjoint regions. `SummaryMerge` additionally requires equal input schemas and +equal `input`/`reduction` on its inputs' producers. Two regions are disjoint when their time ranges do not intersect, or when they give different values for the same population label. Different labels prove nothing. The examples above come out as: @@ -127,10 +129,11 @@ label. Different labels prove nothing. The examples above come out as: | `[0,1)` + `[2,3)` | accepted, **two** regions (gap kept) | | `[0,2)` + `[1,3)` | `PossibleOverlap` | | `region=us` + `region=eu`, same time | accepted, two regions | +| `region=us` + `region=us`, same time | `PossibleOverlap` | | `region=us` + `tier=premium` | `PossibleOverlap` | | `us×[0,1)` + `eu×[1,2)` | accepted, two regions, never widened to `{us,eu}×[0,2)` | | no time bounds + any region of the same population | `PossibleOverlap` | -| different source / input / reduction | `IncompatibleInput` | +| different source | `SourceMismatch` | Adjacent intervals coalesce only when their population maps are identical. Merging an empty input list fails with `EmptyMerge`. @@ -138,8 +141,7 @@ Merging an empty input list fails with `EmptyMerge`. ## Trusted declarations Coverage is declared by the composition rule or catalog that built the -subtree. Nothing is inferred from SQL. Only `input` and `reduction` are checked -against the producer. Population is not compared with `SummaryAgg.filter`, +subtree. Nothing is inferred from SQL. Population is not compared with `SummaryAgg.filter`, `Filter` nodes or `Scan.predicates`. Time bounds cannot be checked, because `TimeRange` stores a relative duration. So a wrong declaration passes: From bf7501be0f1d687312103e214f8754c6c2d2ccae Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 17:37:33 +0000 Subject: [PATCH 08/18] refactor(cost): rename SourceCoverage to ScanSelection SourceCoverage names the rows a physical scan reads for cost comparison, not which observations a summary state holds; rename it so it is not confused with SummaryCoverage. Co-Authored-By: Claude Opus 5.5 --- .../asap-aware-mapping/src/analytical_cost.rs | 77 +++++++++---------- .../src/physical_operator_statistics.rs | 8 +- .../src/physical_plan_cost_model.rs | 12 +-- .../src/query_physical_lowering.rs | 51 ++++++------ crates/asap-aware-mapping/src/storage_io.rs | 2 +- .../src/summary_maintenance_cost/estimator.rs | 8 +- .../src/summary_maintenance_cost/evidence.rs | 2 +- .../src/summary_maintenance_cost/model.rs | 28 +++---- .../tests/physical_handoff_cost.rs | 12 +-- crates/asap-aware-mapping/tests/storage_io.rs | 14 ++-- crates/devtools/src/bin/dag_export.rs | 8 +- .../architecture/physical-plan-integration.md | 2 +- .../architecture/planner-runtime-contract.md | 2 +- .../analytical-resource-cost.md | 20 ++--- 14 files changed, 121 insertions(+), 125 deletions(-) diff --git a/crates/asap-aware-mapping/src/analytical_cost.rs b/crates/asap-aware-mapping/src/analytical_cost.rs index 3c19960a9..ad921d23a 100644 --- a/crates/asap-aware-mapping/src/analytical_cost.rs +++ b/crates/asap-aware-mapping/src/analytical_cost.rs @@ -14,7 +14,7 @@ use serde::{Deserialize, Serialize}; use crate::physical_operator_statistics::{ validate_comparison_scopes, ComparisonScope, EdgeStatistics, OperatorStatistics, - OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, SourceCoverage, + OperatorStatisticsProvider, PromqlEdgeStatistics, PromqlValueKind, ScanSelection, }; /// Version of the analytical formulas applied to evidenced physical plans. @@ -383,9 +383,9 @@ pub struct PhysicalDAGNode { pub operator: PhysicalOperator, pub children: Vec, /// Exact comparison-scope coverage consumed by a scan. Non-scan nodes - /// leave this empty. Reusing `SourceCoverage` prevents a physical plan + /// leave this empty. Reusing `ScanSelection` prevents a physical plan /// from naming a source independently of its snapshot and predicates. - pub source_coverage: Option, + pub scan_selection: Option, /// Maximum transient edge buffer, distinct from logical `output_bytes`. pub output_buffer_bytes: u64, /// State that remains live after this node finishes (zero for ordinary @@ -573,9 +573,10 @@ pub fn estimate_physical_dag_with_cache( let node_statistics = &resolved_statistics[id]; match node.operator { PhysicalOperator::Scan => { - let coverage = node.source_coverage.as_ref().ok_or_else(|| { - AnalyticalCostError::MissingScanSourceCoverage(node.id.clone()) - })?; + let coverage = node + .scan_selection + .as_ref() + .ok_or_else(|| AnalyticalCostError::MissingScanSelection(node.id.clone()))?; if !scope.sources.contains(coverage) { return Err(AnalyticalCostError::ScanOutsideComparisonScope( node.id.clone(), @@ -585,9 +586,9 @@ pub fn estimate_physical_dag_with_cache( consumed_sources.push(coverage); } } - _ if node.source_coverage.is_some() => { + _ if node.scan_selection.is_some() => { return Err(AnalyticalCostError::InvalidPhysicalDAG( - "only scan nodes may declare source coverage", + "only scan nodes may declare scan selection", )); } _ => {} @@ -2199,9 +2200,9 @@ pub enum AnalyticalCostError { UnsupportedSummaryOperation(&'static str), #[error("required comparison-scope field {0} is missing")] MissingComparisonScope(&'static str), - #[error("scan node {0} does not declare source coverage")] - MissingScanSourceCoverage(String), - #[error("scan node {0} reads source coverage outside the comparison scope")] + #[error("scan node {0} does not declare scan selection")] + MissingScanSelection(String), + #[error("scan node {0} reads scan selection outside the comparison scope")] ScanOutsideComparisonScope(String), #[error("raw and candidate comparison scopes differ in {0}")] ComparisonScopeMismatch(&'static str), @@ -2234,7 +2235,7 @@ mod tests { use crate::physical_operator_statistics::{ validate_comparison_scopes, BinaryEdgeStatistics, ComparisonScope, EdgeStatistics, OperatorStatistics, PartitionStatistics, PromqlBinaryEdgeStatistics, PromqlEdgeStatistics, - PromqlUnaryEdgeStatistics, PromqlValueKind, SourceCoverage, UnaryEdgeStatistics, + PromqlUnaryEdgeStatistics, PromqlValueKind, ScanSelection, UnaryEdgeStatistics, }; /// Analytical estimates reuse the shared dimensions while preserving exact @@ -2475,7 +2476,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(comparison_scope().sources[0].clone()), + scan_selection: Some(comparison_scope().sources[0].clone()), output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2484,7 +2485,7 @@ mod tests { id: "filter".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2814,7 +2815,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2823,7 +2824,7 @@ mod tests { id: "left".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2832,7 +2833,7 @@ mod tests { id: "right".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2841,7 +2842,7 @@ mod tests { id: "root".into(), operator: PhysicalOperator::Concat, children: vec!["left".into(), "right".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2905,7 +2906,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::Once, @@ -2914,7 +2915,7 @@ mod tests { id: "state".into(), operator: aggregate_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 16, retained_bytes: 32, execution: ExecutionMultiplicity::Once, @@ -2926,7 +2927,7 @@ mod tests { offset: 0, }, children: vec!["state".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 16, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -2987,7 +2988,7 @@ mod tests { lookback: Some(DurationMs(300_000)), as_of: Some(TimestampMs(1_000)), }, - sources: vec![SourceCoverage { + sources: vec![ScanSelection { source: Source::Table { table_ref: "metrics".into(), }, @@ -3059,7 +3060,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3068,7 +3069,7 @@ mod tests { id: "filter".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3134,7 +3135,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3143,7 +3144,7 @@ mod tests { id: "filter".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 4, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3197,7 +3198,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3206,7 +3207,7 @@ mod tests { id: "filter".into(), operator: filter_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3245,7 +3246,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(SourceCoverage { + scan_selection: Some(ScanSelection { source: asap_types::pre_asap::query_expr::Source::Table { table_ref: "other_metrics".into(), }, @@ -3283,7 +3284,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3302,9 +3303,7 @@ mod tests { assert_eq!( estimate_physical_dag(&nodes, "scan", &comparison_scope(), &provided), - Err(AnalyticalCostError::MissingScanSourceCoverage( - "scan".into() - )) + Err(AnalyticalCostError::MissingScanSelection("scan".into())) ); } @@ -3312,7 +3311,7 @@ mod tests { fn physical_dag_rejects_an_unconsumed_scope_source() { let mut scope = comparison_scope(); let coverage = scope.sources[0].clone(); - scope.sources.push(SourceCoverage { + scope.sources.push(ScanSelection { source: asap_types::pre_asap::query_expr::Source::Table { table_ref: "auxiliary".into(), }, @@ -3324,7 +3323,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3357,7 +3356,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3366,7 +3365,7 @@ mod tests { id: "aggregate".into(), operator: aggregate_operator(), children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 16, retained_bytes: 32, execution: ExecutionMultiplicity::Once, @@ -3411,7 +3410,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 10, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -3642,7 +3641,7 @@ mod tests { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(comparison_scope().sources[0].clone()), + scan_selection: Some(comparison_scope().sources[0].clone()), output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, diff --git a/crates/asap-aware-mapping/src/physical_operator_statistics.rs b/crates/asap-aware-mapping/src/physical_operator_statistics.rs index 9cf8045bb..7be1d292c 100644 --- a/crates/asap-aware-mapping/src/physical_operator_statistics.rs +++ b/crates/asap-aware-mapping/src/physical_operator_statistics.rs @@ -26,12 +26,12 @@ pub struct ComparisonScope { pub horizon: DurationMs, pub recurrence: QueryRecurrence, pub time_selection: TimeSelection, - pub sources: Vec, + pub sources: Vec, } /// Exact source selection covered by a physical plan. #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct SourceCoverage { +pub struct ScanSelection { pub source: Source, /// Provider-owned stable identifier for the physical source contents, /// such as a catalog snapshot, table version, or object generation. @@ -53,7 +53,7 @@ impl ComparisonScope { query: &QueryWorkloadEntry, planning_time: TimestampMs, horizon: DurationMs, - sources: Vec, + sources: Vec, ) -> Result { let scope = Self { data_arrival: data.arrival, @@ -87,7 +87,7 @@ impl ComparisonScope { .any(|(index, source)| self.sources[..index].contains(source)) { return Err(AnalyticalCostError::MissingComparisonScope( - "duplicate source coverage", + "duplicate scan selection", )); } if self diff --git a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs index 307fb9a61..509e6a13a 100644 --- a/crates/asap-aware-mapping/src/physical_plan_cost_model.rs +++ b/crates/asap-aware-mapping/src/physical_plan_cost_model.rs @@ -367,7 +367,7 @@ mod tests { use crate::analytical_cost::{ExecutionMultiplicity, PhysicalDAGNode, PhysicalOperator}; use crate::physical_operator_statistics::{ - EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; use crate::replacement::ReplacementStrategy; @@ -438,7 +438,7 @@ mod tests { lookback: Some(DurationMs(10_000)), as_of: Some(TimestampMs(1_000)), }, - sources: vec![SourceCoverage { + sources: vec![ScanSelection { source: Source::Table { table_ref: "events".into(), }, @@ -506,7 +506,7 @@ mod tests { id: "candidate-scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(scope.sources[0].clone()), + scan_selection: Some(scope.sources[0].clone()), output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::Once, @@ -518,7 +518,7 @@ mod tests { accumulator_count: 1, }, children: vec!["candidate-scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 8, retained_bytes: 8, execution: ExecutionMultiplicity::Once, @@ -527,7 +527,7 @@ mod tests { id: "candidate-read".into(), operator: PhysicalOperator::PassThrough, children: vec!["candidate-state".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 8, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -1006,7 +1006,7 @@ mod tests { ) -> Result { let mut dag = self.0.summary_physical_dag(snapshot, summary, target)?; dag.nodes[0] - .source_coverage + .scan_selection .as_mut() .unwrap() .source_snapshot_id = "other".into(); diff --git a/crates/asap-aware-mapping/src/query_physical_lowering.rs b/crates/asap-aware-mapping/src/query_physical_lowering.rs index 1b8e5a751..65b7fab93 100644 --- a/crates/asap-aware-mapping/src/query_physical_lowering.rs +++ b/crates/asap-aware-mapping/src/query_physical_lowering.rs @@ -9,7 +9,7 @@ use crate::analytical_cost::{ PromqlSeriesSampleKind, PromqlVectorCardinality, }; use crate::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, + ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, }; pub struct PhysicalNodeRequest<'a> { @@ -18,7 +18,7 @@ pub struct PhysicalNodeRequest<'a> { pub occurrence: usize, pub synthetic: bool, pub children: &'a [String], - pub source_coverage: Option<&'a SourceCoverage>, + pub scan_selection: Option<&'a ScanSelection>, } pub trait PhysicalNodeEvidenceProvider { @@ -78,7 +78,7 @@ pub fn lower_query_physical_dag( occurrence: usize, synthetic: bool, children: &[String], - source_coverage: Option<&SourceCoverage>, + scan_selection: Option<&ScanSelection>, ) -> Result { let evidence = self.provider.evidence(PhysicalNodeRequest { logical_node: query, @@ -86,7 +86,7 @@ pub fn lower_query_physical_dag( occurrence, synthetic, children, - source_coverage, + scan_selection, })?; if evidence.physical_id.is_empty() { return Err(AnalyticalCostError::InvalidPhysicalDAG( @@ -101,14 +101,14 @@ pub fn lower_query_physical_dag( evidence: PhysicalNodeEvidence, operator: PhysicalOperator, children: Vec, - source_coverage: Option, + scan_selection: Option, ) -> Result { let id = evidence.physical_id.clone(); let node = PhysicalDAGNode { id: id.clone(), operator, children, - source_coverage, + scan_selection, output_buffer_bytes: evidence.output_buffer_bytes, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -837,7 +837,7 @@ fn validate_source_consumption( let consumed = nodes .iter() .filter(|node| matches!(node.operator, PhysicalOperator::Scan)) - .filter_map(|node| node.source_coverage.as_ref()) + .filter_map(|node| node.scan_selection.as_ref()) .collect::>(); for coverage in &consumed { if !scope.sources.contains(coverage) { @@ -959,7 +959,7 @@ fn bind_scan_coverage( source: &asap_types::pre_asap::Source, predicates: &[asap_types::pre_asap::Predicate], scope: &ComparisonScope, -) -> Result { +) -> Result { let mut matches = scope.sources.iter().filter(|coverage| { coverage.source == *source && coverage.predicates == predicates @@ -971,7 +971,7 @@ fn bind_scan_coverage( .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; if matches.any(|candidate| candidate != &coverage) { return Err(AnalyticalCostError::InvalidPhysicalDAG( - "scan source coverage is ambiguous", + "scan selection is ambiguous", )); } Ok(coverage) @@ -981,7 +981,7 @@ fn bind_info_coverage( node_id: &str, selector: &[asap_types::pre_asap::InfoMatcher], scope: &ComparisonScope, -) -> Result { +) -> Result { use asap_types::pre_asap::{CompareOpKind, Source}; let mut metric: Option<&str> = None; @@ -1009,7 +1009,7 @@ fn bind_info_coverage( .ok_or_else(|| AnalyticalCostError::ScanOutsideComparisonScope(node_id.into()))?; if matches.next().is_some() { return Err(AnalyticalCostError::InvalidPhysicalDAG( - "info source coverage is ambiguous", + "info scan selection is ambiguous", )); } Ok(coverage) @@ -1383,7 +1383,7 @@ mod tests { } } - fn scope(sources: Vec) -> ComparisonScope { + fn scope(sources: Vec) -> ComparisonScope { ComparisonScope { data_arrival: DataArrival::AtRest, planning_time: TimestampMs(1_000), @@ -1404,8 +1404,8 @@ mod tests { fn coverage( source: asap_types::pre_asap::Source, predicates: Vec, - ) -> SourceCoverage { - SourceCoverage { + ) -> ScanSelection { + ScanSelection { source, source_snapshot_id: "snapshot-1".into(), predicates, @@ -1414,7 +1414,7 @@ mod tests { } #[test] - fn info_source_coverage_includes_symbolic_selector_matchers() { + fn info_scan_selection_includes_symbolic_selector_matchers() { use asap_types::pre_asap::{CompareOpKind, InfoMatcher, Source}; let selector = vec![InfoMatcher { @@ -1422,7 +1422,7 @@ mod tests { op: CompareOpKind::Eq, value: "prod".into(), }]; - let info_coverage = SourceCoverage { + let info_coverage = ScanSelection { source: Source::TimeSeries { metric: "target_info".into(), }, @@ -1611,10 +1611,7 @@ mod tests { )); let physical_scan = &dag.nodes[0]; assert_eq!(physical_scan.id, "query-2-scan"); - assert_eq!( - physical_scan.source_coverage, - Some(scope.sources[0].clone()) - ); + assert_eq!(physical_scan.scan_selection, Some(scope.sources[0].clone())); assert_eq!(physical_scan.output_buffer_bytes, 1_024); assert_ne!( physical_scan.output_buffer_bytes, @@ -1661,13 +1658,13 @@ mod tests { left: Rc::clone(&shared), right: Rc::clone(&shared), }); - let source_coverage = coverage( + let scan_selection = coverage( Source::Table { table_ref: "dimensions".into(), }, vec![], ); - let independent_scope = scope(vec![source_coverage.clone()]); + let independent_scope = scope(vec![scan_selection.clone()]); let scan_statistics = scan_stats(edge(100, 800), 800); let join_statistics = OperatorStatistics::HashJoin { edges: BinaryEdgeStatistics { @@ -1703,7 +1700,7 @@ mod tests { statistics, }) }; - let shared_scope = scope(vec![source_coverage]); + let shared_scope = scope(vec![scan_selection]); let no_cache = crate::analytical_cost::CacheProfile::no_cache(); let shared_dag = lower_query_physical_dag(&root, &shared_scope, &shared_provider).unwrap(); assert_eq!(shared_dag.nodes.len(), 2); @@ -1860,13 +1857,13 @@ mod tests { child: limit, }); - let source_coverage = coverage( + let scan_selection = coverage( Source::Table { table_ref: "events".into(), }, vec![], ); - let scope = scope(vec![source_coverage]); + let scope = scope(vec![scan_selection]); let scan_statistics = scan_stats(edge(1_000, 8_000), 8_000); let dedup_statistics = OperatorStatistics::HashDeduplicate { edges: unary_edges(edge(800, 3_200), edge(500, 2_000)), @@ -2032,7 +2029,7 @@ mod tests { assert_eq!( duplicate_scope.validate(), Err(AnalyticalCostError::MissingComparisonScope( - "duplicate source coverage" + "duplicate scan selection" )) ); @@ -2142,7 +2139,7 @@ mod tests { assert_eq!( lower_query_physical_dag(&root, &ambiguous_scope, &scripted(&conflicting)), Err(AnalyticalCostError::InvalidPhysicalDAG( - "scan source coverage is ambiguous" + "scan selection is ambiguous" )) ); diff --git a/crates/asap-aware-mapping/src/storage_io.rs b/crates/asap-aware-mapping/src/storage_io.rs index 646a3532c..125ec915b 100644 --- a/crates/asap-aware-mapping/src/storage_io.rs +++ b/crates/asap-aware-mapping/src/storage_io.rs @@ -121,7 +121,7 @@ pub fn estimate_storage_io( profile: &StorageIoProfile, evidence_version: &str, ) -> Result { - // Also prove source coverage, edge consistency, execution legality and DAG + // Also prove scan selection, edge consistency, execution legality and DAG // identity before using supplementary deployment evidence. estimate_physical_dag(&dag.nodes, &dag.root, scope, dag)?; let evaluations = scope.validate()?; diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs index 5bf42d2e4..eac94a2cb 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/estimator.rs @@ -86,14 +86,14 @@ pub(super) fn estimate_heterogeneous_summary( }; let inputs = node_evidence.inputs.validate()?; validate_arrival_rate(scope.data_arrival, inputs.ingestion_rate_per_second)?; - match node_evidence.source_coverage_index { + match node_evidence.scan_selection_index { Some(index) => { let declared = scope .sources .get(index) .ok_or(AnalyticalCostError::MissingComparisonScope( - "summary source coverage", + "summary scan selection", ))?; if !matches!(&child.expr, SummaryExpr::KeepPreAsap(_)) || inputs.initial_input_rows != raw.planning_time_input_rows @@ -178,7 +178,7 @@ pub(super) fn estimate_heterogeneous_summary( let bootstrap_extra_rows = bootstrap .checked_sub(inputs.initial_input_rows) .ok_or(AnalyticalCostError::Overflow)?; - let source_scan_bytes = if node_evidence.source_coverage_index.is_some() { + let source_scan_bytes = if node_evidence.scan_selection_index.is_some() { inputs .initial_source_scan_bytes .checked_add( @@ -223,7 +223,7 @@ pub(super) fn estimate_heterogeneous_summary( .checked_add(state_bytes) .ok_or(AnalyticalCostError::Overflow)?; } - if let Some(source_index) = node_evidence.source_coverage_index { + if let Some(source_index) = node_evidence.scan_selection_index { if node_evidence.bootstrap_read_identity.is_empty() { return Err(AnalyticalCostError::MissingOrStale( "bootstrap_read_identity", diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs index f396e3cd8..7c8607685 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/evidence.rs @@ -161,7 +161,7 @@ pub struct SummaryAggregateEvidence { /// Index into `ComparisonScope.sources` when this state bootstraps directly /// from storage. `None` means its input is an already-materialized child /// edge and therefore has no additional source read. - pub source_coverage_index: Option, + pub scan_selection_index: Option, /// Provider-owned identity of the physical bootstrap read. Equal source /// coverage alone does not prove two independent builds share I/O. pub bootstrap_read_identity: String, diff --git a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs b/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs index 331b7e1a5..7859db9b8 100644 --- a/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs +++ b/crates/asap-aware-mapping/src/summary_maintenance_cost/model.rs @@ -165,19 +165,19 @@ fn validate_physical_scope_coverage( .filter(|node| node.operator == PhysicalOperator::Scan) { let coverage = node - .source_coverage + .scan_selection .as_ref() - .ok_or_else(|| AnalyticalCostError::MissingScanSourceCoverage(node.id.clone()))?; + .ok_or_else(|| AnalyticalCostError::MissingScanSelection(node.id.clone()))?; let Some(index) = scope.sources.iter().position(|value| value == coverage) else { return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical source coverage", + "physical scan selection", )); }; covered.insert(index); } if covered.len() != scope.sources.len() { return Err(AnalyticalCostError::ComparisonScopeMismatch( - "physical source coverage", + "physical scan selection", )); } Ok(()) @@ -939,7 +939,7 @@ mod tests { query, asap_types::workload::TimestampMs(planning_time_ms), asap_types::workload::DurationMs(horizon_ms), - vec![crate::physical_operator_statistics::SourceCoverage { + vec![crate::physical_operator_statistics::ScanSelection { source: Source::TimeSeries { metric: "metrics".into(), }, @@ -1611,7 +1611,7 @@ mod tests { id: "raw-concat".into(), operator: PhysicalOperator::Concat, children: vec!["raw-scan".into(), "raw-scan-2".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::Once, @@ -1693,7 +1693,7 @@ mod tests { let mut extra = streaming_scope(); extra .sources - .push(crate::physical_operator_statistics::SourceCoverage { + .push(crate::physical_operator_statistics::ScanSelection { source: Source::TimeSeries { metric: "unused".into(), }, @@ -1720,7 +1720,7 @@ mod tests { let mut info_scope = streaming_scope(); info_scope .sources - .push(crate::physical_operator_statistics::SourceCoverage { + .push(crate::physical_operator_statistics::ScanSelection { source: Source::TimeSeries { metric: "target_info".into(), }, @@ -2057,7 +2057,7 @@ mod tests { physical_id: "left-state".into(), input: test_edge(), output: test_edge(), - source_coverage_index: Some(0), + scan_selection_index: Some(0), bootstrap_read_identity: "left-bootstrap".into(), inputs: streaming_inputs(), insert_cpu_ops: streaming_cpu().insert_cpu_ops.unwrap(), @@ -2084,7 +2084,7 @@ mod tests { physical_id: "right-state".into(), input: test_edge(), output: test_edge(), - source_coverage_index: Some(0), + scan_selection_index: Some(0), bootstrap_read_identity: "right-bootstrap".into(), inputs: second_inputs, insert_cpu_ops: second_cpu.insert_cpu_ops.unwrap(), @@ -2150,7 +2150,7 @@ mod tests { 64.0 ); - // Equal SourceCoverage does not imply that two independent state + // Equal ScanSelection does not imply that two independent state // builds share one physical read. Only a provider-owned read identity // permits scan de-duplication. let mut shared_read = model; @@ -3264,7 +3264,7 @@ mod tests { id: "raw-scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(scope.sources[0].clone()), + scan_selection: Some(scope.sources[0].clone()), output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::Once, @@ -3415,7 +3415,7 @@ mod tests { physical_id: format!("agg-{node:p}"), input: test_edge(), output: test_edge(), - source_coverage_index: source_root.then_some(0), + scan_selection_index: source_root.then_some(0), bootstrap_read_identity: if source_root { "shared-bootstrap".into() } else { @@ -3637,7 +3637,7 @@ mod tests { &entry, asap_types::workload::TimestampMs(0), asap_types::workload::DurationMs(5_000), - vec![crate::physical_operator_statistics::SourceCoverage { + vec![crate::physical_operator_statistics::ScanSelection { source: Source::TimeSeries { metric: "metrics".into(), }, diff --git a/crates/asap-aware-mapping/tests/physical_handoff_cost.rs b/crates/asap-aware-mapping/tests/physical_handoff_cost.rs index 0885db71c..a8a9ed42d 100644 --- a/crates/asap-aware-mapping/tests/physical_handoff_cost.rs +++ b/crates/asap-aware-mapping/tests/physical_handoff_cost.rs @@ -3,7 +3,7 @@ use asap_aware_mapping::analytical_cost::{ PhysicalOperator, }; use asap_aware_mapping::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; use asap_types::pre_asap::query_expr::Source; use asap_types::workload::{ @@ -12,7 +12,7 @@ use asap_types::workload::{ use std::collections::HashMap; fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { - let coverage = SourceCoverage { + let coverage = ScanSelection { source: Source::Table { table_ref: "events".into(), }, @@ -49,7 +49,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -58,7 +58,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "left".into(), operator: PhysicalOperator::PassThrough, children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -67,7 +67,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "right".into(), operator: PhysicalOperator::PassThrough, children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -76,7 +76,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "root".into(), operator: PhysicalOperator::Concat, children: vec!["left".into(), "right".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, diff --git a/crates/asap-aware-mapping/tests/storage_io.rs b/crates/asap-aware-mapping/tests/storage_io.rs index a12118f09..e9290d20a 100644 --- a/crates/asap-aware-mapping/tests/storage_io.rs +++ b/crates/asap-aware-mapping/tests/storage_io.rs @@ -3,7 +3,7 @@ use asap_aware_mapping::analytical_cost::{ PhysicalOperator, }; use asap_aware_mapping::physical_operator_statistics::{ - ComparisonScope, EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + ComparisonScope, EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; use asap_types::pre_asap::query_expr::Source; use asap_types::workload::{ @@ -12,7 +12,7 @@ use asap_types::workload::{ use std::collections::HashMap; fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { - let coverage = SourceCoverage { + let coverage = ScanSelection { source: Source::Table { table_ref: "events".into(), }, @@ -49,7 +49,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "scan".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -58,7 +58,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "left".into(), operator: PhysicalOperator::PassThrough, children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -67,7 +67,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "right".into(), operator: PhysicalOperator::PassThrough, children: vec!["scan".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -76,7 +76,7 @@ fn fixture() -> (EvidenceBackedPhysicalDAG, ComparisonScope) { id: "root".into(), operator: PhysicalOperator::Concat, children: vec!["left".into(), "right".into()], - source_coverage: None, + scan_selection: None, output_buffer_bytes: 0, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, @@ -353,7 +353,7 @@ fn storage_node_identity_statistics_and_calibration_provenance_are_bound() { .get_mut("scan") .unwrap() .node - .source_coverage + .scan_selection .as_mut() .unwrap() .source_snapshot_id = "another-source".into(); diff --git a/crates/devtools/src/bin/dag_export.rs b/crates/devtools/src/bin/dag_export.rs index b0615f017..2728dab12 100644 --- a/crates/devtools/src/bin/dag_export.rs +++ b/crates/devtools/src/bin/dag_export.rs @@ -152,7 +152,7 @@ struct ComparisonScopeEvidence { time_scope: String, lookback_ms: Option, as_of_ms: Option, - sources: Vec, + sources: Vec, #[serde(default = "CacheProfile::no_cache")] cache_profile: CacheProfile, } @@ -1700,7 +1700,7 @@ mod tests { ExecutionMultiplicity, PhysicalDAGNode, PhysicalOperator, }; use asap_aware_mapping::physical_operator_statistics::{ - EdgeStatistics, OperatorStatistics, SourceCoverage, UnaryEdgeStatistics, + EdgeStatistics, OperatorStatistics, ScanSelection, UnaryEdgeStatistics, }; use asap_aware_mapping::query_physical_lowering::lower_query_physical_dag; use asap_devtools::PromqlError; @@ -2178,7 +2178,7 @@ mod tests { time_scope: "longitudinal".into(), lookback_ms: Some(10_000), as_of_ms: Some(1_000), - sources: vec![SourceCoverage { + sources: vec![ScanSelection { source: Source::Table { table_ref: "events".into(), }, @@ -2261,7 +2261,7 @@ mod tests { id: "summary-read".into(), operator: PhysicalOperator::Scan, children: vec![], - source_coverage: Some(coverage), + scan_selection: Some(coverage), output_buffer_bytes: 2_400, retained_bytes: 0, execution: ExecutionMultiplicity::PerEvaluation, diff --git a/docs/design_docs/architecture/physical-plan-integration.md b/docs/design_docs/architecture/physical-plan-integration.md index 7fffc9fdf..057ba7c17 100644 --- a/docs/design_docs/architecture/physical-plan-integration.md +++ b/docs/design_docs/architecture/physical-plan-integration.md @@ -75,7 +75,7 @@ lose post-ASAP summary implementations, while aligning it directly with Physical lowering is complete only when it recursively lowers the entire selected candidate DAG. It must: -1. preserve the semantics and source coverage of the logical candidate; +1. preserve the semantics and scan selection of the logical candidate; 2. select an explicit physical algorithm for every logical operation; 3. carry algorithm configuration on the physical operator rather than in a generic statistics record; diff --git a/docs/design_docs/architecture/planner-runtime-contract.md b/docs/design_docs/architecture/planner-runtime-contract.md index 2d071e654..58e3f06b6 100644 --- a/docs/design_docs/architecture/planner-runtime-contract.md +++ b/docs/design_docs/architecture/planner-runtime-contract.md @@ -73,7 +73,7 @@ and rollback are not an end-to-end Planner protocol. 2. A physical-plan provider maps those candidates to executor-feasible complete alternatives. Unsupported candidates are omitted or explicitly rejected. 3. The provider binds a stable alternative identity and complete evidence: - source coverage, input/output edges, operation counts, update and bootstrap + scan selection, input/output edges, operation counts, update and bootstrap fanout, retained state, CPU, memory, I/O, and accuracy facts. 4. ASAPPlanner keeps constructible candidates with missing evidence visible in `CandidateLogicalASAPDAGs` but does not certify unknown accuracy. The diff --git a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md index 520501731..b742a8970 100644 --- a/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md +++ b/docs/design_docs/proposals/asap-aware-mapping/analytical-resource-cost.md @@ -365,11 +365,11 @@ coverage set must equal the scope source set: a Scan query with an empty scope, or a source-free query with a non-empty scope, fails closed. Empty snapshot identifiers, invalid recurrence, or a zero horizon also fail closed. -Every reachable physical `Scan` carries one exact `SourceCoverage` copied from +Every reachable physical `Scan` carries one exact `ScanSelection` copied from this scope. That coverage includes the existing `Source`, its provider-owned snapshot ID, and canonical ordinary predicates or info-metric matchers. A scan with no coverage, or coverage not present in `ComparisonScope.sources`, makes the plan unavailable. Other -operators cannot declare source coverage. This prevents a DAG over source B +operators cannot declare scan selection. This prevents a DAG over source B from being estimated under source A's comparison scope. ## General DAG costing @@ -554,7 +554,7 @@ counts, releases transient output after its last consumer, and keeps retained state live. Consequently a shared scan is charged once per execution and a fan-out's memory includes the outputs that really coexist. -Each estimate independently requires the semantic set of source coverages on +Each estimate independently requires the semantic set of scan selections on its reachable Scan nodes to equal `ComparisonScope.sources`. Multiple physical Scans may repeat one coverage, but no scope source may be omitted and no Scan may add another coverage. This invariant is enforced by the estimator itself, @@ -596,7 +596,7 @@ It consumes the existing query and physical-operator enums; it does not introduce a parallel logical operator vocabulary. For every occurrence, the lowerer sends a `PhysicalNodeRequest` containing the logical node, selected existing `PhysicalOperator`, occurrence and synthetic-role metadata, already-lowered -child physical IDs, and any source coverage to a +child physical IDs, and any scan selection to a `PhysicalNodeEvidenceProvider`. The provider atomically returns its own stable `physical_id`, the authoritative `OperatorStatistics`, and explicit `output_buffer_bytes`; logical edge bytes are never substituted for an @@ -604,14 +604,14 @@ allocation. Missing evidence makes the entire query unavailable. The returned `EvidenceBackedPhysicalDAG` snapshots this evidence so costing does not re-read a live catalog after lowering. -Each lowered Scan is bound to exactly one `SourceCoverage` in the comparison +Each lowered Scan is bound to exactly one `ScanSelection` in the comparison scope by the existing source and canonical predicate values. The bound value therefore also supplies the provider-owned snapshot ID. Zero matches fail as outside scope; multiple matching coverages fail as ambiguous rather than choosing an arbitrary snapshot. When a predicate-bearing logical Scan expands to Scan → Filter, the synthetic Scan has its own physical ID, statistics, and buffer evidence and carries that exact coverage; the Filter has separate -evidence and no source coverage. +evidence and no scan selection. `ComparisonScope.sources` is an order-independent set of semantic coverages; duplicates are invalid. After lowering, every reachable physical Scan must use a member of that set and every member must be used by at least one Scan. @@ -837,7 +837,7 @@ its complete evidence and physical child identities also agree. The cost model holds owning `Rc` references for bound target and summary roots, so pointer keys cannot become stale and alias a later allocation. -A `SummaryAgg` that reads storage declares `source_coverage_index = Some(i)`, +A `SummaryAgg` that reads storage declares `scan_selection_index = Some(i)`, a non-empty bootstrap-read identity, and positive physical source bytes. An aggregate over an already-materialized summary edge declares `None`, an empty read identity, and zero source bytes. Its logical input rows and bytes remain @@ -854,7 +854,7 @@ those evolving evaluations over the complete horizon. Marking its nodes rejected. Validation follows only nodes reachable from the physical root. If the raw algorithm intentionally reads the same semantic source more than once, each reachable scan carries the same evolved source statistics and is charged -separately; equal source coverage does not deduplicate physical I/O. +separately; equal scan selection does not deduplicate physical I/O. This raw-evolution contract currently supports exactly one distinct source coverage. A multi-source streaming target is unavailable until per-source @@ -962,14 +962,14 @@ bound physical DAG for a `SummaryExpr` candidate. The deployment implements `PlannerPhysicalPlanProvider`: query-node evidence is consumed atomically by the generic query lowerer, while summary binding returns a complete `EvidenceBackedPhysicalDAG`, including embedded raw work, build/read operators, retained -state, execution multiplicity, and source coverage. The adapter calls +state, execution multiplicity, and scan selection. The adapter calls `estimate_physical_dag_comparison`; it never calls `DefaultCostModel` or a structural-node-count fallback for final cost. A candidate is exposed to global selection only when both complete DAGs are valid and its calibrated cost is strictly below the raw baseline. Missing or stale evidence, an unknown physical algorithm, invalid edges, incomplete -source coverage, or a candidate that is not cheaper yields `None`. When no +scan selection, or a candidate that is not cheaper yields `None`. When no candidate remains, `chosen = None` preserves the raw pre-ASAP target. Logical CSE share/recompute rewrites are not complete physical alternatives: From 5b74bc08fd6338198b6486cbfe69ed4709de8480 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:32:26 +0000 Subject: [PATCH 09/18] docs: turn summary coverage into a design document Move it to docs/design_docs/proposals with problem and motivation, requirements, design, alternatives and key code interfaces. Co-Authored-By: Claude Opus 5.5 --- docs/design_docs/proposals/README.md | 1 + .../design_docs/proposals/summary-coverage.md | 224 ++++++++++++++++++ docs/develop_docs/summary-coverage.md | 172 -------------- 3 files changed, 225 insertions(+), 172 deletions(-) create mode 100644 docs/design_docs/proposals/summary-coverage.md delete mode 100644 docs/develop_docs/summary-coverage.md diff --git a/docs/design_docs/proposals/README.md b/docs/design_docs/proposals/README.md index cb44fadbb..fc7fde96f 100644 --- a/docs/design_docs/proposals/README.md +++ b/docs/design_docs/proposals/README.md @@ -11,3 +11,4 @@ extensions. A design document is not a promise of downstream runtime support. - [Operator sharing](operator-sharing.md) - [Decoupling operators from scalar expressions](decoupling_op_and_expr.md) - [ASAPPlanner layering](planner-layering.md) +- [Summary coverage](summary-coverage.md) diff --git a/docs/design_docs/proposals/summary-coverage.md b/docs/design_docs/proposals/summary-coverage.md new file mode 100644 index 000000000..a3520734d --- /dev/null +++ b/docs/design_docs/proposals/summary-coverage.md @@ -0,0 +1,224 @@ +# Summary Coverage + +> Status: implemented in `ir::summary_coverage` (#567), with `SummaryMerge` +> derivation (#560) and logical transport/CSE (#537). Checking declared +> population against filters is open ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). +> Audience: planner designers and architects. +> Companions: [Operator sharing](operator-sharing.md) §2.1 (schema model), +> [ASAPPlanner layering](planner-layering.md) Pass 2 (window composition). + +## Goal and problem + +Record **which observations a summary state was built from**, its time range and +population, so the planner can tell when combining or reusing summary states +is correct. + +[Operator sharing](operator-sharing.md) §2.1 gives every edge one `Schema`. For +a summary edge, the schema records the field layout and the committed state +type, such as `(job: Utf8, state: KLL{k=200})`. It deliberately leaves out +filters, group keys and windows. That is enough while a summary is consumed +right after its producer. It stops being enough once +[planner layering](planner-layering.md) composes existing states: + +- Pass 2's window-composition rule merges tumbling or EH summaries into a query + window. +- `SummaryMerge` combines partial states. +- Sub-DAG sharing, and the reuse of ingested panes, hand one state to several + consumers. + +In these cases only the schema is left to compare. Every example below uses +two states with **equal schemas**, `Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}))`. + +**Example 1: time.** + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| latency, `[00:00, 00:01)` | latency, `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | +| latency, `[00:00, 00:02)` | latency, `[00:01, 00:03)` | **wrong**: `[00:01, 00:02)` is counted twice | +| latency, `[00:00, 00:01)` | latency, `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`, not for the continuous `[0,3)` | + +`Schema.time_index` is a column position. A KLL state has no timestamp column, +so the schema cannot tell these apart. + +**Example 2: population (label values).** + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| `region='us'` | `region='eu'` | correct: `us ∪ eu` within each `job` | +| `region='us'` | `tier='premium'` | **wrong**: premium US requests are in both | +| `region='us'` | `region='us'` | **wrong**: everything is counted twice | + +`region` is a filter label, not an output column. The `job` field says how the +state is grouped, not which rows contributed. + +**Example 3: time and population together.** Merging `us × [0,1)` with +`eu × [1,2)` covers exactly those two blocks. Recording one time range and one +label set would give `{us,eu} × [0,2)`, which claims data that was never read. + +**Example 4: reuse.** For `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) +GROUP BY job`, a stored state with a matching schema could hold the right data, +EU data, or only 10:00–10:03. The schema shows that the state *type* fits, not +that the *contents* fit. + +Schema equality is necessary but not sufficient. Without this metadata, the +planner must either refuse every composition or accept silent double counting +and missing data. + +### Requirements + +1. Represent time and population **jointly**, per block, never as independent + bounds. +2. Accept a merge only when the inputs are **provably disjoint**, and fail + closed otherwise. Merging does not imply that a summary family can remove + duplicates. +3. Leave `Schema` and its equality unchanged. +4. Duplicate nothing the operator already records. +5. Support sources without a time column (plain tables). + +## Design + +### Coverage is a node property, beside the schema + +```text +OperatorNode +├── schema: Schema what each output row looks like (operator sharing §2.1) +├── guarantee, timing accuracy and execution phase (§2.2, §2.3) +└── coverage: Option which observations the state holds +``` + +Coverage is not part of `Schema`. `SummaryMerge` requires equal input schemas, +and the inputs of every useful merge (`[0,1)` + `[1,2)`) have different coverage. +It is also not an operator parameter: a merge *derives* it from its inputs, like +the schema. + +### What coverage records + +A `SummaryCoverage` names one observation `source`, using the same `Source` as +`Scan`: a table or a time series. It holds a **union of regions**. Each region +pairs: + +- `time_ms`: half-open bounds on the source's time column, or `None` for no + time restriction; +- `population`: a conjunction of non-null `label = value` predicates, where + empty means all observations. + +Every observation in a region contributes once to the state. `regions = []` +means known empty coverage. + +Coverage records only what no other node field records (requirement 4). What +each observation contributes and how states are grouped are already +`SummaryAgg.input` and `SummaryAgg.reduction`. `SummaryMerge` compares those on +its producers directly. + +### Composition is a provably disjoint union + +`merge_disjoint` accepts inputs with the same source whose regions are pairwise +disjoint. Two regions are disjoint when their time ranges do not intersect, or +when they assign different values to the same label. Different labels prove +nothing, and a region with no time bounds overlaps any region it is not +population-disjoint from. The union keeps gaps and the time/population pairing. +Adjacent intervals coalesce only when their populations are identical. + +| Case | Result | +|---|---| +| `[0,1)` + `[1,2)`, same population | one region `[0,2)` | +| `[0,1)` + `[2,3)` | two regions (gap kept) | +| `[0,2)` + `[1,3)` | rejected: possible overlap | +| `region=us` + `region=eu`, same time | two regions | +| `region=us` + `region=us`, or + `tier=premium` | rejected: possible overlap | +| `us×[0,1)` + `eu×[1,2)` | two regions, never `{us,eu}×[0,2)` | +| different source | rejected | + +Equality conjunctions are a deliberately narrow proof vocabulary. A richer +predicate needs an explicit disjointness rule before it can be declared. + +### Lifecycle + +- **Required on summary nodes.** `SummaryAgg` and `SummaryMerge` cannot pass + structural validation without coverage. Other nodes leave it `None`. The + field is an `Option` only because all operators share `OperatorNode`. +- **Declared at build.** The composition rule or catalog that builds a + `SummaryAgg` declares its coverage. +- **Derived at merge.** `SummaryMerge` computes the disjoint union of its + inputs, and validation rejects a retained value that differs. +- **Cleared on rewrite.** Rewriting a node's inputs clears its coverage, like + other assessed metadata. The rewriter must declare it again. +- **Preserved downstream.** Logical export keeps coverage, and CSE shares two + nodes only if their coverage is equal. + +### Trust boundary + +Declarations are trusted. Population is not yet checked against +`SummaryAgg.filter`, `Filter` nodes or `Scan.predicates`, so a wrong declaration +passes: + +```text +A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong +B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) +merge_disjoint(A, B) is accepted, and every US observation is counted twice. +``` + +[#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570) adds the check: the +declared population must equal the `column = literal` predicates between the +`SummaryAgg` and its `Scan`. Time bounds stay trusted, because `TimeRange` is +relative to the evaluation time. + +### Alternatives considered + +| Alternative | Why not | +|---|---| +| Put coverage in `Schema` | Schema equality gates merges; merge inputs always differ in coverage. | +| One time range plus one label set | Invents the missing blocks (Example 3). | +| Copy `input` and `reduction` into coverage | Duplicates `SummaryAgg` and needs a consistency check; producers already carry them. | +| Arbitrary predicates per region | No general disjointness proof; overlap would be silently accepted. | +| Free-form string `source` | Two spellings of one table compare unequal; `Scan` already has `Source`. | +| Snapshot `revision` field | Deployment concern; the planner does not own catalog versions. | + +## Key code interfaces + +```rust +// crates/types/src/ir/summary_coverage.rs +pub struct SummaryCoverage { + pub source: Source, // same type as Scan.source + pub regions: Vec, // union; never a product of independent bounds +} +pub struct CoverageRegion { + pub time_ms: Option>, // half-open; None = no time restriction + pub population: BTreeMap, // label = value AND …; empty = all +} +impl SummaryCoverage { + pub fn validate(&self) -> Result<(), CoverageError>; + pub fn merge_disjoint(inputs: &[Self]) -> Result; +} +pub enum CoverageError { + InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, + NotState, Missing, // node checks + UnknownInput, MergeOutputMismatch, // SummaryMerge (#560) +} + +// crates/types/src/ir/node.rs +pub struct OperatorNode { + // operator, result_kind, schema, guarantee, timing, … + pub coverage: Option, +} +impl OperatorNode { + pub fn with_coverage(self, c: SummaryCoverage) -> Result; + pub fn requires_coverage(&self) -> bool; // SummaryAgg, SummaryMerge + pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; // #560 +} +// SchemaDerivationError::Coverage(CoverageError) reports every failure above. +``` + +`OperatorNode::validate_structure` enforces the lifecycle rules. The documented +examples are built as real `Scan → SummaryAgg → SummaryMerge` plans in +`crates/types/tests/summary_coverage_examples.rs`. + +## Not covered + +- **Query containment:** checking that coverage contains a requested window or + population (Example 4). That is a later Stage 1 check that uses this data. +- **Population check:** comparing declared population with filters + ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). +- **Richer predicates:** predicates beyond non-null equality conjunctions, and + idempotent set-union families. +- **Runtime concerns:** merge kernels, accuracy, storage and execution timing. diff --git a/docs/develop_docs/summary-coverage.md b/docs/develop_docs/summary-coverage.md deleted file mode 100644 index 0de7d30fc..000000000 --- a/docs/develop_docs/summary-coverage.md +++ /dev/null @@ -1,172 +0,0 @@ -# Summary coverage contract - -## Problem: a schema says what a summary *is*, not what it *summarizes* - -Every ASAP edge has one `Schema`. For a summary edge it records the field -layout and the committed state type: - -```rust -pub struct Schema { - pub fields: Vec, // e.g. job: Plain(Utf8), state: Sketch(KLL{k=200}, PerSubpopulationInstance) - pub time_index: Option, // position of a timestamp column, not a time range - pub unique_keys: Vec>, - pub closed: bool, -} -``` - -Nothing in it says **which time range** or **which population (label values)** -the state was built from. Filters, group keys and windows are deliberately not -`Schema` or `Field` members. This becomes a gap once the planner combines -existing summary states (`SummaryMerge`, reuse of ingested panes, sub-DAG -sharing). The producer no longer shows where a state came from, so only the -schema is left to compare. Every example below uses two states with -**exactly equal schemas**: - -```text -Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}, PerSubpopulationInstance)), result_kind = State -``` - -### Example 1: time. Equal schemas, different answers - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| latency, `[00:00, 00:01)` | latency, `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | -| latency, `[00:00, 00:02)` | latency, `[00:01, 00:03)` | **wrong**: every observation in `[00:01, 00:02)` is counted twice, which skews the quantile and doubles counts or frequencies | -| latency, `[00:00, 00:01)` | latency, `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`; **wrong** if used for the continuous window `[00:00, 00:03)` | - -`time_index` is a column position. A KLL state has no timestamp column, so -`time_index` is `None` in all three rows and the schema cannot tell them apart. - -### Example 2: population (label values). Equal schemas, different answers - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| `region='us'` | `region='eu'` | correct: p99 for `us ∪ eu` within each `job` | -| `region='us'` | `tier='premium'` | **wrong**: premium US requests are counted in both inputs | -| `region='us'` | `region='us'` | **wrong**: everything is counted twice | - -`region` is a filter label, not an output column, so it never appears in the -schema. The `job` field only says the state is grouped by job. It does not say -which jobs or which rows contributed. - -### Example 3: time and population together - -A = `us × [0,1)` and B = `eu × [1,2)`. The merged state covers exactly those two -blocks. Storing a time range and a label set separately would give -`{us,eu} × [0,2)`. That claims EU data for `[0,1)` and US data for `[1,2)` -that was never read. Time and population must stay **paired per region**. - -### Example 4: answering a query from a stored state - -Query: `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) GROUP BY job`. -A stored state with the matching schema could hold US data for 10:00–10:05, EU -data, or US data for only 10:00–10:03. All three have the same schema. The -schema confirms that the state *type* fits, not that the *contents* fit. - -**Conclusion.** Schema equality is necessary but not sufficient for composing -or reusing summaries. Without time and population metadata, the planner must -either refuse every composition or accept silent double counting and missing -data. - -## The contract - -`Schema` stays the layout contract and does **not** describe coverage. Coverage -is a separate field on the node, next to `schema`: - -```text -OperatorNode -├── schema: Schema what each output row looks like -└── coverage: Option which observations the state holds -``` - -Coverage cannot live inside `Schema`. `SummaryMerge` requires equal input -schemas, and the inputs of a useful merge (`[0,1)` + `[1,2)`) always have -different coverage. - -```rust -pub struct SummaryCoverage { - pub source: Source, // as named by Scan: Table { table_ref } or TimeSeries { metric } - pub regions: Vec, // union of time × population blocks -} -pub struct CoverageRegion { - pub time_ms: Option>, // half-open, on the source's time column; None = no time restriction - pub population: BTreeMap, // label = value AND …; empty = all observations -} -``` - -Rules: - -- Coverage is **required on summary nodes**. `validate_structure` rejects a - `SummaryAgg` or `SummaryMerge` whose coverage is `None` with - `CoverageError::Missing`. Other nodes leave it `None`. The field is an - `Option` only because all operators share `OperatorNode`. -- Coverage holds only what the operator does not already record. What is fed - into the state and how it is grouped stay on `SummaryAgg.input` and - `SummaryAgg.reduction`; `SummaryMerge` compares those on its inputs. `source` - uses the same `Source` type as `Scan`, so equal sources compare equal. -- `with_coverage` validates the declaration and requires `State` output - (`NotState`). `validate_structure` re-checks it. -- `SummaryMerge` derives its coverage from its inputs. `validate_structure` - rejects a retained value that differs from that union. -- Rewriting a node's inputs clears its coverage. The rewriter must declare it - again with `with_coverage`. -- Every observation in a region contributes once to the state. `regions = []` - means known empty coverage. -- `time_ms: None` is for sources without a time column. Such a region overlaps - every region it is not population-disjoint from. - -## Merging - -`SummaryCoverage::merge_disjoint` requires equal `source` and provably -disjoint regions. `SummaryMerge` additionally requires equal input schemas and -equal `input`/`reduction` on its inputs' producers. Two regions are disjoint when their time ranges -do not intersect, or when they give different values for the same population -label. Different labels prove nothing. The examples above come out as: - -| Case | Result | -|---|---| -| `[0,1)` + `[1,2)`, same population | accepted, coalesced to one region `[0,2)` | -| `[0,1)` + `[2,3)` | accepted, **two** regions (gap kept) | -| `[0,2)` + `[1,3)` | `PossibleOverlap` | -| `region=us` + `region=eu`, same time | accepted, two regions | -| `region=us` + `region=us`, same time | `PossibleOverlap` | -| `region=us` + `tier=premium` | `PossibleOverlap` | -| `us×[0,1)` + `eu×[1,2)` | accepted, two regions, never widened to `{us,eu}×[0,2)` | -| no time bounds + any region of the same population | `PossibleOverlap` | -| different source | `SourceMismatch` | - -Adjacent intervals coalesce only when their population maps are identical. -Merging an empty input list fails with `EmptyMerge`. - -## Trusted declarations - -Coverage is declared by the composition rule or catalog that built the -subtree. Nothing is inferred from SQL. Population is not compared with `SummaryAgg.filter`, -`Filter` nodes or `Scan.predicates`. Time bounds cannot be checked, because -`TimeRange` stores a relative duration. So a wrong declaration passes: - -```text -A = SummaryAgg(filter: region='us', input: latency, reduction: by job) - declared coverage: {region: eu} × [0,1) ← wrong; the state holds US data -B = SummaryAgg(filter: region='us', input: latency, reduction: by job) - declared coverage: {region: us} × [0,1) - -merge_disjoint(A, B) → accepted ("eu" ≠ "us" proves disjoint) -actual merged state → every US observation in [0,1) counted twice -a query for region='eu' could also be answered from A, which holds no EU data -``` - -Issue #570 tracks the check. The declared population must exactly equal the -`column = literal` predicates collected between the `SummaryAgg` and its -`Scan`, and any other predicate shape fails closed. It starts strict about -which operators may sit on that path (only `Filter` and `TimeRange`), because -`Project` or `Join` can rename columns or change rows. - -## Not covered - -- Checking that coverage *contains* a requested query window or population - (Example 4). That is a later query-relative check, which uses this data. -- Predicates beyond non-null equality conjunctions; idempotent set-union - families. -- Runtime merge kernels, accuracy certificates, storage policy or execution - timing. From 0b058bec67479c74ff59c25ab1208062a763e00e Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 19:45:21 +0000 Subject: [PATCH 10/18] docs: design doc for ASAP primitive schema and summary semantics Co-Authored-By: Claude Opus 5.5 --- docs/design_docs/concepts/post-asap-ir.md | 3 +- .../physical-planning-and-deployment.md | 28 +- docs/design_docs/proposals/README.md | 2 +- .../proposals/asap-primitive-schema.md | 466 ++++++++++++++++++ .../proposals/decoupling_op_and_expr.md | 2 +- .../design_docs/proposals/operator-sharing.md | 156 +----- .../design_docs/proposals/summary-coverage.md | 224 --------- .../proposals/univmon-frequency-summary.md | 3 +- .../asap-aware-mapping-contracts.md | 24 +- docs/develop_docs/pre-asap-ir.md | 24 +- 10 files changed, 497 insertions(+), 435 deletions(-) create mode 100644 docs/design_docs/proposals/asap-primitive-schema.md delete mode 100644 docs/design_docs/proposals/summary-coverage.md diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 6c9aa1461..08c670af4 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -49,7 +49,8 @@ summary family supports incremental maintenance. and selects the joined rows. Completeness evidence belongs to pruning, not ranking. A `SummaryNode` carries its expression, schema and optional result guarantee. -State and query values have different contracts. Exact operations over +State and query values have different contracts; see +[Schema and physical data for ASAP primitives](../proposals/asap-primitive-schema.md). Exact operations over approximate readouts still require composed accuracy guarantees. See the [accuracy implementation companion](../../develop_docs/end-to-end-accuracy-guarantees.md) and [physical-plan integration](../architecture/physical-plan-integration.md) diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index 274e4974c..c48dfed54 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -117,26 +117,14 @@ operators from logical candidates has not completed this integration. ### Input semantics and summary semantics -`source`, `filter`, `grouping` and `window` describe input-data semantics: -where records originate, which records qualify, how they are grouped and which -time interval applies. They are not a complete description of arbitrary summary -computation. In particular, the same four fields can summarize different value -expressions or produce different states. - -| Concern | Required semantic information | -| --- | --- | -| Input computation | Source identities and schemas, filters, joins/transforms and their order, or a reference to the canonical input sub-DAG | -| Values and grouping | Value expressions, item identities and weights where applicable, group keys and types, and operation-defined null/duplicate handling | -| Time | Time column and interpretation, interval bounds, evaluation alignment, and distinction between query range and maintained panes | -| Summary computation | Exact operation or sketch family, algorithm and parameters, and supported build/merge behavior | -| Output | State versus finalized value, output schema/type, and readout parameters when part of the output computation | - -For example, KLL over `latency_seconds` and KLL over `log(latency_seconds)` differ -even with identical source, filter, grouping and window. Likewise, weighted -frequency state needs both item and weight expressions. More complex inputs -must retain their computation DAG; four descriptive fields cannot replace it. - -The canonical selected computation is authoritative. These categories describe +`source`, `filter`, `grouping` and `window` describe input-data semantics but not +a complete summary computation: the same four fields can summarize different +value expressions or produce different states. The semantic information a summary +depends on, and where the IR records each part (field type, producing operator, +or coverage), is specified in +[Schema and physical data for ASAP primitives](proposals/asap-primitive-schema.md#23-consideration-3-the-metadata-preserves-summary-semantics). + +The canonical selected computation is authoritative. Those categories describe what must be preserved, not a new flat IR or a second expression language. Operator-defined behavior should be referenced through its canonical contract, not independently configured in deployment metadata. Unsupported or unresolved diff --git a/docs/design_docs/proposals/README.md b/docs/design_docs/proposals/README.md index fc7fde96f..6c4ed49f8 100644 --- a/docs/design_docs/proposals/README.md +++ b/docs/design_docs/proposals/README.md @@ -11,4 +11,4 @@ extensions. A design document is not a promise of downstream runtime support. - [Operator sharing](operator-sharing.md) - [Decoupling operators from scalar expressions](decoupling_op_and_expr.md) - [ASAPPlanner layering](planner-layering.md) -- [Summary coverage](summary-coverage.md) +- [Schema and physical data for ASAP primitives](asap-primitive-schema.md) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md new file mode 100644 index 000000000..ca55ae469 --- /dev/null +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -0,0 +1,466 @@ +# Schema and Physical Data for ASAP Primitives + +> Status: the edge schema and `FieldDataType` are implemented (operator sharing, +> #511). `SummaryCoverage` is implemented in `ir::summary_coverage` (#567). +> `SummaryMerge` coverage derivation lands in #560, and logical export/CSE of +> coverage in #537. Checking declared population against filters is open +> ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). +> Audience: planner designers and architects. +> Companions: [Operator sharing](operator-sharing.md) (unified operator node), +> [Decoupling operators from scalar expressions](decoupling_op_and_expr.md), +> [ASAPPlanner layering](planner-layering.md) Pass 2 (window composition), +> [Physical planning and deployment](../physical-planning-and-deployment.md). + +This document is the single source of truth for how an edge of the operator DAG +is typed, how a field carries an ASAP primitive (summary or exact-accumulator +state), what metadata says which data that state summarizes, and how such a field +is carried as data at runtime. + +## 1. Goal and problem + +ASAP primitives are compact summaries over raw data: a KLL sketch over latency +samples, an exact `Sum` accumulator, a Count-Min sketch over request keys. Once a +plan contains them, three things must be explicit: + +1. **What flows on an edge.** Every operator, before and after ASAP optimization, + needs one typed output contract, so a projection above a summary and one below + it are the same operator. +2. **That a field can be a primitive.** A state column is not a number. It has a + family, an algorithm and parameters, and it can only be read through a + readout. +3. **What a primitive summarizes.** Two states of the same type can hold different + data. The planner must know which observations each holds before it combines or + reuses them. + +The schema alone answers the first two but not the third. Every example below +uses two states with **equal schemas**, +`Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}))`. + +**Example 1: time.** + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| `[00:00, 00:01)` | `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | +| `[00:00, 00:02)` | `[00:01, 00:03)` | **wrong**: `[00:01, 00:02)` is counted twice | +| `[00:00, 00:01)` | `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`, not for `[0,3)` | + +`Schema.time_index` is a column position. A KLL state has no timestamp column. + +**Example 2: population.** + +| Input A | Input B | Merging A and B is… | +|---|---|---| +| `region='us'` | `region='eu'` | correct: `us ∪ eu` within each `job` | +| `region='us'` | `tier='premium'` | **wrong**: premium US requests are in both | +| `region='us'` | `region='us'` | **wrong**: everything is counted twice | + +`region` is a filter label, not an output column. `job` says how the state is +grouped, not which rows contributed. + +**Example 3: time and population together.** Merging `us × [0,1)` with +`eu × [1,2)` covers exactly those two blocks. One time range plus one label set +would give `{us,eu} × [0,2)`, which claims data that was never read. + +**Example 4: reuse.** For `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) +GROUP BY job`, a stored state with a matching schema could hold the right data, +EU data, or only 10:00–10:03. The schema shows that the state *type* fits, not +that the *contents* fit. + +These compositions arise in Pass 2 window composition +([planner layering](planner-layering.md)), in `SummaryMerge` of partial states, +and in sub-DAG sharing and pane reuse. Schema equality is necessary but not +sufficient. Without more metadata, the planner must refuse every composition or +accept silent double counting and missing data. + +## 2. Design considerations + +```text +OperatorNode +├── operator: Operator the operation; SummaryAgg holds input, reduction, filter (C3) +├── result_kind: OperatorResultKind Relation | InstantVector | RangeVector | State (C2) +├── schema: Schema the outgoing edge: fields, time, keys, closedness (C1) +│ └── fields[i].dtype: FieldDataType Plain(DataType) or an ASAP primitive state family (C2) +├── guarantee, timing accuracy and execution phase (operator sharing §2.2, §2.3) +└── coverage: Option which observations the state holds (C3) +``` + +### 2.1 Consideration 1: the schema is the edge between two nodes + +A node's `schema` types its output edge. The DAG is type-checked: the schema is +derived from the operator and its inputs (`Operator::output_schema`), retained on +the node, and verifiable without surrounding context. One `Schema` type serves +every operator before and after ASAP optimization. It replaced the separate +pre-ASAP `Schema`/`Column` and post-ASAP `SummarySchema`/`SummaryField` (old +plans still deserialize). + +| Schema member | Meaning and requirement | +|---|---| +| `fields: Vec` | Ordered fields. `Field = name + dtype: FieldDataType + nullable + table`. `table` preserves SQL qualified resolution through joins. A `Field` holds metadata, never data. | +| `time_index` | Position of the `Plain(Timestamp)` time column, if any. It does not distinguish an instant vector from a range vector. | +| `unique_keys` | Proven column combinations identifying rows; empty asserts no known key. Rewrites that change identity recompute them. | +| `closed` | Whether `fields` is complete. A schemaless PromQL leaf is open; the first `Aggregate`/`Project` that fully determines its output closes it. Open schemas skip closed-world validation. | + +**`ColumnId` versus `ColumnRef`.** These are different roles, not competing +representations: + +| Name | Role | Holds runtime values? | +|---|---|---| +| `Schema`, `Field` | Edge metadata | No | +| `ColumnRef` | Unresolved logical reference: `Named`, `Qualified`, `SampleValue`, `Wildcard` | No | +| `ColumnId = usize` | Resolved position in one particular input/output schema | No | +| Runtime batch | Values conforming to a schema (§3) | Yes | + +Resolution binds a `ColumnRef` to a `ColumnId` before operator nodes are built. +The same position indexes `schema.fields` for type checking and selects the +value at execution: resolving `t.bytes` to `1` gives its type from +`schema.fields[1]`, and `ScalarExpr::Column(1)` reads `row[1]` in the native +executor (or array `1` in a columnar one). `time_index`, `unique_keys` and group +keys use the same positions. A `ColumnId` is local to its schema, not a stable +identity across projections or joins, so there is no `FieldId`. ASAP payloads +that refer to input data before resolution (`SummaryUpdate`, `SketchStatistic::PointCount`) +keep `ColumnRef`. + +**Derivation and validation.** + +- `Scan.schema` declares source columns and `Values.schema` the constructed rows; + every other `OperatorNode.schema` is derived. Planning may override only output + names and qualifiers (`OperatorNode::with_schema`); all structural metadata must + equal derivation. +- `ScalarExpr::scalar_type(input)` types an expression against its column scope + (child schema, both join inputs, or aggregate outputs for `HAVING`). +- `OperatorNode::validate_structure()` walks the reachable DAG: input contracts, + scalar typing, retained-versus-derived schema and result kind, and coverage + (§2.3). It permits `timing = None`. +- `OperatorNode::validate_execution_timing()` adds assigned timing and phase + dependencies, for executable candidates. Neither method proves accuracy; + guarantees stay with planner assessment (#509). + +### 2.2 Consideration 2: a field can have an ASAP primitive type + +`FieldDataType` types every field. `Plain` is an ordinary readable value; every +other variant is the state of one ASAP primitive family and carries the identity +and parameters required by that family: + +```rust +enum FieldDataType { + Plain(DataType), // readable value + ExactAggregate(ExactKind, ExactParams), // Sum, Count, Min, Max, Increase, Rate, IRate + Sketch(SketchKind, GroupingStrategy), // KLL, DDSketch, HLL, CMS, CountSketch, UnivMon, … + Sample(SamplingKind, SamplingParams), + Wavelet(WaveletKind, WaveletParams), + StatModel(StatModelKind, StatModelParams), +} +``` + +**Identity levels.** A sketch has one more level than the other families, because +several algorithms serve one query category (KLL and DDSketch both answer +quantiles): + +| Level | Type | Example | +|---|---|---| +| family | `FieldDataType` variant | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | +| category | `SketchCategory` | `Quantile`, `Cardinality`, `Frequency`, `TopK`, `Universal` | +| algorithm | `SketchAlgorithm` | `Kll`, `DDSketch`; `Hll`, `Theta`, `Kmv`; `Cms`, `CountSketch`, … | +| committed choice | `SketchKind` | one validated category + algorithm + `SketchParams` | + +`SketchKind::new(algorithm, params)` is the only constructor: it rejects a +parameter variant from another algorithm and classifies the pair into its +category; `.category()`, `.algorithm()` and `.params()` expose the committed +values. `Sample`, `Wavelet` and `StatModel` use flat `(Kind, Params)` pairs; +`ExactParams` is per-kind so a mismatched pair is a type error. +`GroupingStrategy` records the physical layout across `by` subpopulations: +`PerSubpopulationInstance` (default) or `SharedMultiSubpopulation { HydraKind, +HydraParams }`. It is part of the type because a shared Hydra structure and +independent instances are not merge-compatible even with the same algorithm. + +Because the full identity is in the type, incompatible states fail at plan +construction: a merge over `Sketch(Kll, …)` and `Sketch(Cms, …)`, or a `Sketch` +read as a `Sample`, is a schema error. + +**Rules for state fields.** + +- **Top-level only.** Nested `List`/`Struct` elements are `Field`, not + `Field`, so a nested field cannot carry state. +- **Produced only by state operators.** `SummaryAgg.family` is never `Plain`; its + input must be values, not state. Its output is the grouping columns plus one + non-nullable `state` field of that family. +- **State is not a value.** `scalar_type` rejects a state column ("read it out + first"). `Filter`, `BinaryOp`, `Join`, `SetOp`, `Concat`, `Aggregate` and + `Dedup` reject `State` inputs; a bare-column `Project` may pass a state field + through unchanged (its result stays `State`). + Copying a state column does not make it readable. +- **Result kind.** `OperatorResultKind::State` marks an output carrying + unfinalized state; its schema may also contain plain grouping keys. Matching + columns never make result kinds interchangeable. + +**Readout / finalization boundary.** State becomes plain values only through an +explicit ASAP readout, which takes its input's relation/vector kind: + +| Readout | Input | Output field | +|---|---|---| +| `SummaryEstimate { query: SketchStatistic }` | exactly one `Sketch` state field whose category supports `query` | `Plain`: `quantile`/`frequency_l2`/`frequency_entropy` `Float64`, `cardinality`/`count` `Int64`, `topk` `Utf8` | +| `FinalizeExactAccumulator` | `ExactAggregate` state | the finalized aggregate value | +| `EvaluatePopulation` | `MaintainPopulation` state | the requested population statistic | + +For example, a KLL build outputs `State` with a `Sketch(KLL{k=200})` column; its +p99 readout outputs `Plain(Float64)`. A numeric predicate can use the readout but +not the state. + +### 2.3 Consideration 3: the metadata preserves summary semantics + +A state is only meaningful together with what it summarizes. The information a +summary's semantics depends on is: + +| Concern | Required semantic information | +|---|---| +| Input computation | Source identities and schemas, filters, joins/transforms and their order, or the canonical input sub-DAG | +| Values and grouping | Value expressions, item identities and weights, group keys and types, null/duplicate handling | +| Time | Time column and interpretation, interval bounds, evaluation alignment, query range versus maintained panes | +| Summary computation | Exact operation or sketch family, algorithm and parameters, build/merge behavior | +| Output | State versus finalized value, output schema/type, readout parameters | + +KLL over `latency_seconds` and KLL over `log(latency_seconds)` differ even with +identical source, filter, grouping and window. Weighted frequency state needs both +item and weight expressions. Four descriptive fields (`source`, `filter`, +`grouping`, `window`) cannot replace the computation DAG. + +The design splits this information by what it varies with, and records each fact +once: + +| Where | What it records | Why there | +|---|---|---| +| Field type (`FieldDataType`) | Family, algorithm, parameters, grouping layout | It determines merge compatibility and which readouts apply, so it gates schema equality. | +| Producer operator (`SummaryAgg`) | `input: SummaryUpdate` (item, weight, `weight_domain` proof), `reduction` (group keys or per-entity), `filter`, `grouping`; the child sub-DAG is the input computation | These are the operation's parameters; copying them elsewhere would need a consistency check. | +| Node (`OperatorNode.coverage`) | Which observations: a source and a union of time × population regions | It differs between states that must still merge, and it cannot be derived from `SummaryAgg` alone. | +| Result kind and readout node | State versus value, readout statistic | Derived from the operator (§2.2). | + +Time alignment, panes and maintenance lifecycle are planning and deployment +concerns ([planner layering](planner-layering.md), +[physical planning](../physical-planning-and-deployment.md)). + +#### Coverage is beside the schema, not inside it + +Coverage is not part of `Schema`. `SummaryMerge` requires equal input schemas, +and the inputs of every useful merge (`[0,1)` + `[1,2)`) have different coverage. +Coverage also describes the whole state output, not one field. It is not an +operator parameter either: a merge *derives* it from its inputs, like the schema. + +Requirements: + +1. Represent time and population **jointly**, per region, never as independent + bounds. +2. Accept a merge only when the inputs are **provably disjoint**; fail closed. + Merging does not imply that a family can remove duplicates. +3. Leave `Schema` and its equality unchanged. +4. Duplicate nothing the operator already records. +5. Support sources without a time column (plain tables). + +#### What coverage records + +`SummaryCoverage` names one observation `source`, using the same `Source` as +`Scan` (a table or a time series), and holds a **union of regions**. Each +`CoverageRegion` pairs: + +- `time_ms`: half-open bounds on the source's time column, or `None` for no time + restriction; +- `population`: a conjunction of non-null `label = value` predicates; empty means + all observations. + +Every observation in a region contributes once to the state. `regions = []` means +known empty coverage. What each observation contributes and how states are +grouped stay on `SummaryAgg.input` and `SummaryAgg.reduction` (requirement 4); +`SummaryMerge` compares those on its producers directly (#560). + +#### Composition is a provably disjoint union + +`merge_disjoint` accepts inputs with the same source whose regions are pairwise +disjoint. Two regions are disjoint when their time ranges do not intersect, or +when they assign different values to the same label. Different labels prove +nothing, and a region without time bounds overlaps any region it is not +population-disjoint from. The union keeps gaps and the time/population pairing; +adjacent intervals coalesce only when their populations are identical. + +| Case | Result | +|---|---| +| `[0,1)` + `[1,2)`, same population | one region `[0,2)` | +| `[0,1)` + `[2,3)` | two regions (gap kept) | +| `[0,2)` + `[1,3)` | rejected: possible overlap | +| `region=us` + `region=eu`, same time | two regions | +| `region=us` + `region=us`, or + `tier=premium` | rejected: possible overlap | +| `us×[0,1)` + `eu×[1,2)` | two regions, never `{us,eu}×[0,2)` | +| different source | rejected | + +Equality conjunctions are a deliberately narrow proof vocabulary. A richer +predicate needs an explicit disjointness rule before it can be declared. + +#### Lifecycle + +- **Required on summary nodes.** `SummaryAgg` cannot pass `validate_structure` + without coverage; `SummaryMerge` joins it in #560. Coverage on a non-`State` + node is rejected. The field is an `Option` only because all operators share + `OperatorNode`. +- **Declared at build.** The composition rule or catalog that builds a + `SummaryAgg` declares it (`with_coverage`). No production builder declares it + on this branch yet. +- **Derived at merge (#560).** `SummaryMerge` computes the disjoint union of its + inputs, and validation rejects a retained value that differs. +- **Cleared on rewrite.** `map_children` rebuilds the node without coverage, like + `guarantee` and `timing`. The rewriter must declare it again. +- **Preserved downstream (#537).** Logical export keeps coverage, and CSE shares + two nodes only if their coverage is equal. + +#### Trust boundary + +Declarations are trusted. Population is not yet checked against +`SummaryAgg.filter`, `Filter` nodes or `Scan.predicates`, so a wrong declaration +passes: + +```text +A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong +B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) +merge_disjoint(A, B) is accepted, and every US observation is counted twice. +``` + +[#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570) adds the check: the +declared population must equal the `column = literal` predicates between the +`SummaryAgg` and its `Scan`. Time bounds stay trusted, because `TimeRange` is +relative to the evaluation time. + +## 3. Physical data: how a state column is carried + +The runtime uses the same `Schema` as planning (`SchemaRef = Arc`). The +native executor (`asap-physical-operators`) stores `Batch { schema, rows: +Vec> }`; the row/column layout is executor-specific. A state column +holds a typed value: + +```rust +enum Value { + Null, Bool(..), Int64(..), Float64(..), Utf8(..), Timestamp(..), Date(..), + Interval { .. }, List(..), Struct(..), Map(..), + Summary { family: FieldDataType, state: Arc }, +} +``` + +- **Typed at the boundary.** `Batch::try_new` checks each `Summary` value's + `family` equals the field's `FieldDataType`, and that the payload's shape + (algorithm and parameters, for example KLL `k` or CMS width × depth) matches it. + State fields must be non-nullable and of a natively supported family. +- **Not a key.** A `Summary` value cannot be a grouping key or be ordered. +- **Kernels.** `summary_kernels` adapt `asap_sketchlib` structures and exact + Planner state behind `AggregateCore`: `merge_with` (same family and shape), + `estimate(SketchStatistic)`, and `approx_memory_bytes` for memory reservations. + `create_planner_accumulator(family, input, grouping)` builds the updater a + `SummaryAgg` declares and rejects a family/grouping disagreement. +- **Native coverage.** Exact Sum/Count/Min/Max/Rate/Increase, KLL, DDSketch, HLL, + Count-Min (stored state only), and weighted CMS/CountSketch with heaps. + `SharedMultiSubpopulation` grouping, `Sample`, `Wavelet` and `StatModel` have + no native kernel and are rejected at binding. +- **No encoding here.** `Value::Summary` is not serialized; byte encodings belong + to `asap_sketchlib` and deployments. + +Coverage is plan metadata and is not carried in runtime values. Physical merge of +states by group key checks family equality only; disjointness is proven at +planning time (§2.3). + +## 4. Alternatives considered + +| Alternative | Why not | +|---|---| +| Separate pre-ASAP and post-ASAP schema types | A projection above a summary needs a different representation from one below it; one `Schema` removes the barrier. | +| An opaque "state" type without family identity | KLL + CMS merges, and sketch-versus-sample confusion, would only fail at runtime. | +| State inside `List`/`Struct` fields | Nested state would escape the readout boundary and state validation. | +| Put coverage in `Schema` | Schema equality gates merges; merge inputs always differ in coverage. | +| One time range plus one label set | Invents the missing blocks (Example 3). | +| Copy `input` and `reduction` into coverage | Duplicates `SummaryAgg` and needs a consistency check; producers already carry them. | +| Arbitrary predicates per region | No general disjointness proof; overlap would be silently accepted. | +| Free-form string `source` | Two spellings of one table compare unequal; `Scan` already has `Source`. | +| Snapshot `revision` field | Deployment concern; the planner does not own catalog versions. | + +## 5. Key code interfaces + +```rust +// crates/types/src/pre_asap/schema.rs +pub type ColumnId = usize; +pub struct Field { pub name: String, pub dtype: T, pub nullable: bool, pub table: Option } +pub struct Schema { + pub fields: Vec, + pub time_index: Option, + pub unique_keys: Vec>, + pub closed: bool, +} +pub enum FieldDataType { Plain(DataType), ExactAggregate(..), Sketch(SketchKind, GroupingStrategy), Sample(..), Wavelet(..), StatModel(..) } +// DataType::List { element: Box> }, DataType::Struct { fields: Vec> } + +// crates/types/src/post_asap/sketch.rs +impl SketchKind { pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; } +pub enum GroupingStrategy { PerSubpopulationInstance, SharedMultiSubpopulation { kind: HydraKind, params: HydraParams } } +pub struct SummaryUpdate { pub item: Option, pub weight: SummaryInputExpr, pub weight_domain: WeightDomain } + +// crates/types/src/ir/asap.rs +pub enum ASAPOp { + SummaryAgg { child, family: FieldDataType, input: SummaryUpdate, reduction: Reduction, + grouping: GroupingStrategy, filter: Option }, + SummaryEstimate { summary_input, query: SketchStatistic }, + FinalizeExactAccumulator { child }, + SummaryMerge { children }, // reserved here; structure in #560 + // MaintainPopulation, EvaluatePopulation, SummarySubtract, SummaryDelete, SummaryJoin, Extension +} + +// crates/types/src/ir/summary_coverage.rs +pub struct SummaryCoverage { pub source: Source, pub regions: Vec } +pub struct CoverageRegion { + pub time_ms: Option>, // half-open; None = no time restriction + pub population: BTreeMap, // label = value AND …; empty = all +} +impl SummaryCoverage { + pub fn validate(&self) -> Result<(), CoverageError>; + pub fn merge_disjoint(inputs: &[Self]) -> Result; +} +pub enum CoverageError { + InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, + NotState, Missing, + // #560: UnknownInput, MergeOutputMismatch +} + +// crates/types/src/ir/node.rs +pub enum OperatorResultKind { Relation, InstantVector, RangeVector, State } +pub struct OperatorNode { + pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, + pub guarantee: Option, pub timing: Option, + pub coverage: Option, +} +impl OperatorNode { + pub fn with_schema(operator: Operator, schema: Schema) -> Self; + pub fn with_coverage(self, c: SummaryCoverage) -> Result; + pub fn requires_coverage(&self) -> bool; // SummaryAgg; SummaryMerge in #560 + pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError>; + pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError>; + // #560: pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; +} +// SchemaDerivationError::Coverage(CoverageError) reports coverage failures. + +// crates/asap-physical-operators/src/{values.rs, summary_kernels/traits.rs} +pub enum Value { /* plain variants */ Summary { family: FieldDataType, state: Arc } } +pub trait AggregateCore { + fn merge_with(&self, other: &dyn AggregateCore) -> Result, KernelError>; + fn estimate(&self, query: &SketchStatistic) -> Result; + fn approx_memory_bytes(&self) -> usize; +} +``` + +Coverage composition is tested in `crates/types/tests/summary_coverage.rs`. The +documented examples are built as real `Scan → SummaryAgg → SummaryMerge` plans in +`crates/types/tests/summary_coverage_examples.rs` (#560). + +## 6. Not covered + +- **Query containment:** checking that coverage contains a requested window or + population (Example 4). That is a later Stage 1 check that uses this data. +- **Population check:** comparing declared population with filters + ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). +- **Richer predicates:** predicates beyond non-null equality conjunctions, and + idempotent set-union families. +- **Runtime concerns:** state encoding, storage, retention, scheduling, kernel + accuracy and performance. +- **Persisted semantic identity:** the stored-definition format and any tenant or + dataset binding belong to the deployment. diff --git a/docs/design_docs/proposals/decoupling_op_and_expr.md b/docs/design_docs/proposals/decoupling_op_and_expr.md index f87456a9f..f850116bb 100644 --- a/docs/design_docs/proposals/decoupling_op_and_expr.md +++ b/docs/design_docs/proposals/decoupling_op_and_expr.md @@ -62,7 +62,7 @@ operator inputs and scalar query-result references use `Rc`. read by expressions. Keep `ScalarExpr::Column(ColumnId)`: the ID selects a field for type checking and the corresponding input value for evaluation, independently of the executor's row/column storage layout. See the -[fields versus column references contract](operator-sharing.md#21-one-schema-model-for-values-and-state). +[fields versus column references contract](asap-primitive-schema.md#21-consideration-1-the-schema-is-the-edge-between-two-nodes). Names are resolved to `ColumnId` before constructing these nodes. Parsing and unresolved `ColumnRef` handling remain frontend concerns; no alternative generic diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index 1851d1809..c9ab193a4 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -365,155 +365,13 @@ caching or mutation mechanism. ### 2.1 One schema model for values and state -Use one `Schema` for operator outputs before and after optimization. Rename today's -`SummaryFamilyType` to `FieldDataType`: it types every field, and `Plain` is not a summary -family. Rename `Column` to `Field` and `Schema.columns` to `Schema.fields`: the struct -describes a column and holds none of its data. Retain the current `Schema` metadata. -The following is the proposed resolved interface; it is not the current Rust definition. - -```rust -struct Field { - name: String, - dtype: FieldDataType, - nullable: bool, - table: Option, -} - -struct Schema { - fields: Vec, - time_index: Option, - unique_keys: Vec>, - closed: bool, -} - -// Today's `SummaryFamilyType`, renamed; variants and payloads unchanged. -enum FieldDataType { - Plain(DataType), - ExactAggregate(ExactKind, ExactParams), - Sketch(SketchKind, GroupingStrategy), - Sample(SamplingKind, SamplingParams), - Wavelet(WaveletKind, WaveletParams), - StatModel(StatModelKind, StatModelParams), -} - -// Proposed derived output classification, separate from column types. -enum OperatorResultKind { - Relation, - InstantVector, - RangeVector, - State, -} - -impl Operator { - fn output_schema(&self) -> Result; - fn output_kind(&self) -> Result; - fn validate_inputs(&self) -> Result<(), QueryExprError>; -} - -impl OperatorNode { - fn validate_structure(&self) -> Result<(), QueryExprError>; - fn validate_execution_timing(&self) -> Result<(), QueryExprError>; -} - -impl ScalarExpr { - fn scalar_type(&self, input: &Schema) -> Result<(DataType, bool), QueryExprError>; -} -``` - -**Fields versus column references.** These names describe different roles, not -competing representations of the same object: - -| Name | Role | Holds runtime values? | -|---|---|---| -| `Schema` | Ordered `Field` metadata, plus key/time/closedness information | No | -| `Field` | Name, type, nullability and optional qualifier for one output column | No | -| `ColumnRef` | Unresolved logical reference: `Named`, `Qualified`, `SampleValue`, or `Wildcard` | No | -| `ColumnId = usize` | Resolved column position in a particular input/output schema | No | -| Runtime batch | Values conforming to a schema; storage layout is executor-specific | Yes | - -Keep `ColumnRef`, `ColumnId`, and `ScalarExpr::Column(ColumnId)`. Renaming the -metadata struct `Column` to `Field` does not rename column references to field -references. The same position identifies metadata during planning and values -during execution; it is not a stable field identity across projections or joins. -Schema `unique_keys` and `time_index` also use these column positions. - -For example, resolving `t.bytes` to position `1` produces `ColumnId = 1`. -`schema.fields[1]` supplies its type and nullability; evaluating -`ScalarExpr::Column(1)` reads the corresponding value. The native executor -currently reads `row[1]` from `Batch { schema, rows: Vec> }`. A columnar -executor would select array `1` instead. No physical `Column` container is -introduced by the metadata rename, and the old metadata `Column` struct is not -retained as a second type. - -**Relationship to current types.** `Field` is today's pre-ASAP `Column` with `dtype` -widened from `DataType` to `FieldDataType`. `FieldDataType` is today's `SummaryFamilyType` -under a name that also fits its `Plain` case. The proposed common `Schema` replaces -the separate operator-edge roles of pre-ASAP `Schema` and post-ASAP `SummarySchema` / -`SummaryField`; it does not rename `DataType`. A pre-ASAP value column becomes -`Plain(dtype)`. -Frontend validation permits only ordinary value columns, preserving the current -pre-ASAP restriction even though the common schema can also express state. - -| Field | Meaning and requirement | -|---|---| -| `fields` | Ordered named fields. `Plain(DataType)` is a readable value; other variants retain the identity and parameters of summary or exact-accumulator state. | -| `Field.nullable`, `Field.table` | Preserve SQL nullability and qualified column resolution. | -| `time_index` | Identifies the time column when present; it does not by itself distinguish an instant vector from a range vector. | -| `unique_keys` | Proven column combinations identifying rows; an empty list asserts no known key. Recompute these proofs when a rewrite changes identity. | -| `closed` | Whether `fields` completely describes the output. An open PromQL schema must retain unlisted labels through the existing complete-series-identity contract. | - -`OperatorResultKind` is derived from the operation and its inputs and retained as -`OperatorNode.result_kind`. `State` describes an output carrying unfinalized state; its -schema may also contain ordinary grouping keys. `SummaryEstimate`, -`FinalizeExactAccumulator` and other readouts derive the appropriate relation or -vector kind from their operation and input context. Matching numeric columns do -not make those kinds interchangeable. - -**Interface contracts.** `Operator::output_schema` and `output_kind` derive output -metadata from the payload and validated inputs. `validate_inputs` checks local -producer/consumer compatibility, such as vector inputs for `BinaryOp` or the -required state family for a summary readout. Scalar typing checks the input-kind -contract of `PromqlScalarFromVector` and other scalar plan reads. - -| Validation entry | Scope and stage | -|---|---| -| `OperatorNode::validate_structure()` | Walks the reachable operator DAG, including scalar plan references; checks input contracts, scalar typing and agreement between retained and derived output metadata. Valid for logical and physical plans; permits `timing = None`. | -| `OperatorNode::validate_execution_timing()` | Includes structural validation, then requires assigned timing on every executable operator and checks phase dependencies. Used for executable physical candidates. | -| Existing planner assessment and selection (#509) | Establishes guarantees using the existing accuracy models and checks them against request requirements and deployment capabilities. Neither node method re-proves a guarantee or decides request feasibility. | - -The two node methods need only the DAG and its annotations. Request requirements -and deployment models remain inputs to the existing planning/selection workflow, -not implicit globals of `validate_structure`. Passing the timing check alone does -not establish that a physical candidate satisfies the query's accuracy requirement. - -`Scan.schema` declares the source columns; `Values.schema` declares the constructed -row shape. `OperatorNode.schema` is the derived output for any operation. A scan's -predicates cannot change its declared output columns; a Values row must match the -declared arity, types and nullability. These leaf outputs retain the declaration's -column layout and time/identity information, with only justified metadata changes. -The declaration and derived output therefore have distinct roles, and structural -validation rejects disagreement rather than trusting two independent schemas. - -`scalar_type` keeps the existing method name and `(DataType, nullable)` result. -Its `input` is the applicable column scope: the child schema for a projection, -both input schemas for a join predicate, or aggregate outputs for `HAVING`. -Explicit subquery/conversion expressions validate their referenced producer using -the contracts above. Numeric expressions cannot consume state columns as numbers. -A standalone scalar expression is checked with an empty column scope and needs no fabricated -relation output schema. `QueryExprError` retains the existing error-type name; -result-kind, state-family, schema and execution-phase mismatches require -corresponding validation errors. - -For example, a KLL build outputs `State` with a -`Sketch(SketchKind, GroupingStrategy)` column identifying KLL and its parameters. -Its p99 readout outputs an ordinary `Plain(Float64)` column in the appropriate -relation/vector schema. A numeric predicate can use that readout, but not the KLL -state. Exact accumulator state similarly requires `FinalizeExactAccumulator`. -An ordinary operator may pass state through only where its input/output contract -permits it. A bare-column projection can preserve the field's `FieldDataType` -directly during `output_schema` derivation; `scalar_type` applies when that column -is used as a scalar value and rejects state. Copying a state column does not turn -it into a readable scalar. +Every operator output, before and after optimization, uses one `Schema` whose +`Field`s are typed by `FieldDataType`: `Plain(DataType)` for a readable value, or +the family, algorithm and parameters of summary or exact-accumulator state. +`OperatorResultKind` marks state outputs, and state becomes a value only through +an explicit readout. The schema model, `ColumnRef` versus `ColumnId`, the +validation entry points and the readout boundary are specified in +[Schema and physical data for ASAP primitives](asap-primitive-schema.md). ### 2.2 Preserve existing accuracy semantics diff --git a/docs/design_docs/proposals/summary-coverage.md b/docs/design_docs/proposals/summary-coverage.md deleted file mode 100644 index a3520734d..000000000 --- a/docs/design_docs/proposals/summary-coverage.md +++ /dev/null @@ -1,224 +0,0 @@ -# Summary Coverage - -> Status: implemented in `ir::summary_coverage` (#567), with `SummaryMerge` -> derivation (#560) and logical transport/CSE (#537). Checking declared -> population against filters is open ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). -> Audience: planner designers and architects. -> Companions: [Operator sharing](operator-sharing.md) §2.1 (schema model), -> [ASAPPlanner layering](planner-layering.md) Pass 2 (window composition). - -## Goal and problem - -Record **which observations a summary state was built from**, its time range and -population, so the planner can tell when combining or reusing summary states -is correct. - -[Operator sharing](operator-sharing.md) §2.1 gives every edge one `Schema`. For -a summary edge, the schema records the field layout and the committed state -type, such as `(job: Utf8, state: KLL{k=200})`. It deliberately leaves out -filters, group keys and windows. That is enough while a summary is consumed -right after its producer. It stops being enough once -[planner layering](planner-layering.md) composes existing states: - -- Pass 2's window-composition rule merges tumbling or EH summaries into a query - window. -- `SummaryMerge` combines partial states. -- Sub-DAG sharing, and the reuse of ingested panes, hand one state to several - consumers. - -In these cases only the schema is left to compare. Every example below uses -two states with **equal schemas**, `Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}))`. - -**Example 1: time.** - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| latency, `[00:00, 00:01)` | latency, `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | -| latency, `[00:00, 00:02)` | latency, `[00:01, 00:03)` | **wrong**: `[00:01, 00:02)` is counted twice | -| latency, `[00:00, 00:01)` | latency, `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`, not for the continuous `[0,3)` | - -`Schema.time_index` is a column position. A KLL state has no timestamp column, -so the schema cannot tell these apart. - -**Example 2: population (label values).** - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| `region='us'` | `region='eu'` | correct: `us ∪ eu` within each `job` | -| `region='us'` | `tier='premium'` | **wrong**: premium US requests are in both | -| `region='us'` | `region='us'` | **wrong**: everything is counted twice | - -`region` is a filter label, not an output column. The `job` field says how the -state is grouped, not which rows contributed. - -**Example 3: time and population together.** Merging `us × [0,1)` with -`eu × [1,2)` covers exactly those two blocks. Recording one time range and one -label set would give `{us,eu} × [0,2)`, which claims data that was never read. - -**Example 4: reuse.** For `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) -GROUP BY job`, a stored state with a matching schema could hold the right data, -EU data, or only 10:00–10:03. The schema shows that the state *type* fits, not -that the *contents* fit. - -Schema equality is necessary but not sufficient. Without this metadata, the -planner must either refuse every composition or accept silent double counting -and missing data. - -### Requirements - -1. Represent time and population **jointly**, per block, never as independent - bounds. -2. Accept a merge only when the inputs are **provably disjoint**, and fail - closed otherwise. Merging does not imply that a summary family can remove - duplicates. -3. Leave `Schema` and its equality unchanged. -4. Duplicate nothing the operator already records. -5. Support sources without a time column (plain tables). - -## Design - -### Coverage is a node property, beside the schema - -```text -OperatorNode -├── schema: Schema what each output row looks like (operator sharing §2.1) -├── guarantee, timing accuracy and execution phase (§2.2, §2.3) -└── coverage: Option which observations the state holds -``` - -Coverage is not part of `Schema`. `SummaryMerge` requires equal input schemas, -and the inputs of every useful merge (`[0,1)` + `[1,2)`) have different coverage. -It is also not an operator parameter: a merge *derives* it from its inputs, like -the schema. - -### What coverage records - -A `SummaryCoverage` names one observation `source`, using the same `Source` as -`Scan`: a table or a time series. It holds a **union of regions**. Each region -pairs: - -- `time_ms`: half-open bounds on the source's time column, or `None` for no - time restriction; -- `population`: a conjunction of non-null `label = value` predicates, where - empty means all observations. - -Every observation in a region contributes once to the state. `regions = []` -means known empty coverage. - -Coverage records only what no other node field records (requirement 4). What -each observation contributes and how states are grouped are already -`SummaryAgg.input` and `SummaryAgg.reduction`. `SummaryMerge` compares those on -its producers directly. - -### Composition is a provably disjoint union - -`merge_disjoint` accepts inputs with the same source whose regions are pairwise -disjoint. Two regions are disjoint when their time ranges do not intersect, or -when they assign different values to the same label. Different labels prove -nothing, and a region with no time bounds overlaps any region it is not -population-disjoint from. The union keeps gaps and the time/population pairing. -Adjacent intervals coalesce only when their populations are identical. - -| Case | Result | -|---|---| -| `[0,1)` + `[1,2)`, same population | one region `[0,2)` | -| `[0,1)` + `[2,3)` | two regions (gap kept) | -| `[0,2)` + `[1,3)` | rejected: possible overlap | -| `region=us` + `region=eu`, same time | two regions | -| `region=us` + `region=us`, or + `tier=premium` | rejected: possible overlap | -| `us×[0,1)` + `eu×[1,2)` | two regions, never `{us,eu}×[0,2)` | -| different source | rejected | - -Equality conjunctions are a deliberately narrow proof vocabulary. A richer -predicate needs an explicit disjointness rule before it can be declared. - -### Lifecycle - -- **Required on summary nodes.** `SummaryAgg` and `SummaryMerge` cannot pass - structural validation without coverage. Other nodes leave it `None`. The - field is an `Option` only because all operators share `OperatorNode`. -- **Declared at build.** The composition rule or catalog that builds a - `SummaryAgg` declares its coverage. -- **Derived at merge.** `SummaryMerge` computes the disjoint union of its - inputs, and validation rejects a retained value that differs. -- **Cleared on rewrite.** Rewriting a node's inputs clears its coverage, like - other assessed metadata. The rewriter must declare it again. -- **Preserved downstream.** Logical export keeps coverage, and CSE shares two - nodes only if their coverage is equal. - -### Trust boundary - -Declarations are trusted. Population is not yet checked against -`SummaryAgg.filter`, `Filter` nodes or `Scan.predicates`, so a wrong declaration -passes: - -```text -A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong -B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) -merge_disjoint(A, B) is accepted, and every US observation is counted twice. -``` - -[#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570) adds the check: the -declared population must equal the `column = literal` predicates between the -`SummaryAgg` and its `Scan`. Time bounds stay trusted, because `TimeRange` is -relative to the evaluation time. - -### Alternatives considered - -| Alternative | Why not | -|---|---| -| Put coverage in `Schema` | Schema equality gates merges; merge inputs always differ in coverage. | -| One time range plus one label set | Invents the missing blocks (Example 3). | -| Copy `input` and `reduction` into coverage | Duplicates `SummaryAgg` and needs a consistency check; producers already carry them. | -| Arbitrary predicates per region | No general disjointness proof; overlap would be silently accepted. | -| Free-form string `source` | Two spellings of one table compare unequal; `Scan` already has `Source`. | -| Snapshot `revision` field | Deployment concern; the planner does not own catalog versions. | - -## Key code interfaces - -```rust -// crates/types/src/ir/summary_coverage.rs -pub struct SummaryCoverage { - pub source: Source, // same type as Scan.source - pub regions: Vec, // union; never a product of independent bounds -} -pub struct CoverageRegion { - pub time_ms: Option>, // half-open; None = no time restriction - pub population: BTreeMap, // label = value AND …; empty = all -} -impl SummaryCoverage { - pub fn validate(&self) -> Result<(), CoverageError>; - pub fn merge_disjoint(inputs: &[Self]) -> Result; -} -pub enum CoverageError { - InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, - NotState, Missing, // node checks - UnknownInput, MergeOutputMismatch, // SummaryMerge (#560) -} - -// crates/types/src/ir/node.rs -pub struct OperatorNode { - // operator, result_kind, schema, guarantee, timing, … - pub coverage: Option, -} -impl OperatorNode { - pub fn with_coverage(self, c: SummaryCoverage) -> Result; - pub fn requires_coverage(&self) -> bool; // SummaryAgg, SummaryMerge - pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; // #560 -} -// SchemaDerivationError::Coverage(CoverageError) reports every failure above. -``` - -`OperatorNode::validate_structure` enforces the lifecycle rules. The documented -examples are built as real `Scan → SummaryAgg → SummaryMerge` plans in -`crates/types/tests/summary_coverage_examples.rs`. - -## Not covered - -- **Query containment:** checking that coverage contains a requested window or - population (Example 4). That is a later Stage 1 check that uses this data. -- **Population check:** comparing declared population with filters - ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). -- **Richer predicates:** predicates beyond non-null equality conjunctions, and - idempotent set-union families. -- **Runtime concerns:** merge kernels, accuracy, storage and execution timing. diff --git a/docs/design_docs/proposals/univmon-frequency-summary.md b/docs/design_docs/proposals/univmon-frequency-summary.md index 75cfd080b..a2d9f96c1 100644 --- a/docs/design_docs/proposals/univmon-frequency-summary.md +++ b/docs/design_docs/proposals/univmon-frequency-summary.md @@ -35,7 +35,8 @@ cardinality alternatives, and exact count remains the cheaper first count candidate. All four readouts have the same unit-weight update, input sub-DAG, grouping, -window, parameter identity and state schema. Existing post-ASAP structural +window, parameter identity and state schema +([ASAP primitive schema](asap-primitive-schema.md)). Existing post-ASAP structural sharing can therefore intern their state producer while preserving distinct readout nodes. Sharing is only legal within the same execution/data scope. Precompute placement, SummaryCatalog installation, retention, and runtime diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 447cb3823..011758143 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -325,24 +325,12 @@ backend inspection. Automatic selection skips those unproven ratios. Use ### Family, category, algorithm, and parameters -Sketches separate their query category from the concrete algorithm and its parameters: +A summary's identity has four levels: family (`FieldDataType` variant), sketch +category (`SketchCategory`), algorithm (`SketchAlgorithm`), and the validated +committed choice (`SketchKind`). The levels and their validation are specified in +[Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#22-consideration-2-a-field-can-have-an-asap-primitive-type). -| Level | Type | Example | -| --- | --- | --- | -| **family** | `SummaryFamilyType` | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | -| **category** | `SketchCategory` | `Quantile`, `Cardinality`, `Frequency`, `TopK` | -| **algorithm** | `SketchAlgorithm` | `Kll` / `DDSketch` (both quantile); `Hll` (HyperLogLog) / `Theta` / `Kmv` (K-Minimum Values), all cardinality | -| **committed choice** | `SketchKind` | one validated category + algorithm + parameter combination | - -A `SketchKind` is a validated committed choice. Its public constructor, -`SketchKind::new(algorithm, params)`, verifies that the parameter variant belongs -to the selected algorithm and classifies the pair into its category. The public -`.category()`, `.algorithm()`, and `.params()` accessors expose the committed -values without permitting an invalid combination. - -Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `SummaryFamilyType::Sketch(SketchKind)`. - -`Sample`, `Wavelet`, and `StatModel` each use a flat `(Kind, Params)` pair. `Sketch` needs the additional algorithm level because multiple algorithms can serve the same purpose—for example, KLL and DDSketch both answer quantile queries. +Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `FieldDataType::Sketch(SketchKind, GroupingStrategy)`. --- @@ -378,7 +366,7 @@ The crate provides no default `Matcher` implementation because the answer depend Concretely, `explanation.rs` reports three candidate kinds from each `TargetSubDAGCandidates`: -- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `SummaryFamilyType::Sketch(..)`, not just an exact/pass-through candidate. +- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `FieldDataType::Sketch(..)`, not just an exact/pass-through candidate. - `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the set's `target`). - `ExplanationKind::ExactComposition` — the candidate set contains an exact operation diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index abf5dc50b..2492e28d7 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -17,26 +17,10 @@ The pre-ASAP IR is defined using the `QueryExpr` enum. We discuss some of import ## Fields and column references -`Schema` owns `Field` metadata: name, type, nullability, and an optional table -qualifier. A `Field` contains no runtime values. The former schema `Column` -struct served this same metadata role; it was renamed to `Field`, not retained -as a second data container. - -`ColumnRef` is an unresolved logical reference (`Named`, `Qualified`, -`SampleValue`, or `Wildcard`). Resolution binds a reference to `ColumnId`, a -`usize` position within a particular schema. `QueryExpr::Column(ColumnId)` -reads that column; the same position indexes `Schema::fields` for type checking -and a runtime row for its value. Group keys, unique keys, and `time_index` also -use these column positions. They are not stable identities across projections -or joins, so the positional reference remains `ColumnId`, not `FieldId`. - -The native runtime names shared ownership `SchemaRef = Arc` and stores -`Batch { schema: SchemaRef, rows: Vec> }`. `Schema` is the same metadata -model during planning and execution; the `Ref` suffix only distinguishes ownership. -It has no physical `Column`/array container. A column reference expresses what -to read independently of whether an executor stores its data as rows or arrays. -For example, resolving `t.bytes` to `ColumnId = 1` obtains its type from -`schema.fields[1]`; native execution reads `row[1]`. +`Schema` holds `Field` metadata (name, type, nullability, qualifier) and no +values; an unresolved `ColumnRef` resolves to a positional `ColumnId` within one +schema. The design, including how the same position selects a runtime value, is +in [Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#21-consideration-1-the-schema-is-the-edge-between-two-nodes). ## Node index From da241a6592814303c294020e61cdd278e93d2448 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 15:49:55 -0400 Subject: [PATCH 11/18] Update asap-primitive-schema.md --- .../proposals/asap-primitive-schema.md | 29 +------------------ 1 file changed, 1 insertion(+), 28 deletions(-) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index ca55ae469..e8cff6444 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -1,20 +1,6 @@ # Schema and Physical Data for ASAP Primitives -> Status: the edge schema and `FieldDataType` are implemented (operator sharing, -> #511). `SummaryCoverage` is implemented in `ir::summary_coverage` (#567). -> `SummaryMerge` coverage derivation lands in #560, and logical export/CSE of -> coverage in #537. Checking declared population against filters is open -> ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). -> Audience: planner designers and architects. -> Companions: [Operator sharing](operator-sharing.md) (unified operator node), -> [Decoupling operators from scalar expressions](decoupling_op_and_expr.md), -> [ASAPPlanner layering](planner-layering.md) Pass 2 (window composition), -> [Physical planning and deployment](../physical-planning-and-deployment.md). - -This document is the single source of truth for how an edge of the operator DAG -is typed, how a field carries an ASAP primitive (summary or exact-accumulator -state), what metadata says which data that state summarizes, and how such a field -is carried as data at runtime. +This document is the single source of truth for the schema, and column design for ASAP Primitives. This is used in the logical stage (LogicalASAPDAG), and physical stage (PhysicalASAPDAG). ## 1. Goal and problem @@ -451,16 +437,3 @@ pub trait AggregateCore { Coverage composition is tested in `crates/types/tests/summary_coverage.rs`. The documented examples are built as real `Scan → SummaryAgg → SummaryMerge` plans in `crates/types/tests/summary_coverage_examples.rs` (#560). - -## 6. Not covered - -- **Query containment:** checking that coverage contains a requested window or - population (Example 4). That is a later Stage 1 check that uses this data. -- **Population check:** comparing declared population with filters - ([#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570)). -- **Richer predicates:** predicates beyond non-null equality conjunctions, and - idempotent set-union families. -- **Runtime concerns:** state encoding, storage, retention, scheduling, kernel - accuracy and performance. -- **Persisted semantic identity:** the stored-definition format and any tenant or - dataset binding belong to the deployment. From 91bd15d672c89ed47d502e60bebc05788131de18 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:01:33 -0400 Subject: [PATCH 12/18] Update asap-primitive-schema.md --- .../proposals/asap-primitive-schema.md | 32 ++++++++++++------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index e8cff6444..2825a4e36 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -4,17 +4,27 @@ This document is the single source of truth for the schema, and column design fo ## 1. Goal and problem -ASAP primitives are compact summaries over raw data: a KLL sketch over latency -samples, an exact `Sum` accumulator, a Count-Min sketch over request keys. Once a -plan contains them, three things must be explicit: - -1. **What flows on an edge.** Every operator, before and after ASAP optimization, - needs one typed output contract, so a projection above a summary and one below - it are the same operator. -2. **That a field can be a primitive.** A state column is not a number. It has a - family, an algorithm and parameters, and it can only be read through a - readout. -3. **What a primitive summarizes.** Two states of the same type can hold different +Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, + +Once an [ASAP Operator](https://github.com/ProjectASAP/ASAPPlanner/blob/main/docs/design_docs/proposals/operator-sharing.md) is a summary state operator, the ASAP Operator node in LogicalASAPDAG and PhysicalASAPDAG should represent the following information: + + +## Schema Design + +Schema represents the metadata of information flow along an edge between two nodes in a logical or physical DAG. The schema field is associated with a node in the DAG. + +Schema definition here is shared between LogicalDAG, LogicalASAPDAG, and PhysicalASAPDAG. The schema contain fields, and each field is mapping to a column in the physical data representation. +Each field should contain the following information. +1. **The data type of a column.** A state column can be a raw data type (e.g., numerical number, string). It can also be a [summary type](), e.g., the summary family is sketch, and the sketch type is quantile KLL sketch algorithm, and KLL sketch has K as parameter. It has a + family, an algorithm and parameters. +2. ** **. + + + +## Node design + +Each node in the DAG should contain the information of the instance this nodes is computing, in addition to schema or metadata. +4. **What a ASAP primitive summarizes.** Two states of the same type can hold different data. The planner must know which observations each holds before it combines or reuses them. From 659c172ec4d3def6346cc0f6f98ec51fc2aec4f4 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:27:36 -0400 Subject: [PATCH 13/18] Update asap-primitive-schema.md --- .../proposals/asap-primitive-schema.md | 31 +++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 2825a4e36..626affe57 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -2,11 +2,36 @@ This document is the single source of truth for the schema, and column design for ASAP Primitives. This is used in the logical stage (LogicalASAPDAG), and physical stage (PhysicalASAPDAG). -## 1. Goal and problem +## 1. Goal, problem, and requirements -Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, +Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, it introduces new requirement when we design the schema and node definitions for LogicalASAPDAG and PhysicalASAPDAG. -Once an [ASAP Operator](https://github.com/ProjectASAP/ASAPPlanner/blob/main/docs/design_docs/proposals/operator-sharing.md) is a summary state operator, the ASAP Operator node in LogicalASAPDAG and PhysicalASAPDAG should represent the following information: +Assuming we have the Logical DAG defined for a canonicalized representation for a batch of queries. [TODO: add links for this here. ] +The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives: SummaryCreation?, SummaryUpdate, SummaryMerge, SummaryDelete, SummarySubtraction [TODO: check what is the complete list or discuss with others about the list]. +Each of the Summary operators also require the ASAP primitive information above to inter-operate correctly, preserving semantic correctness. + +Basically, the following information should be represented to preserve the equivalent query semantics when we introduce ASAP Primitives to logical query representation, and following physical one. + +- What type of the ASAP Primitive is +- What is the ASAP Primitive parameters +- What data sources a ASAP primitive summarizes +- What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents + + + +And these information will be combined with relational or time series query operator information, such as group by/reduction, filtering, projection, join, time series selection, together. + +Therefore, these requirements drive the following schema and metadata, node information, and column design. + + + + + + + + + +--------don't read below------------- ## Schema Design From 338a1d3862a1cadaf3cb1c37d147eb805733e0b9 Mon Sep 17 00:00:00 2001 From: Zeying Zhu <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 16:40:14 -0400 Subject: [PATCH 14/18] Update asap-primitive-schema.md --- .../proposals/asap-primitive-schema.md | 447 +----------------- 1 file changed, 12 insertions(+), 435 deletions(-) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 626affe57..9f66ba70c 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -7,7 +7,7 @@ This document is the single source of truth for the schema, and column design fo Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, it introduces new requirement when we design the schema and node definitions for LogicalASAPDAG and PhysicalASAPDAG. Assuming we have the Logical DAG defined for a canonicalized representation for a batch of queries. [TODO: add links for this here. ] -The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives: SummaryCreation?, SummaryUpdate, SummaryMerge, SummaryDelete, SummarySubtraction [TODO: check what is the complete list or discuss with others about the list]. +The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives: SummaryCreation?, SummaryUpdate, SummaryMerge, SummaryDelete, SummarySubtraction, SummaryEstimate [TODO: check what is the complete list or discuss with others about the list]. Each of the Summary operators also require the ASAP primitive information above to inter-operate correctly, preserving semantic correctness. Basically, the following information should be represented to preserve the equivalent query semantics when we introduce ASAP Primitives to logical query representation, and following physical one. @@ -25,450 +25,27 @@ Therefore, these requirements drive the following schema and metadata, node info +## 2. Existing database terminology for schema, table, column, and physical data layout - - - - ---------don't read below------------- - - -## Schema Design - -Schema represents the metadata of information flow along an edge between two nodes in a logical or physical DAG. The schema field is associated with a node in the DAG. +## 3. Proposed schema design +Schema represents the **metadata** of information flow along an **edge** between two nodes in a logical or physical DAG. The schema field is associated with the node in the DAG. The consumer of the node in the DAG takes the schema from the producer node as input. Schema definition here is shared between LogicalDAG, LogicalASAPDAG, and PhysicalASAPDAG. The schema contain fields, and each field is mapping to a column in the physical data representation. -Each field should contain the following information. -1. **The data type of a column.** A state column can be a raw data type (e.g., numerical number, string). It can also be a [summary type](), e.g., the summary family is sketch, and the sketch type is quantile KLL sketch algorithm, and KLL sketch has K as parameter. It has a - family, an algorithm and parameters. -2. ** **. - - - -## Node design - -Each node in the DAG should contain the information of the instance this nodes is computing, in addition to schema or metadata. -4. **What a ASAP primitive summarizes.** Two states of the same type can hold different - data. The planner must know which observations each holds before it combines or - reuses them. - -The schema alone answers the first two but not the third. Every example below -uses two states with **equal schemas**, -`Schema(job: Plain(Utf8), state: Sketch(KLL{k=200}))`. - -**Example 1: time.** - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| `[00:00, 00:01)` | `[00:01, 00:02)` | correct: p99 over `[00:00, 00:02)` | -| `[00:00, 00:02)` | `[00:01, 00:03)` | **wrong**: `[00:01, 00:02)` is counted twice | -| `[00:00, 00:01)` | `[00:02, 00:03)` | correct only for `[0,1) ∪ [2,3)`, not for `[0,3)` | - -`Schema.time_index` is a column position. A KLL state has no timestamp column. - -**Example 2: population.** - -| Input A | Input B | Merging A and B is… | -|---|---|---| -| `region='us'` | `region='eu'` | correct: `us ∪ eu` within each `job` | -| `region='us'` | `tier='premium'` | **wrong**: premium US requests are in both | -| `region='us'` | `region='us'` | **wrong**: everything is counted twice | - -`region` is a filter label, not an output column. `job` says how the state is -grouped, not which rows contributed. - -**Example 3: time and population together.** Merging `us × [0,1)` with -`eu × [1,2)` covers exactly those two blocks. One time range plus one label set -would give `{us,eu} × [0,2)`, which claims data that was never read. - -**Example 4: reuse.** For `p99(latency) WHERE region='us' AND ts IN [10:00, 10:05) -GROUP BY job`, a stored state with a matching schema could hold the right data, -EU data, or only 10:00–10:03. The schema shows that the state *type* fits, not -that the *contents* fit. - -These compositions arise in Pass 2 window composition -([planner layering](planner-layering.md)), in `SummaryMerge` of partial states, -and in sub-DAG sharing and pane reuse. Schema equality is necessary but not -sufficient. Without more metadata, the planner must refuse every composition or -accept silent double counting and missing data. - -## 2. Design considerations - -```text -OperatorNode -├── operator: Operator the operation; SummaryAgg holds input, reduction, filter (C3) -├── result_kind: OperatorResultKind Relation | InstantVector | RangeVector | State (C2) -├── schema: Schema the outgoing edge: fields, time, keys, closedness (C1) -│ └── fields[i].dtype: FieldDataType Plain(DataType) or an ASAP primitive state family (C2) -├── guarantee, timing accuracy and execution phase (operator sharing §2.2, §2.3) -└── coverage: Option which observations the state holds (C3) -``` - -### 2.1 Consideration 1: the schema is the edge between two nodes - -A node's `schema` types its output edge. The DAG is type-checked: the schema is -derived from the operator and its inputs (`Operator::output_schema`), retained on -the node, and verifiable without surrounding context. One `Schema` type serves -every operator before and after ASAP optimization. It replaced the separate -pre-ASAP `Schema`/`Column` and post-ASAP `SummarySchema`/`SummaryField` (old -plans still deserialize). - -| Schema member | Meaning and requirement | -|---|---| -| `fields: Vec` | Ordered fields. `Field = name + dtype: FieldDataType + nullable + table`. `table` preserves SQL qualified resolution through joins. A `Field` holds metadata, never data. | -| `time_index` | Position of the `Plain(Timestamp)` time column, if any. It does not distinguish an instant vector from a range vector. | -| `unique_keys` | Proven column combinations identifying rows; empty asserts no known key. Rewrites that change identity recompute them. | -| `closed` | Whether `fields` is complete. A schemaless PromQL leaf is open; the first `Aggregate`/`Project` that fully determines its output closes it. Open schemas skip closed-world validation. | - -**`ColumnId` versus `ColumnRef`.** These are different roles, not competing -representations: - -| Name | Role | Holds runtime values? | -|---|---|---| -| `Schema`, `Field` | Edge metadata | No | -| `ColumnRef` | Unresolved logical reference: `Named`, `Qualified`, `SampleValue`, `Wildcard` | No | -| `ColumnId = usize` | Resolved position in one particular input/output schema | No | -| Runtime batch | Values conforming to a schema (§3) | Yes | - -Resolution binds a `ColumnRef` to a `ColumnId` before operator nodes are built. -The same position indexes `schema.fields` for type checking and selects the -value at execution: resolving `t.bytes` to `1` gives its type from -`schema.fields[1]`, and `ScalarExpr::Column(1)` reads `row[1]` in the native -executor (or array `1` in a columnar one). `time_index`, `unique_keys` and group -keys use the same positions. A `ColumnId` is local to its schema, not a stable -identity across projections or joins, so there is no `FieldId`. ASAP payloads -that refer to input data before resolution (`SummaryUpdate`, `SketchStatistic::PointCount`) -keep `ColumnRef`. - -**Derivation and validation.** - -- `Scan.schema` declares source columns and `Values.schema` the constructed rows; - every other `OperatorNode.schema` is derived. Planning may override only output - names and qualifiers (`OperatorNode::with_schema`); all structural metadata must - equal derivation. -- `ScalarExpr::scalar_type(input)` types an expression against its column scope - (child schema, both join inputs, or aggregate outputs for `HAVING`). -- `OperatorNode::validate_structure()` walks the reachable DAG: input contracts, - scalar typing, retained-versus-derived schema and result kind, and coverage - (§2.3). It permits `timing = None`. -- `OperatorNode::validate_execution_timing()` adds assigned timing and phase - dependencies, for executable candidates. Neither method proves accuracy; - guarantees stay with planner assessment (#509). - -### 2.2 Consideration 2: a field can have an ASAP primitive type - -`FieldDataType` types every field. `Plain` is an ordinary readable value; every -other variant is the state of one ASAP primitive family and carries the identity -and parameters required by that family: - -```rust -enum FieldDataType { - Plain(DataType), // readable value - ExactAggregate(ExactKind, ExactParams), // Sum, Count, Min, Max, Increase, Rate, IRate - Sketch(SketchKind, GroupingStrategy), // KLL, DDSketch, HLL, CMS, CountSketch, UnivMon, … - Sample(SamplingKind, SamplingParams), - Wavelet(WaveletKind, WaveletParams), - StatModel(StatModelKind, StatModelParams), -} -``` - -**Identity levels.** A sketch has one more level than the other families, because -several algorithms serve one query category (KLL and DDSketch both answer -quantiles): - -| Level | Type | Example | -|---|---|---| -| family | `FieldDataType` variant | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | -| category | `SketchCategory` | `Quantile`, `Cardinality`, `Frequency`, `TopK`, `Universal` | -| algorithm | `SketchAlgorithm` | `Kll`, `DDSketch`; `Hll`, `Theta`, `Kmv`; `Cms`, `CountSketch`, … | -| committed choice | `SketchKind` | one validated category + algorithm + `SketchParams` | - -`SketchKind::new(algorithm, params)` is the only constructor: it rejects a -parameter variant from another algorithm and classifies the pair into its -category; `.category()`, `.algorithm()` and `.params()` expose the committed -values. `Sample`, `Wavelet` and `StatModel` use flat `(Kind, Params)` pairs; -`ExactParams` is per-kind so a mismatched pair is a type error. -`GroupingStrategy` records the physical layout across `by` subpopulations: -`PerSubpopulationInstance` (default) or `SharedMultiSubpopulation { HydraKind, -HydraParams }`. It is part of the type because a shared Hydra structure and -independent instances are not merge-compatible even with the same algorithm. - -Because the full identity is in the type, incompatible states fail at plan -construction: a merge over `Sketch(Kll, …)` and `Sketch(Cms, …)`, or a `Sketch` -read as a `Sample`, is a schema error. +Based on our requirement, each field should contain the following information. +1. **What type of the ASAP Primitive is** A state column can be a raw data type (e.g., numerical number, string). It can also be a [summary type](TODO: add link), e.g., the summary family is sketch, and the sketch type is quantile KLL sketch algorithm, and KLL sketch has K as parameter as the schema. (TODO: confirm the terminology with corresponding code/doc) It has a family, an algorithm and parameters. +2. **What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents** This information is being mapped based on the primitive type. -**Rules for state fields.** -- **Top-level only.** Nested `List`/`Struct` elements are `Field`, not - `Field`, so a nested field cannot carry state. -- **Produced only by state operators.** `SummaryAgg.family` is never `Plain`; its - input must be values, not state. Its output is the grouping columns plus one - non-nullable `state` field of that family. -- **State is not a value.** `scalar_type` rejects a state column ("read it out - first"). `Filter`, `BinaryOp`, `Join`, `SetOp`, `Concat`, `Aggregate` and - `Dedup` reject `State` inputs; a bare-column `Project` may pass a state field - through unchanged (its result stays `State`). - Copying a state column does not make it readable. -- **Result kind.** `OperatorResultKind::State` marks an output carrying - unfinalized state; its schema may also contain plain grouping keys. Matching - columns never make result kinds interchangeable. +## 4. Proposed Node field design -**Readout / finalization boundary.** State becomes plain values only through an -explicit ASAP readout, which takes its input's relation/vector kind: - -| Readout | Input | Output field | -|---|---|---| -| `SummaryEstimate { query: SketchStatistic }` | exactly one `Sketch` state field whose category supports `query` | `Plain`: `quantile`/`frequency_l2`/`frequency_entropy` `Float64`, `cardinality`/`count` `Int64`, `topk` `Utf8` | -| `FinalizeExactAccumulator` | `ExactAggregate` state | the finalized aggregate value | -| `EvaluatePopulation` | `MaintainPopulation` state | the requested population statistic | - -For example, a KLL build outputs `State` with a `Sketch(KLL{k=200})` column; its -p99 readout outputs `Plain(Float64)`. A numeric predicate can use the readout but -not the state. - -### 2.3 Consideration 3: the metadata preserves summary semantics - -A state is only meaningful together with what it summarizes. The information a -summary's semantics depends on is: - -| Concern | Required semantic information | -|---|---| -| Input computation | Source identities and schemas, filters, joins/transforms and their order, or the canonical input sub-DAG | -| Values and grouping | Value expressions, item identities and weights, group keys and types, null/duplicate handling | -| Time | Time column and interpretation, interval bounds, evaluation alignment, query range versus maintained panes | -| Summary computation | Exact operation or sketch family, algorithm and parameters, build/merge behavior | -| Output | State versus finalized value, output schema/type, readout parameters | - -KLL over `latency_seconds` and KLL over `log(latency_seconds)` differ even with -identical source, filter, grouping and window. Weighted frequency state needs both -item and weight expressions. Four descriptive fields (`source`, `filter`, -`grouping`, `window`) cannot replace the computation DAG. - -The design splits this information by what it varies with, and records each fact -once: - -| Where | What it records | Why there | -|---|---|---| -| Field type (`FieldDataType`) | Family, algorithm, parameters, grouping layout | It determines merge compatibility and which readouts apply, so it gates schema equality. | -| Producer operator (`SummaryAgg`) | `input: SummaryUpdate` (item, weight, `weight_domain` proof), `reduction` (group keys or per-entity), `filter`, `grouping`; the child sub-DAG is the input computation | These are the operation's parameters; copying them elsewhere would need a consistency check. | -| Node (`OperatorNode.coverage`) | Which observations: a source and a union of time × population regions | It differs between states that must still merge, and it cannot be derived from `SummaryAgg` alone. | -| Result kind and readout node | State versus value, readout statistic | Derived from the operator (§2.2). | - -Time alignment, panes and maintenance lifecycle are planning and deployment -concerns ([planner layering](planner-layering.md), -[physical planning](../physical-planning-and-deployment.md)). - -#### Coverage is beside the schema, not inside it - -Coverage is not part of `Schema`. `SummaryMerge` requires equal input schemas, -and the inputs of every useful merge (`[0,1)` + `[1,2)`) have different coverage. -Coverage also describes the whole state output, not one field. It is not an -operator parameter either: a merge *derives* it from its inputs, like the schema. - -Requirements: - -1. Represent time and population **jointly**, per region, never as independent - bounds. -2. Accept a merge only when the inputs are **provably disjoint**; fail closed. - Merging does not imply that a family can remove duplicates. -3. Leave `Schema` and its equality unchanged. -4. Duplicate nothing the operator already records. -5. Support sources without a time column (plain tables). - -#### What coverage records - -`SummaryCoverage` names one observation `source`, using the same `Source` as -`Scan` (a table or a time series), and holds a **union of regions**. Each -`CoverageRegion` pairs: - -- `time_ms`: half-open bounds on the source's time column, or `None` for no time - restriction; -- `population`: a conjunction of non-null `label = value` predicates; empty means - all observations. - -Every observation in a region contributes once to the state. `regions = []` means -known empty coverage. What each observation contributes and how states are -grouped stay on `SummaryAgg.input` and `SummaryAgg.reduction` (requirement 4); -`SummaryMerge` compares those on its producers directly (#560). - -#### Composition is a provably disjoint union - -`merge_disjoint` accepts inputs with the same source whose regions are pairwise -disjoint. Two regions are disjoint when their time ranges do not intersect, or -when they assign different values to the same label. Different labels prove -nothing, and a region without time bounds overlaps any region it is not -population-disjoint from. The union keeps gaps and the time/population pairing; -adjacent intervals coalesce only when their populations are identical. - -| Case | Result | -|---|---| -| `[0,1)` + `[1,2)`, same population | one region `[0,2)` | -| `[0,1)` + `[2,3)` | two regions (gap kept) | -| `[0,2)` + `[1,3)` | rejected: possible overlap | -| `region=us` + `region=eu`, same time | two regions | -| `region=us` + `region=us`, or + `tier=premium` | rejected: possible overlap | -| `us×[0,1)` + `eu×[1,2)` | two regions, never `{us,eu}×[0,2)` | -| different source | rejected | - -Equality conjunctions are a deliberately narrow proof vocabulary. A richer -predicate needs an explicit disjointness rule before it can be declared. - -#### Lifecycle - -- **Required on summary nodes.** `SummaryAgg` cannot pass `validate_structure` - without coverage; `SummaryMerge` joins it in #560. Coverage on a non-`State` - node is rejected. The field is an `Option` only because all operators share - `OperatorNode`. -- **Declared at build.** The composition rule or catalog that builds a - `SummaryAgg` declares it (`with_coverage`). No production builder declares it - on this branch yet. -- **Derived at merge (#560).** `SummaryMerge` computes the disjoint union of its - inputs, and validation rejects a retained value that differs. -- **Cleared on rewrite.** `map_children` rebuilds the node without coverage, like - `guarantee` and `timing`. The rewriter must declare it again. -- **Preserved downstream (#537).** Logical export keeps coverage, and CSE shares - two nodes only if their coverage is equal. - -#### Trust boundary - -Declarations are trusted. Population is not yet checked against -`SummaryAgg.filter`, `Filter` nodes or `Scan.predicates`, so a wrong declaration -passes: - -```text -A = SummaryAgg(filter: region='us'), declared {region: eu} × [0,1) ← wrong -B = SummaryAgg(filter: region='us'), declared {region: us} × [0,1) -merge_disjoint(A, B) is accepted, and every US observation is counted twice. -``` - -[#570](https://github.com/ProjectASAP/ASAPPlanner/issues/570) adds the check: the -declared population must equal the `column = literal` predicates between the -`SummaryAgg` and its `Scan`. Time bounds stay trusted, because `TimeRange` is -relative to the evaluation time. - -## 3. Physical data: how a state column is carried - -The runtime uses the same `Schema` as planning (`SchemaRef = Arc`). The -native executor (`asap-physical-operators`) stores `Batch { schema, rows: -Vec> }`; the row/column layout is executor-specific. A state column -holds a typed value: +A node in the physical data will represent the data or summary instance, so a node has a field for **What data sources a ASAP primitive summarizes**. +Based on the above the proposed OperatorNode interface is as below: ```rust -enum Value { - Null, Bool(..), Int64(..), Float64(..), Utf8(..), Timestamp(..), Date(..), - Interval { .. }, List(..), Struct(..), Map(..), - Summary { family: FieldDataType, state: Arc }, -} ``` -- **Typed at the boundary.** `Batch::try_new` checks each `Summary` value's - `family` equals the field's `FieldDataType`, and that the payload's shape - (algorithm and parameters, for example KLL `k` or CMS width × depth) matches it. - State fields must be non-nullable and of a natively supported family. -- **Not a key.** A `Summary` value cannot be a grouping key or be ordered. -- **Kernels.** `summary_kernels` adapt `asap_sketchlib` structures and exact - Planner state behind `AggregateCore`: `merge_with` (same family and shape), - `estimate(SketchStatistic)`, and `approx_memory_bytes` for memory reservations. - `create_planner_accumulator(family, input, grouping)` builds the updater a - `SummaryAgg` declares and rejects a family/grouping disagreement. -- **Native coverage.** Exact Sum/Count/Min/Max/Rate/Increase, KLL, DDSketch, HLL, - Count-Min (stored state only), and weighted CMS/CountSketch with heaps. - `SharedMultiSubpopulation` grouping, `Sample`, `Wavelet` and `StatModel` have - no native kernel and are rejected at binding. -- **No encoding here.** `Value::Summary` is not serialized; byte encodings belong - to `asap_sketchlib` and deployments. - -Coverage is plan metadata and is not carried in runtime values. Physical merge of -states by group key checks family equality only; disjointness is proven at -planning time (§2.3). +## 5. Examples on how OperatorNode, schema, and physical data information are being used with Summary operators -## 4. Alternatives considered - -| Alternative | Why not | -|---|---| -| Separate pre-ASAP and post-ASAP schema types | A projection above a summary needs a different representation from one below it; one `Schema` removes the barrier. | -| An opaque "state" type without family identity | KLL + CMS merges, and sketch-versus-sample confusion, would only fail at runtime. | -| State inside `List`/`Struct` fields | Nested state would escape the readout boundary and state validation. | -| Put coverage in `Schema` | Schema equality gates merges; merge inputs always differ in coverage. | -| One time range plus one label set | Invents the missing blocks (Example 3). | -| Copy `input` and `reduction` into coverage | Duplicates `SummaryAgg` and needs a consistency check; producers already carry them. | -| Arbitrary predicates per region | No general disjointness proof; overlap would be silently accepted. | -| Free-form string `source` | Two spellings of one table compare unequal; `Scan` already has `Source`. | -| Snapshot `revision` field | Deployment concern; the planner does not own catalog versions. | - -## 5. Key code interfaces - -```rust -// crates/types/src/pre_asap/schema.rs -pub type ColumnId = usize; -pub struct Field { pub name: String, pub dtype: T, pub nullable: bool, pub table: Option } -pub struct Schema { - pub fields: Vec, - pub time_index: Option, - pub unique_keys: Vec>, - pub closed: bool, -} -pub enum FieldDataType { Plain(DataType), ExactAggregate(..), Sketch(SketchKind, GroupingStrategy), Sample(..), Wavelet(..), StatModel(..) } -// DataType::List { element: Box> }, DataType::Struct { fields: Vec> } - -// crates/types/src/post_asap/sketch.rs -impl SketchKind { pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; } -pub enum GroupingStrategy { PerSubpopulationInstance, SharedMultiSubpopulation { kind: HydraKind, params: HydraParams } } -pub struct SummaryUpdate { pub item: Option, pub weight: SummaryInputExpr, pub weight_domain: WeightDomain } - -// crates/types/src/ir/asap.rs -pub enum ASAPOp { - SummaryAgg { child, family: FieldDataType, input: SummaryUpdate, reduction: Reduction, - grouping: GroupingStrategy, filter: Option }, - SummaryEstimate { summary_input, query: SketchStatistic }, - FinalizeExactAccumulator { child }, - SummaryMerge { children }, // reserved here; structure in #560 - // MaintainPopulation, EvaluatePopulation, SummarySubtract, SummaryDelete, SummaryJoin, Extension -} - -// crates/types/src/ir/summary_coverage.rs -pub struct SummaryCoverage { pub source: Source, pub regions: Vec } -pub struct CoverageRegion { - pub time_ms: Option>, // half-open; None = no time restriction - pub population: BTreeMap, // label = value AND …; empty = all -} -impl SummaryCoverage { - pub fn validate(&self) -> Result<(), CoverageError>; - pub fn merge_disjoint(inputs: &[Self]) -> Result; -} -pub enum CoverageError { - InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, - NotState, Missing, - // #560: UnknownInput, MergeOutputMismatch -} - -// crates/types/src/ir/node.rs -pub enum OperatorResultKind { Relation, InstantVector, RangeVector, State } -pub struct OperatorNode { - pub operator: Operator, pub result_kind: OperatorResultKind, pub schema: Schema, - pub guarantee: Option, pub timing: Option, - pub coverage: Option, -} -impl OperatorNode { - pub fn with_schema(operator: Operator, schema: Schema) -> Self; - pub fn with_coverage(self, c: SummaryCoverage) -> Result; - pub fn requires_coverage(&self) -> bool; // SummaryAgg; SummaryMerge in #560 - pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError>; - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError>; - // #560: pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; -} -// SchemaDerivationError::Coverage(CoverageError) reports coverage failures. - -// crates/asap-physical-operators/src/{values.rs, summary_kernels/traits.rs} -pub enum Value { /* plain variants */ Summary { family: FieldDataType, state: Arc } } -pub trait AggregateCore { - fn merge_with(&self, other: &dyn AggregateCore) -> Result, KernelError>; - fn estimate(&self, query: &SketchStatistic) -> Result; - fn approx_memory_bytes(&self) -> usize; -} -``` +Given that these information requirements are introduced by summary operators to work correctly semantically, we show the examples of how the defined OperatorNode, schema, and physical data information work with each kind of summary operators. -Coverage composition is tested in `crates/types/tests/summary_coverage.rs`. The -documented examples are built as real `Scan → SummaryAgg → SummaryMerge` plans in -`crates/types/tests/summary_coverage_examples.rs` (#560). From 87851b7da173bd1ef28bb53c93a1d7e755726ab1 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:45:44 +0000 Subject: [PATCH 15/18] docs: add code interfaces and per-operator examples to the ASAP primitive schema design Co-Authored-By: Claude Opus 5.5 --- .../proposals/asap-primitive-schema.md | 389 +++++++++++++++++- 1 file changed, 385 insertions(+), 4 deletions(-) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md index 9f66ba70c..2302e3040 100644 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ b/docs/design_docs/proposals/asap-primitive-schema.md @@ -7,7 +7,7 @@ This document is the single source of truth for the schema, and column design fo Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, it introduces new requirement when we design the schema and node definitions for LogicalASAPDAG and PhysicalASAPDAG. Assuming we have the Logical DAG defined for a canonicalized representation for a batch of queries. [TODO: add links for this here. ] -The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives: SummaryCreation?, SummaryUpdate, SummaryMerge, SummaryDelete, SummarySubtraction, SummaryEstimate [TODO: check what is the complete list or discuss with others about the list]. +The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives. The complete list is `ASAPOp` in `crates/types/src/ir/asap.rs`: `SummaryAgg` (creates and updates state; there is no separate SummaryCreation or SummaryUpdate operator, and `SummaryUpdate` is `SummaryAgg`'s update-expression parameter), `SummaryEstimate`, `FinalizeExactAccumulator`, `MaintainPopulation` and `EvaluatePopulation` (implemented); `SummaryMerge` (reserved here, enabled by #560); and `SummarySubtract`, `SummaryDelete`, `SummaryJoin` and `Extension` (reserved). See §5. Each of the Summary operators also require the ASAP primitive information above to inter-operate correctly, preserving semantic correctness. Basically, the following information should be represented to preserve the equivalent query semantics when we introduce ASAP Primitives to logical query representation, and following physical one. @@ -33,19 +33,400 @@ Schema represents the **metadata** of information flow along an **edge** between Schema definition here is shared between LogicalDAG, LogicalASAPDAG, and PhysicalASAPDAG. The schema contain fields, and each field is mapping to a column in the physical data representation. Based on our requirement, each field should contain the following information. -1. **What type of the ASAP Primitive is** A state column can be a raw data type (e.g., numerical number, string). It can also be a [summary type](TODO: add link), e.g., the summary family is sketch, and the sketch type is quantile KLL sketch algorithm, and KLL sketch has K as parameter as the schema. (TODO: confirm the terminology with corresponding code/doc) It has a family, an algorithm and parameters. -2. **What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents** This information is being mapped based on the primitive type. +1. **What type of the ASAP Primitive is** A state column can be a raw data type (e.g., numerical number, string). It can also be a summary type (§6.1, §6.2), e.g., the summary **family** is sketch (`FieldDataType::Sketch`), its **category** is quantile (`SketchCategory::Quantile`), its **algorithm** is KLL (`SketchAlgorithm::Kll`), and its **parameters** are `SketchParams::Kll { k: 200 }`. A sketch field also records its `GroupingStrategy` (one instance per group, or one shared Hydra structure). Non-sketch families (`ExactAggregate`, `Sample`, `Wavelet`, `StatModel`) have a kind and parameters but no category. +2. **What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents** This information is being mapped based on the primitive type: it is not stored in the field. `SummaryEstimate::validate_inputs` accepts a readout only when the `SketchStatistic` matches the state's `SketchCategory` (Quantile→`Quantile`; Cardinality→`Cardinality`/`Universal`; PointCount→`Frequency`/`Universal`; FrequencyL2/FrequencyEntropy→`Universal`; TopK→`TopK`/`Universal`). Exact accumulators are read by `FinalizeExactAccumulator` instead. `Sample`, `Wavelet` and `StatModel` state has no readout operator yet. +Whether an edge carries **state or values** is recorded on the producing node as `OperatorResultKind`, not inferred from field types. Usually they agree: a `SummaryAgg` output has exactly one non-plain field and kind `State`. They can differ, though. `MaintainPopulation` keeps an all-plain schema but its kind is `State`, and a `Project` that passes a state column through keeps kind `State`. Consumers check `result_kind` (`validate_inputs` requires `State` for every readout). ## 4. Proposed Node field design A node in the physical data will represent the data or summary instance, so a node has a field for **What data sources a ASAP primitive summarizes**. -Based on the above the proposed OperatorNode interface is as below: +In code this field is `OperatorNode::coverage`. It records *which observations* a summary state covers: a source plus a union of joint (time × population) regions. It sits beside the schema, not inside a field, because two states with identical schemas can cover different data, and only disjoint coverage can be merged once-per-observation. + +Based on the above the proposed OperatorNode interface is as below (`crates/types/src/ir/node.rs`, `ir/summary_coverage.rs`): ```rust +pub enum OperatorResultKind { Relation, InstantVector, RangeVector, /** unfinalized summary/accumulator state */ State } + +pub enum Operator { NonASAP(NonASAPOp), ASAP(ASAPOp) } + +pub struct OperatorNode { + pub operator: Operator, + pub result_kind: OperatorResultKind, // derived from operator + children + pub schema: Schema, // derived; may override names/qualifiers only + pub guarantee: Option, // None until accuracy assessment; None != exact + pub timing: Option, // IngestionTime | QueryTime; None until assigned + #[serde(default)] + pub coverage: Option, // observations a State output summarizes +} + +impl OperatorNode { + pub fn new(op: Operator) -> Result; // derives schema + kind; coverage = None + pub fn with_schema(op: Operator, schema: Schema) -> Self; // caller-supplied names + pub fn new_shared(op: Operator) -> Result, SchemaDerivationError>; + pub fn with_guarantee(self, g: Option) -> Self; + pub fn with_timing(self, t: Option) -> Self; + /// Validates the coverage, then rejects a non-State node (CoverageError::NotState). + pub fn with_coverage(self, c: SummaryCoverage) -> Result; + /// This branch: true only for SummaryAgg. #560: SummaryAgg | SummaryMerge. + pub fn requires_coverage(&self) -> bool; + /// #560: (update expression, reduction) of a SummaryAgg, or shared by a SummaryMerge's inputs. + pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; + /// Rebuilds with new inputs; re-derives schema; clears guarantee, timing and coverage. + pub fn map_children(&self, f: impl FnMut(&Rc) -> Rc) -> Result; + /// Per node: coverage well-formed (and present if required), validate_inputs, + /// result_kind and schema structure agree with derivation. + pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError>; + pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError>; + // also: asap(), non_asap(), is_asap(), children(), contains_asap(), reachable() +} + +pub struct SummaryCoverage { + pub source: Source, // Source::Table { table_ref } | Source::TimeSeries { metric } + pub regions: Vec, // union of joint regions, never a Cartesian product +} +pub struct CoverageRegion { + pub time_ms: Option>, // half-open, on the source's time column; None = unrestricted + pub population: BTreeMap, // conjunction of equality predicates; empty = unrestricted +} +impl SummaryCoverage { + /// Intervals non-empty, dimension names non-empty, regions pairwise provably disjoint. + pub fn validate(&self) -> Result<(), CoverageError>; + /// Union of provably disjoint inputs from one source; coalesces adjacent + /// intervals with identical populations and keeps gaps. + pub fn merge_disjoint(inputs: &[Self]) -> Result; +} +pub enum CoverageError { + InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, NotState, Missing, + UnknownInput, // #560: a merge input has no coverage + MergeOutputMismatch, // #560: retained merge coverage differs from the input union +} ``` +Two regions are provably disjoint only if their time ranges do not intersect or they bind the same population dimension to different values. A region without time bounds overlaps any region it is not population-disjoint from. Coverage is caller-established: `validate` checks that it is well-formed, not that it matches the child's predicates. Any rewrite through `map_children` drops it, so the rewriter must declare it again. `#537` adds export and CSE of the logical DAG (`ir/export.rs`, `ir/cse.rs`); no interface in this document depends on it. + ## 5. Examples on how OperatorNode, schema, and physical data information are being used with Summary operators Given that these information requirements are introduced by summary operators to work correctly semantically, we show the examples of how the defined OperatorNode, schema, and physical data information work with each kind of summary operators. +Notation: an edge is written `──Kind(field Type, …)──▶`. Schemas are the ones `output_schema()` derives. Planning may rename fields through `OperatorNode::with_schema`, but types, nullability, `time_index`, `unique_keys` and `closed` must match the derivation. All examples use a table source, so values are `Relation`; with a `TimeSeries` source the value side is `InstantVector`. + +### 5.1 `SummaryAgg`: values → state + +Scenario: p99 latency by job, from KLL(k=200), over one minute of table `t`. + +```text +Scan(t: job Utf8, latency Float64) + ──Relation(job Utf8, latency Float64)──▶ +SummaryAgg(family = Sketch(KLL{k=200}, PerSubpopulationInstance), + input = SummaryUpdate::column(Named("latency")), reduction = by[job], + grouping = PerSubpopulationInstance, filter = None) + ──State(job Utf8, state Sketch(KLL{k=200}, PerSubpopulationInstance))──▶ + coverage = { source: Table "t", regions: [{ time_ms: 0..60_000, population: {} }] } +``` + +- Output schema: the `by` keys followed by one non-nullable field `state` typed `family`; `unique_keys = [[0]]`, `closed = true`, no `time_index`. With `Reduction::PerEntity` the input columns are kept and the sample-value column is replaced by `state`. +- Checks: `family` is not `Plain`; the child is not `State`; the `weight`/`item` columns resolve against the child schema; `filter`, if present, types as `Bool`. +- Coverage: **required** and **declared**. `OperatorNode::new` leaves it `None`, `validate_structure` fails with `CoverageError::Missing`, and the planner attaches it with `with_coverage`. +- Boundary: this is where values become state. The sketch family, algorithm and parameters are committed in the field type, and `guarantee` stays `None` because state is not a caller-visible value. + +### 5.2 `SummaryEstimate`: sketch state → value + +Scenario: read p99 from the state in 5.1. + +```text +──State(job Utf8, state Sketch(KLL{k=200}))──▶ +SummaryEstimate(query = SketchStatistic::Quantile { q: 0.99 }) + ──Relation(job Utf8, quantile Float64)──▶ (planner may rename to p99) +``` + +- Output schema: the input schema with the one non-plain field replaced by a non-nullable plain field. Its name and type come from the statistic: `quantile`/`frequency_l2`/`frequency_entropy` Float64, `cardinality`/`count` Int64 (Float64 if the producer is a `PerEntity` `SummaryAgg`), and `topk` Utf8. Keys and metadata pass through. +- Result kind: the value kind of the source the state was built from (`Relation` here). +- Checks: input is `State` with exactly one non-plain field, that field is `Sketch`, and its category accepts the statistic (§3). For example, `Cardinality` on KLL is rejected. +- Coverage: **absent**. The output is a value, and `with_coverage` returns `NotState`. +- Boundary: state is consumed and a value is produced; `guarantee` on this node carries the readout's error bound. + +### 5.3 `FinalizeExactAccumulator`: exact state → value + +Scenario: total bytes by host with an exact Sum accumulator. + +```text +Scan(t: host Utf8, bytes Float64) + ──Relation(host Utf8, bytes Float64)──▶ +SummaryAgg(family = ExactAggregate(Sum, Sum), input = column(Named("bytes")), reduction = by[host]) + ──State(host Utf8, state ExactAggregate(Sum, Sum))──▶ coverage: required, declared +FinalizeExactAccumulator + ──Relation(host Utf8, state Float64)──▶ +``` + +- Output schema: each `ExactAggregate` field keeps its name (`state`) and takes the type and nullability the equivalent `NonASAPOp::Aggregate` would give: Sum/Min/Max follow the input column, Count is Int64, and Rate/IRate/Increase are Float64. If the child is not a `SummaryAgg` directly, Count falls back to Int64 and the others to Float64. `unique_keys`, `closed` and `time_index` are preserved (`schema_rebuilding.rs`). +- Checks: the input is `State` and contains an `ExactAggregate` field; a sketch is rejected (`structure_contract.rs`). +- Coverage: **absent** on the output. +- Boundary: this is the explicit maintenance-to-read boundary for exact state. Exact state is never read through `SummaryEstimate`. + +### 5.4 `MaintainPopulation`: values → maintained membership (state) + +Scenario: keep the full latency population per job, so that p99 and top-10 can be evaluated later. + +```text +Scan(t: job Utf8, latency Float64) [closed schema] + ──Relation(job Utf8, latency Float64)──▶ +MaintainPopulation(population = MaintainedPopulation { + input: PopulationInput::Rows { input: , value_column: 1, grouping: by[job] }, + max_k: 10, quantiles: true }) + ──State(job Utf8, latency Float64)──▶ +``` + +- Output schema: identical to the child's, all plain. Only `result_kind = State` marks it as maintained state. +- Checks: `population.matches_node(child)`. For `Rows`, the child must be the same closed table `Scan`, the value column must be non-null Float64, and grouping must be `by` with in-range keys. For `CurrentSeries`, it must be a `TimeSeries` scan with the same metric, matchers and grouping labels, under an instant `TimeRange` of `lookback_ms` (which may be omitted only for the default 300 s lookback). +- Coverage: **not required**. `with_coverage` accepts it because the output is `State`. +- Boundary: the output is state because it must also track membership changes; downstream operators can only read it through `EvaluatePopulation`. + +### 5.5 `EvaluatePopulation`: maintained membership → value + +Scenario: p99 by job from the population in 5.4. + +```text +──State(job Utf8, latency Float64) [from MaintainPopulation]──▶ +EvaluatePopulation(evaluation = PopulationStatistic::Quantile { q: 0.99 }) + ──Relation(job Utf8, quantile_0_99 Float64)──▶ +``` + +- Output schema: the schema of `Aggregate(by grouping, measure)` over the maintained source. Quantile gives `quantile_` Float64, Sum gives `sum` (value type), Count gives `count` Int64 and Average gives `avg` Float64; `unique_keys = [[0]]`, `closed`. `TopK { k }` instead returns the source schema unchanged (the selected rows). +- Checks: the child is a `MaintainPopulation` node whose `supports(evaluation)` holds: `quantiles` must be set for `Quantile`, and `k <= max_k` for `TopK`. +- Coverage: **absent**. +- Boundary: maintained membership is read as a value; the result kind is the source's (`Relation`). + +### 5.6 `SummaryMerge` (#560): state × N → state + +On this branch, `SummaryMerge { children }` is **reserved**. `is_unimplemented()` returns true, and `output_schema()` and `validate_inputs()` return `UNIMPLEMENTED_ASAP_OP`, so `OperatorNode::new` fails. Only `output_kind()` (= `State`), `children`, `map_children` and `kind_name` work. #560 enables it as follows (`summary_merge_structure.rs` in #560). + +Scenario: combine two one-minute KLL panes over `Scan(t: value Float64)` into a two-minute state. + +```text +SummaryAgg(KLL k=200, column(SampleValue), by[]) ──State(state Sketch(KLL{k=200}))── coverage {t, [0..60_000)} ─┐ +SummaryAgg(KLL k=200, column(SampleValue), by[]) ──State(state Sketch(KLL{k=200}))── coverage {t, [60_000..120_000)} ─┴▶ +SummaryMerge + ──State(state Sketch(KLL{k=200}))──▶ coverage = { t, [0..120_000) } (derived) +``` + +- Output schema: `children[0].schema`. +- Checks: at least one input; exactly one state column; every input is `State` with an identical schema (so family, params, grouping strategy and key positions match); every input has the same `summary_update()` (update expression and reduction); and `merged_coverage()` succeeds. Merging k=200 with k=300 fails, and so does merging raw rows. +- Coverage: **required** and **derived**. `OperatorNode::new` sets it to `SummaryCoverage::merge_disjoint` of the input coverages. An input without coverage gives `UnknownInput`, and overlapping inputs give `PossibleOverlap`. `validate_structure` rejects a retained coverage that differs from the derived one (`MergeOutputMismatch`). Gapped inputs stay as two regions. +- Boundary: state in, state out. No value is produced until a readout. + +### 5.7 Reserved operators (not implemented) + +These variants exist so that plans can name them, but `output_schema()`/`validate_inputs()` return `UNIMPLEMENTED_ASAP_OP`, so no node can be built. `output_kind()` already returns `State` for each of them. The intended edge shapes below follow from their fields; none of them is implemented. + +| Operator | Fields | Intended edge shape | +|---|---|---| +| `SummarySubtract` | `left, right` | State × State → State: remove one window's contribution, e.g. [0,10) − [0,5) | +| `SummaryDelete` | `summary_input, key: ColumnId` | State → State with the entries for `key` removed | +| `SummaryJoin` | `outer, inner, key, family` | State × State → State typed `family` (`produced_state()` returns it), e.g. join-size estimation | +| `Extension` | `child, name` | deployment-named state operator | + +### 5.8 Summary + +| Operator | Input kind | Output kind | Output carries state | Coverage on output | Status | +|---|---|---|---|---|---| +| `SummaryAgg` | value (not `State`) | `State` | yes (one `family` field) | required, declared | implemented | +| `SummaryEstimate` | `State` (one `Sketch` field) | source's value kind | no | absent | implemented | +| `FinalizeExactAccumulator` | `State` (`ExactAggregate`) | source's value kind | no | absent | implemented | +| `MaintainPopulation` | `Relation` (table) / `InstantVector` (series) | `State` | yes (by kind; fields plain) | optional, not required | implemented | +| `EvaluatePopulation` | `State` from `MaintainPopulation` | source's value kind | no | absent | implemented | +| `SummaryMerge` | `State` × N | `State` | yes | required, derived | reserved; enabled by #560 | +| `SummarySubtract` | `State` × 2 | `State` | yes | — | reserved | +| `SummaryDelete` | `State` | `State` | yes | — | reserved | +| `SummaryJoin` | `State` × 2 | `State` | yes | — | reserved | +| `Extension` | any | `State` | yes | — | reserved | + +## 6. Key code interfaces + +`OperatorNode`, `OperatorResultKind` and coverage are in §4. Bodies and serde/derive attributes are elided below. + +### 6.1 Schema and field types (`crates/types/src/pre_asap/schema.rs`) + +```rust +pub type ColumnId = usize; + +pub struct Schema { + pub fields: Vec, + pub time_index: Option, // must point at a plain Timestamp field + pub unique_keys: Vec>, + pub closed: bool, // true = fields enumerate every column +} +impl Schema { + pub fn new(fields: Vec) -> Self; + pub fn with_time_index(fields: Vec, time_index: ColumnId, unique_keys: Vec>) -> Self; + pub fn lifted(fields: Vec, time_index: Option) -> Self; // closed = true + pub fn is_all_plain(&self) -> bool; + pub fn column_id(&self, name: &str) -> Option; + pub fn column_id_qualified(&self, table: &str, name: &str) -> Option; +} + +pub struct Field { + pub name: String, + pub dtype: T, + pub nullable: bool, + pub table: Option, +} +impl Field { + pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self; + pub fn plain_dtype(&self) -> Option<&DataType>; + pub fn is_plain(&self) -> bool; +} + +/// A column's type: a plain value, or summary state of one family. +pub enum FieldDataType { + Plain(DataType), + ExactAggregate(ExactKind, ExactParams), + Sketch(SketchKind, GroupingStrategy), + Sample(SamplingKind, SamplingParams), + Wavelet(WaveletKind, WaveletParams), + StatModel(StatModelKind, StatModelParams), +} + +pub enum DataType { + Null, Int64, Float64, Utf8, Bool, Timestamp, Interval, Date, + List { element: Box> }, + Struct { fields: Vec> }, + Map { key: Box, value: Box, value_nullable: bool }, +} +``` + +### 6.2 State-family parameters (`crates/types/src/post_asap/sketch.rs`) + +```rust +pub enum ExactKind { Sum, Count, Min, Max, Increase, Rate, IRate } +pub enum ExactParams { Sum, Count, Min, Max, Increase, Rate, IRate } // no knobs; mirrors kind + +pub struct SketchKind { category: SketchCategory, algorithm: SketchAlgorithm, params: SketchParams } +impl SketchKind { + /// The only constructor; classifies the category and panics on mismatched params. + pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; + pub fn category(&self) -> SketchCategory; + pub fn algorithm(&self) -> &SketchAlgorithm; + pub fn params(&self) -> &SketchParams; +} +pub enum SketchCategory { Universal, Quantile, Cardinality, Frequency, TopK } +// Universal: UnivMon | Quantile: Kll, DDSketch | Cardinality: Hll, Theta, Kmv +// Frequency: Cms, CountSketch | TopK: CmsWithHeap, CountSketchWithHeap +pub enum SketchAlgorithm { UnivMon, Kll, Cms, Hll, DDSketch, CmsWithHeap, Kmv, Theta, CountSketch, CountSketchWithHeap } +pub enum SketchParams { + UnivMon { heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8 }, + Kll { k: u32 }, + Cms { width: u32, depth: u32 }, + Hll { precision: u8 }, + DDSketch { alpha: f64 }, + CmsWithHeap { width: u32, depth: u32, heap_size: u32 }, + Kmv { k: u32 }, + Theta { k: u32 }, + CountSketch { width: u32, depth: u32 }, + CountSketchWithHeap { width: u32, depth: u32, heap_size: u32 }, +} + +/// How grouped state is instantiated across `by` subpopulations. Orthogonal to family. +pub enum GroupingStrategy { + PerSubpopulationInstance, // Default + SharedMultiSubpopulation { kind: HydraKind, params: HydraParams }, +} +pub enum HydraKind { HydraKll /* experimental, no error bound */, HydraCms, HydraCountSketch } +pub enum HydraParams { + HydraKll { k: u32, shared_buckets: u32 }, + HydraCms { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, + HydraCountSketch { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, +} +pub fn hydra_kind_for(a: &SketchAlgorithm) -> Option; // Cms, CountSketch only + +pub enum SamplingKind { Reservoir } pub enum SamplingParams { Reservoir { size: u32 } } +pub enum WaveletKind { Haar } pub enum WaveletParams { Haar { coefficients: u32 } } +pub enum StatModelKind { Parametric } pub enum StatModelParams { Parametric { family: String } } +``` + +### 6.3 Update input and readouts (`post_asap/sketch.rs`, `post_asap/maintained_population.rs`) + +```rust +/// One state update: `item` keys the update for keyed families; `weight` is applied to state. +pub struct SummaryUpdate { + pub item: Option, + pub weight: SummaryInputExpr, + pub weight_domain: WeightDomain, // serde default: UnknownOrSigned +} +impl SummaryUpdate { pub fn column(c: ColumnRef) -> Self; } // item None, UnknownOrSigned +pub enum WeightDomain { + UnknownOrSigned, // Default; never assumed non-negative + NonNegative { proof: NonNegativeWeightProof }, +} +pub enum NonNegativeWeightProof { UnitCount, ResetAwareCounterDerivative } +pub enum SummaryInputExpr { + Constant(f64), Column(ColumnRef), Tuple(Vec), EntityIdentity(EntityIdentity), +} +pub enum EntityIdentity { PromqlLabelSet { excluding: Vec } } + +/// Readout of sketch state, carried by SummaryEstimate. +pub enum SketchStatistic { + FrequencyL2, FrequencyEntropy, + Quantile { q: f64 }, + PointCount { key: ColumnRef, value: Option }, + Cardinality, + TopK { k: usize }, +} + +/// Readout of a maintained population, carried by EvaluatePopulation. +pub enum PopulationStatistic { Quantile { q: f64 }, TopK { k: usize }, Sum, Count, Average } +pub struct MaintainedPopulation { + pub input: PopulationInput, + pub max_k: usize, // largest TopK it supports + pub quantiles: bool, // whether Quantile is supported +} +pub enum PopulationInput { + CurrentSeries(CurrentSeriesInput), // metric, matchers, grouping, without, lookback_ms + Rows { input: Rc, value_column: usize, grouping: GroupKeys }, +} +``` + +A finalized value's accuracy statement is `ResultGuarantee { metric, bound, failure_probability, provenance }` (`post_asap/guarantee.rs`). It is attached to readout and finalized nodes, never to raw state. + +### 6.4 ASAP operators (`crates/types/src/ir/asap.rs`) + +```rust +pub const UNIMPLEMENTED_ASAP_OP: &str = + "this ASAP operator is reserved: schema, accuracy, timing and export are not implemented"; + +pub enum ASAPOp { + SummaryAgg { + child: Rc, + family: FieldDataType, // never Plain + input: SummaryUpdate, + reduction: Reduction, // Reduce(GroupKeys) | PerEntity + grouping: GroupingStrategy, + filter: Option, // serde default None + }, + SummaryEstimate { summary_input: Rc, query: SketchStatistic }, + FinalizeExactAccumulator { child: Rc }, + MaintainPopulation { child: Rc, population: MaintainedPopulation }, + EvaluatePopulation { child: Rc, evaluation: PopulationStatistic }, + // Reserved on this branch; #560 implements SummaryMerge. + SummaryMerge { children: Vec> }, + SummarySubtract { left: Rc, right: Rc }, + SummaryDelete { summary_input: Rc, key: ColumnId }, + SummaryJoin { outer: Rc, inner: Rc, key: ColumnId, family: FieldDataType }, + Extension { child: Rc, name: String }, +} + +impl ASAPOp { + pub fn children(&self) -> Vec<&Rc>; // SummaryAgg includes its filter's subquery nodes + pub fn map_children(&self, f: impl FnMut(&Rc) -> Rc) -> Self; + pub fn kind_name(&self) -> &'static str; + /// Merge, Subtract, Delete, Join, Extension on this branch; #560 removes Merge. + pub fn is_unimplemented(&self) -> bool; + /// SummaryAgg/SummaryJoin `family`; #560 adds SummaryMerge (its inputs' state type). + pub fn produced_state(&self) -> Option<&FieldDataType>; + /// #560: merge_disjoint of the children's coverage; fails closed. + pub fn merged_coverage(&self) -> Result; + pub fn output_schema(&self) -> Result; + pub fn output_kind(&self) -> OperatorResultKind; + pub fn validate_inputs(&self) -> Result<(), SchemaDerivationError>; +} +``` From 89804d51c0468f86324359df8394861554ab6e28 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Sat, 3 Oct 2026 20:47:13 +0000 Subject: [PATCH 16/18] docs: move the ASAP primitive schema design to #573 The design document and the docs it consolidates are reviewed separately on main. This PR keeps code, tests and the ScanSelection rename in docs. Co-Authored-By: Claude Opus 5.5 --- docs/design_docs/concepts/post-asap-ir.md | 3 +- .../physical-planning-and-deployment.md | 28 +- docs/design_docs/proposals/README.md | 1 - .../proposals/asap-primitive-schema.md | 432 ------------------ .../proposals/decoupling_op_and_expr.md | 2 +- .../design_docs/proposals/operator-sharing.md | 156 ++++++- .../proposals/univmon-frequency-summary.md | 3 +- .../asap-aware-mapping-contracts.md | 24 +- docs/develop_docs/pre-asap-ir.md | 24 +- 9 files changed, 210 insertions(+), 463 deletions(-) delete mode 100644 docs/design_docs/proposals/asap-primitive-schema.md diff --git a/docs/design_docs/concepts/post-asap-ir.md b/docs/design_docs/concepts/post-asap-ir.md index 08c670af4..6c9aa1461 100644 --- a/docs/design_docs/concepts/post-asap-ir.md +++ b/docs/design_docs/concepts/post-asap-ir.md @@ -49,8 +49,7 @@ summary family supports incremental maintenance. and selects the joined rows. Completeness evidence belongs to pruning, not ranking. A `SummaryNode` carries its expression, schema and optional result guarantee. -State and query values have different contracts; see -[Schema and physical data for ASAP primitives](../proposals/asap-primitive-schema.md). Exact operations over +State and query values have different contracts. Exact operations over approximate readouts still require composed accuracy guarantees. See the [accuracy implementation companion](../../develop_docs/end-to-end-accuracy-guarantees.md) and [physical-plan integration](../architecture/physical-plan-integration.md) diff --git a/docs/design_docs/physical-planning-and-deployment.md b/docs/design_docs/physical-planning-and-deployment.md index c48dfed54..274e4974c 100644 --- a/docs/design_docs/physical-planning-and-deployment.md +++ b/docs/design_docs/physical-planning-and-deployment.md @@ -117,14 +117,26 @@ operators from logical candidates has not completed this integration. ### Input semantics and summary semantics -`source`, `filter`, `grouping` and `window` describe input-data semantics but not -a complete summary computation: the same four fields can summarize different -value expressions or produce different states. The semantic information a summary -depends on, and where the IR records each part (field type, producing operator, -or coverage), is specified in -[Schema and physical data for ASAP primitives](proposals/asap-primitive-schema.md#23-consideration-3-the-metadata-preserves-summary-semantics). - -The canonical selected computation is authoritative. Those categories describe +`source`, `filter`, `grouping` and `window` describe input-data semantics: +where records originate, which records qualify, how they are grouped and which +time interval applies. They are not a complete description of arbitrary summary +computation. In particular, the same four fields can summarize different value +expressions or produce different states. + +| Concern | Required semantic information | +| --- | --- | +| Input computation | Source identities and schemas, filters, joins/transforms and their order, or a reference to the canonical input sub-DAG | +| Values and grouping | Value expressions, item identities and weights where applicable, group keys and types, and operation-defined null/duplicate handling | +| Time | Time column and interpretation, interval bounds, evaluation alignment, and distinction between query range and maintained panes | +| Summary computation | Exact operation or sketch family, algorithm and parameters, and supported build/merge behavior | +| Output | State versus finalized value, output schema/type, and readout parameters when part of the output computation | + +For example, KLL over `latency_seconds` and KLL over `log(latency_seconds)` differ +even with identical source, filter, grouping and window. Likewise, weighted +frequency state needs both item and weight expressions. More complex inputs +must retain their computation DAG; four descriptive fields cannot replace it. + +The canonical selected computation is authoritative. These categories describe what must be preserved, not a new flat IR or a second expression language. Operator-defined behavior should be referenced through its canonical contract, not independently configured in deployment metadata. Unsupported or unresolved diff --git a/docs/design_docs/proposals/README.md b/docs/design_docs/proposals/README.md index 6c4ed49f8..cb44fadbb 100644 --- a/docs/design_docs/proposals/README.md +++ b/docs/design_docs/proposals/README.md @@ -11,4 +11,3 @@ extensions. A design document is not a promise of downstream runtime support. - [Operator sharing](operator-sharing.md) - [Decoupling operators from scalar expressions](decoupling_op_and_expr.md) - [ASAPPlanner layering](planner-layering.md) -- [Schema and physical data for ASAP primitives](asap-primitive-schema.md) diff --git a/docs/design_docs/proposals/asap-primitive-schema.md b/docs/design_docs/proposals/asap-primitive-schema.md deleted file mode 100644 index 2302e3040..000000000 --- a/docs/design_docs/proposals/asap-primitive-schema.md +++ /dev/null @@ -1,432 +0,0 @@ -# Schema and Physical Data for ASAP Primitives - -This document is the single source of truth for the schema, and column design for ASAP Primitives. This is used in the logical stage (LogicalASAPDAG), and physical stage (PhysicalASAPDAG). - -## 1. Goal, problem, and requirements - -Unlike existing Database engines, which work on raw data or explicitly defined materialized tables with schema and column names provided by the users, ASAPPlanner is designed for querying and execution over the mix of raw data and ASAP Primitives. ASAP primitives are usually compact summaries over raw data. Therefore, it introduces new requirement when we design the schema and node definitions for LogicalASAPDAG and PhysicalASAPDAG. - -Assuming we have the Logical DAG defined for a canonicalized representation for a batch of queries. [TODO: add links for this here. ] -The LogicalASAPDAG will share/reuse the NonASAP operator and ScalarExpr nodes in LogicalDAG [TODO: link PR 511's doc here], but replacing some operators in LogicalDAG with the operators operated with ASAP Primitives. The complete list is `ASAPOp` in `crates/types/src/ir/asap.rs`: `SummaryAgg` (creates and updates state; there is no separate SummaryCreation or SummaryUpdate operator, and `SummaryUpdate` is `SummaryAgg`'s update-expression parameter), `SummaryEstimate`, `FinalizeExactAccumulator`, `MaintainPopulation` and `EvaluatePopulation` (implemented); `SummaryMerge` (reserved here, enabled by #560); and `SummarySubtract`, `SummaryDelete`, `SummaryJoin` and `Extension` (reserved). See §5. -Each of the Summary operators also require the ASAP primitive information above to inter-operate correctly, preserving semantic correctness. - -Basically, the following information should be represented to preserve the equivalent query semantics when we introduce ASAP Primitives to logical query representation, and following physical one. - -- What type of the ASAP Primitive is -- What is the ASAP Primitive parameters -- What data sources a ASAP primitive summarizes -- What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents - - - -And these information will be combined with relational or time series query operator information, such as group by/reduction, filtering, projection, join, time series selection, together. - -Therefore, these requirements drive the following schema and metadata, node information, and column design. - - - -## 2. Existing database terminology for schema, table, column, and physical data layout - - -## 3. Proposed schema design -Schema represents the **metadata** of information flow along an **edge** between two nodes in a logical or physical DAG. The schema field is associated with the node in the DAG. The consumer of the node in the DAG takes the schema from the producer node as input. - -Schema definition here is shared between LogicalDAG, LogicalASAPDAG, and PhysicalASAPDAG. The schema contain fields, and each field is mapping to a column in the physical data representation. -Based on our requirement, each field should contain the following information. -1. **What type of the ASAP Primitive is** A state column can be a raw data type (e.g., numerical number, string). It can also be a summary type (§6.1, §6.2), e.g., the summary **family** is sketch (`FieldDataType::Sketch`), its **category** is quantile (`SketchCategory::Quantile`), its **algorithm** is KLL (`SketchAlgorithm::Kll`), and its **parameters** are `SketchParams::Kll { k: 200 }`. A sketch field also records its `GroupingStrategy` (one instance per group, or one shared Hydra structure). Non-sketch families (`ExactAggregate`, `Sample`, `Wavelet`, `StatModel`) have a kind and parameters but no category. -2. **What query intent the summarized ASAP Primitive can support, e.g., statistical aggregation intents, time window aggregation intents** This information is being mapped based on the primitive type: it is not stored in the field. `SummaryEstimate::validate_inputs` accepts a readout only when the `SketchStatistic` matches the state's `SketchCategory` (Quantile→`Quantile`; Cardinality→`Cardinality`/`Universal`; PointCount→`Frequency`/`Universal`; FrequencyL2/FrequencyEntropy→`Universal`; TopK→`TopK`/`Universal`). Exact accumulators are read by `FinalizeExactAccumulator` instead. `Sample`, `Wavelet` and `StatModel` state has no readout operator yet. - -Whether an edge carries **state or values** is recorded on the producing node as `OperatorResultKind`, not inferred from field types. Usually they agree: a `SummaryAgg` output has exactly one non-plain field and kind `State`. They can differ, though. `MaintainPopulation` keeps an all-plain schema but its kind is `State`, and a `Project` that passes a state column through keeps kind `State`. Consumers check `result_kind` (`validate_inputs` requires `State` for every readout). - -## 4. Proposed Node field design - -A node in the physical data will represent the data or summary instance, so a node has a field for **What data sources a ASAP primitive summarizes**. - -In code this field is `OperatorNode::coverage`. It records *which observations* a summary state covers: a source plus a union of joint (time × population) regions. It sits beside the schema, not inside a field, because two states with identical schemas can cover different data, and only disjoint coverage can be merged once-per-observation. - -Based on the above the proposed OperatorNode interface is as below (`crates/types/src/ir/node.rs`, `ir/summary_coverage.rs`): -```rust -pub enum OperatorResultKind { Relation, InstantVector, RangeVector, /** unfinalized summary/accumulator state */ State } - -pub enum Operator { NonASAP(NonASAPOp), ASAP(ASAPOp) } - -pub struct OperatorNode { - pub operator: Operator, - pub result_kind: OperatorResultKind, // derived from operator + children - pub schema: Schema, // derived; may override names/qualifiers only - pub guarantee: Option, // None until accuracy assessment; None != exact - pub timing: Option, // IngestionTime | QueryTime; None until assigned - #[serde(default)] - pub coverage: Option, // observations a State output summarizes -} - -impl OperatorNode { - pub fn new(op: Operator) -> Result; // derives schema + kind; coverage = None - pub fn with_schema(op: Operator, schema: Schema) -> Self; // caller-supplied names - pub fn new_shared(op: Operator) -> Result, SchemaDerivationError>; - pub fn with_guarantee(self, g: Option) -> Self; - pub fn with_timing(self, t: Option) -> Self; - /// Validates the coverage, then rejects a non-State node (CoverageError::NotState). - pub fn with_coverage(self, c: SummaryCoverage) -> Result; - /// This branch: true only for SummaryAgg. #560: SummaryAgg | SummaryMerge. - pub fn requires_coverage(&self) -> bool; - /// #560: (update expression, reduction) of a SummaryAgg, or shared by a SummaryMerge's inputs. - pub fn summary_update(&self) -> Option<(&SummaryUpdate, &Reduction)>; - /// Rebuilds with new inputs; re-derives schema; clears guarantee, timing and coverage. - pub fn map_children(&self, f: impl FnMut(&Rc) -> Rc) -> Result; - /// Per node: coverage well-formed (and present if required), validate_inputs, - /// result_kind and schema structure agree with derivation. - pub fn validate_structure(self: &Rc) -> Result<(), SchemaDerivationError>; - pub fn validate_execution_timing(self: &Rc) -> Result<(), SchemaDerivationError>; - // also: asap(), non_asap(), is_asap(), children(), contains_asap(), reachable() -} - -pub struct SummaryCoverage { - pub source: Source, // Source::Table { table_ref } | Source::TimeSeries { metric } - pub regions: Vec, // union of joint regions, never a Cartesian product -} -pub struct CoverageRegion { - pub time_ms: Option>, // half-open, on the source's time column; None = unrestricted - pub population: BTreeMap, // conjunction of equality predicates; empty = unrestricted -} -impl SummaryCoverage { - /// Intervals non-empty, dimension names non-empty, regions pairwise provably disjoint. - pub fn validate(&self) -> Result<(), CoverageError>; - /// Union of provably disjoint inputs from one source; coalesces adjacent - /// intervals with identical populations and keeps gaps. - pub fn merge_disjoint(inputs: &[Self]) -> Result; -} -pub enum CoverageError { - InvalidInterval, InvalidPopulation, SourceMismatch, PossibleOverlap, EmptyMerge, NotState, Missing, - UnknownInput, // #560: a merge input has no coverage - MergeOutputMismatch, // #560: retained merge coverage differs from the input union -} -``` - -Two regions are provably disjoint only if their time ranges do not intersect or they bind the same population dimension to different values. A region without time bounds overlaps any region it is not population-disjoint from. Coverage is caller-established: `validate` checks that it is well-formed, not that it matches the child's predicates. Any rewrite through `map_children` drops it, so the rewriter must declare it again. `#537` adds export and CSE of the logical DAG (`ir/export.rs`, `ir/cse.rs`); no interface in this document depends on it. - -## 5. Examples on how OperatorNode, schema, and physical data information are being used with Summary operators - -Given that these information requirements are introduced by summary operators to work correctly semantically, we show the examples of how the defined OperatorNode, schema, and physical data information work with each kind of summary operators. - -Notation: an edge is written `──Kind(field Type, …)──▶`. Schemas are the ones `output_schema()` derives. Planning may rename fields through `OperatorNode::with_schema`, but types, nullability, `time_index`, `unique_keys` and `closed` must match the derivation. All examples use a table source, so values are `Relation`; with a `TimeSeries` source the value side is `InstantVector`. - -### 5.1 `SummaryAgg`: values → state - -Scenario: p99 latency by job, from KLL(k=200), over one minute of table `t`. - -```text -Scan(t: job Utf8, latency Float64) - ──Relation(job Utf8, latency Float64)──▶ -SummaryAgg(family = Sketch(KLL{k=200}, PerSubpopulationInstance), - input = SummaryUpdate::column(Named("latency")), reduction = by[job], - grouping = PerSubpopulationInstance, filter = None) - ──State(job Utf8, state Sketch(KLL{k=200}, PerSubpopulationInstance))──▶ - coverage = { source: Table "t", regions: [{ time_ms: 0..60_000, population: {} }] } -``` - -- Output schema: the `by` keys followed by one non-nullable field `state` typed `family`; `unique_keys = [[0]]`, `closed = true`, no `time_index`. With `Reduction::PerEntity` the input columns are kept and the sample-value column is replaced by `state`. -- Checks: `family` is not `Plain`; the child is not `State`; the `weight`/`item` columns resolve against the child schema; `filter`, if present, types as `Bool`. -- Coverage: **required** and **declared**. `OperatorNode::new` leaves it `None`, `validate_structure` fails with `CoverageError::Missing`, and the planner attaches it with `with_coverage`. -- Boundary: this is where values become state. The sketch family, algorithm and parameters are committed in the field type, and `guarantee` stays `None` because state is not a caller-visible value. - -### 5.2 `SummaryEstimate`: sketch state → value - -Scenario: read p99 from the state in 5.1. - -```text -──State(job Utf8, state Sketch(KLL{k=200}))──▶ -SummaryEstimate(query = SketchStatistic::Quantile { q: 0.99 }) - ──Relation(job Utf8, quantile Float64)──▶ (planner may rename to p99) -``` - -- Output schema: the input schema with the one non-plain field replaced by a non-nullable plain field. Its name and type come from the statistic: `quantile`/`frequency_l2`/`frequency_entropy` Float64, `cardinality`/`count` Int64 (Float64 if the producer is a `PerEntity` `SummaryAgg`), and `topk` Utf8. Keys and metadata pass through. -- Result kind: the value kind of the source the state was built from (`Relation` here). -- Checks: input is `State` with exactly one non-plain field, that field is `Sketch`, and its category accepts the statistic (§3). For example, `Cardinality` on KLL is rejected. -- Coverage: **absent**. The output is a value, and `with_coverage` returns `NotState`. -- Boundary: state is consumed and a value is produced; `guarantee` on this node carries the readout's error bound. - -### 5.3 `FinalizeExactAccumulator`: exact state → value - -Scenario: total bytes by host with an exact Sum accumulator. - -```text -Scan(t: host Utf8, bytes Float64) - ──Relation(host Utf8, bytes Float64)──▶ -SummaryAgg(family = ExactAggregate(Sum, Sum), input = column(Named("bytes")), reduction = by[host]) - ──State(host Utf8, state ExactAggregate(Sum, Sum))──▶ coverage: required, declared -FinalizeExactAccumulator - ──Relation(host Utf8, state Float64)──▶ -``` - -- Output schema: each `ExactAggregate` field keeps its name (`state`) and takes the type and nullability the equivalent `NonASAPOp::Aggregate` would give: Sum/Min/Max follow the input column, Count is Int64, and Rate/IRate/Increase are Float64. If the child is not a `SummaryAgg` directly, Count falls back to Int64 and the others to Float64. `unique_keys`, `closed` and `time_index` are preserved (`schema_rebuilding.rs`). -- Checks: the input is `State` and contains an `ExactAggregate` field; a sketch is rejected (`structure_contract.rs`). -- Coverage: **absent** on the output. -- Boundary: this is the explicit maintenance-to-read boundary for exact state. Exact state is never read through `SummaryEstimate`. - -### 5.4 `MaintainPopulation`: values → maintained membership (state) - -Scenario: keep the full latency population per job, so that p99 and top-10 can be evaluated later. - -```text -Scan(t: job Utf8, latency Float64) [closed schema] - ──Relation(job Utf8, latency Float64)──▶ -MaintainPopulation(population = MaintainedPopulation { - input: PopulationInput::Rows { input: , value_column: 1, grouping: by[job] }, - max_k: 10, quantiles: true }) - ──State(job Utf8, latency Float64)──▶ -``` - -- Output schema: identical to the child's, all plain. Only `result_kind = State` marks it as maintained state. -- Checks: `population.matches_node(child)`. For `Rows`, the child must be the same closed table `Scan`, the value column must be non-null Float64, and grouping must be `by` with in-range keys. For `CurrentSeries`, it must be a `TimeSeries` scan with the same metric, matchers and grouping labels, under an instant `TimeRange` of `lookback_ms` (which may be omitted only for the default 300 s lookback). -- Coverage: **not required**. `with_coverage` accepts it because the output is `State`. -- Boundary: the output is state because it must also track membership changes; downstream operators can only read it through `EvaluatePopulation`. - -### 5.5 `EvaluatePopulation`: maintained membership → value - -Scenario: p99 by job from the population in 5.4. - -```text -──State(job Utf8, latency Float64) [from MaintainPopulation]──▶ -EvaluatePopulation(evaluation = PopulationStatistic::Quantile { q: 0.99 }) - ──Relation(job Utf8, quantile_0_99 Float64)──▶ -``` - -- Output schema: the schema of `Aggregate(by grouping, measure)` over the maintained source. Quantile gives `quantile_` Float64, Sum gives `sum` (value type), Count gives `count` Int64 and Average gives `avg` Float64; `unique_keys = [[0]]`, `closed`. `TopK { k }` instead returns the source schema unchanged (the selected rows). -- Checks: the child is a `MaintainPopulation` node whose `supports(evaluation)` holds: `quantiles` must be set for `Quantile`, and `k <= max_k` for `TopK`. -- Coverage: **absent**. -- Boundary: maintained membership is read as a value; the result kind is the source's (`Relation`). - -### 5.6 `SummaryMerge` (#560): state × N → state - -On this branch, `SummaryMerge { children }` is **reserved**. `is_unimplemented()` returns true, and `output_schema()` and `validate_inputs()` return `UNIMPLEMENTED_ASAP_OP`, so `OperatorNode::new` fails. Only `output_kind()` (= `State`), `children`, `map_children` and `kind_name` work. #560 enables it as follows (`summary_merge_structure.rs` in #560). - -Scenario: combine two one-minute KLL panes over `Scan(t: value Float64)` into a two-minute state. - -```text -SummaryAgg(KLL k=200, column(SampleValue), by[]) ──State(state Sketch(KLL{k=200}))── coverage {t, [0..60_000)} ─┐ -SummaryAgg(KLL k=200, column(SampleValue), by[]) ──State(state Sketch(KLL{k=200}))── coverage {t, [60_000..120_000)} ─┴▶ -SummaryMerge - ──State(state Sketch(KLL{k=200}))──▶ coverage = { t, [0..120_000) } (derived) -``` - -- Output schema: `children[0].schema`. -- Checks: at least one input; exactly one state column; every input is `State` with an identical schema (so family, params, grouping strategy and key positions match); every input has the same `summary_update()` (update expression and reduction); and `merged_coverage()` succeeds. Merging k=200 with k=300 fails, and so does merging raw rows. -- Coverage: **required** and **derived**. `OperatorNode::new` sets it to `SummaryCoverage::merge_disjoint` of the input coverages. An input without coverage gives `UnknownInput`, and overlapping inputs give `PossibleOverlap`. `validate_structure` rejects a retained coverage that differs from the derived one (`MergeOutputMismatch`). Gapped inputs stay as two regions. -- Boundary: state in, state out. No value is produced until a readout. - -### 5.7 Reserved operators (not implemented) - -These variants exist so that plans can name them, but `output_schema()`/`validate_inputs()` return `UNIMPLEMENTED_ASAP_OP`, so no node can be built. `output_kind()` already returns `State` for each of them. The intended edge shapes below follow from their fields; none of them is implemented. - -| Operator | Fields | Intended edge shape | -|---|---|---| -| `SummarySubtract` | `left, right` | State × State → State: remove one window's contribution, e.g. [0,10) − [0,5) | -| `SummaryDelete` | `summary_input, key: ColumnId` | State → State with the entries for `key` removed | -| `SummaryJoin` | `outer, inner, key, family` | State × State → State typed `family` (`produced_state()` returns it), e.g. join-size estimation | -| `Extension` | `child, name` | deployment-named state operator | - -### 5.8 Summary - -| Operator | Input kind | Output kind | Output carries state | Coverage on output | Status | -|---|---|---|---|---|---| -| `SummaryAgg` | value (not `State`) | `State` | yes (one `family` field) | required, declared | implemented | -| `SummaryEstimate` | `State` (one `Sketch` field) | source's value kind | no | absent | implemented | -| `FinalizeExactAccumulator` | `State` (`ExactAggregate`) | source's value kind | no | absent | implemented | -| `MaintainPopulation` | `Relation` (table) / `InstantVector` (series) | `State` | yes (by kind; fields plain) | optional, not required | implemented | -| `EvaluatePopulation` | `State` from `MaintainPopulation` | source's value kind | no | absent | implemented | -| `SummaryMerge` | `State` × N | `State` | yes | required, derived | reserved; enabled by #560 | -| `SummarySubtract` | `State` × 2 | `State` | yes | — | reserved | -| `SummaryDelete` | `State` | `State` | yes | — | reserved | -| `SummaryJoin` | `State` × 2 | `State` | yes | — | reserved | -| `Extension` | any | `State` | yes | — | reserved | - -## 6. Key code interfaces - -`OperatorNode`, `OperatorResultKind` and coverage are in §4. Bodies and serde/derive attributes are elided below. - -### 6.1 Schema and field types (`crates/types/src/pre_asap/schema.rs`) - -```rust -pub type ColumnId = usize; - -pub struct Schema { - pub fields: Vec, - pub time_index: Option, // must point at a plain Timestamp field - pub unique_keys: Vec>, - pub closed: bool, // true = fields enumerate every column -} -impl Schema { - pub fn new(fields: Vec) -> Self; - pub fn with_time_index(fields: Vec, time_index: ColumnId, unique_keys: Vec>) -> Self; - pub fn lifted(fields: Vec, time_index: Option) -> Self; // closed = true - pub fn is_all_plain(&self) -> bool; - pub fn column_id(&self, name: &str) -> Option; - pub fn column_id_qualified(&self, table: &str, name: &str) -> Option; -} - -pub struct Field { - pub name: String, - pub dtype: T, - pub nullable: bool, - pub table: Option, -} -impl Field { - pub fn plain(name: impl Into, dtype: DataType, nullable: bool) -> Self; - pub fn plain_dtype(&self) -> Option<&DataType>; - pub fn is_plain(&self) -> bool; -} - -/// A column's type: a plain value, or summary state of one family. -pub enum FieldDataType { - Plain(DataType), - ExactAggregate(ExactKind, ExactParams), - Sketch(SketchKind, GroupingStrategy), - Sample(SamplingKind, SamplingParams), - Wavelet(WaveletKind, WaveletParams), - StatModel(StatModelKind, StatModelParams), -} - -pub enum DataType { - Null, Int64, Float64, Utf8, Bool, Timestamp, Interval, Date, - List { element: Box> }, - Struct { fields: Vec> }, - Map { key: Box, value: Box, value_nullable: bool }, -} -``` - -### 6.2 State-family parameters (`crates/types/src/post_asap/sketch.rs`) - -```rust -pub enum ExactKind { Sum, Count, Min, Max, Increase, Rate, IRate } -pub enum ExactParams { Sum, Count, Min, Max, Increase, Rate, IRate } // no knobs; mirrors kind - -pub struct SketchKind { category: SketchCategory, algorithm: SketchAlgorithm, params: SketchParams } -impl SketchKind { - /// The only constructor; classifies the category and panics on mismatched params. - pub fn new(algorithm: SketchAlgorithm, params: SketchParams) -> Self; - pub fn category(&self) -> SketchCategory; - pub fn algorithm(&self) -> &SketchAlgorithm; - pub fn params(&self) -> &SketchParams; -} -pub enum SketchCategory { Universal, Quantile, Cardinality, Frequency, TopK } -// Universal: UnivMon | Quantile: Kll, DDSketch | Cardinality: Hll, Theta, Kmv -// Frequency: Cms, CountSketch | TopK: CmsWithHeap, CountSketchWithHeap -pub enum SketchAlgorithm { UnivMon, Kll, Cms, Hll, DDSketch, CmsWithHeap, Kmv, Theta, CountSketch, CountSketchWithHeap } -pub enum SketchParams { - UnivMon { heap_size: u32, sketch_rows: u32, sketch_cols: u32, layers: u8 }, - Kll { k: u32 }, - Cms { width: u32, depth: u32 }, - Hll { precision: u8 }, - DDSketch { alpha: f64 }, - CmsWithHeap { width: u32, depth: u32, heap_size: u32 }, - Kmv { k: u32 }, - Theta { k: u32 }, - CountSketch { width: u32, depth: u32 }, - CountSketchWithHeap { width: u32, depth: u32, heap_size: u32 }, -} - -/// How grouped state is instantiated across `by` subpopulations. Orthogonal to family. -pub enum GroupingStrategy { - PerSubpopulationInstance, // Default - SharedMultiSubpopulation { kind: HydraKind, params: HydraParams }, -} -pub enum HydraKind { HydraKll /* experimental, no error bound */, HydraCms, HydraCountSketch } -pub enum HydraParams { - HydraKll { k: u32, shared_buckets: u32 }, - HydraCms { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, - HydraCountSketch { width: u32, depth: u32, shared_rows: u32, shared_columns: u32 }, -} -pub fn hydra_kind_for(a: &SketchAlgorithm) -> Option; // Cms, CountSketch only - -pub enum SamplingKind { Reservoir } pub enum SamplingParams { Reservoir { size: u32 } } -pub enum WaveletKind { Haar } pub enum WaveletParams { Haar { coefficients: u32 } } -pub enum StatModelKind { Parametric } pub enum StatModelParams { Parametric { family: String } } -``` - -### 6.3 Update input and readouts (`post_asap/sketch.rs`, `post_asap/maintained_population.rs`) - -```rust -/// One state update: `item` keys the update for keyed families; `weight` is applied to state. -pub struct SummaryUpdate { - pub item: Option, - pub weight: SummaryInputExpr, - pub weight_domain: WeightDomain, // serde default: UnknownOrSigned -} -impl SummaryUpdate { pub fn column(c: ColumnRef) -> Self; } // item None, UnknownOrSigned -pub enum WeightDomain { - UnknownOrSigned, // Default; never assumed non-negative - NonNegative { proof: NonNegativeWeightProof }, -} -pub enum NonNegativeWeightProof { UnitCount, ResetAwareCounterDerivative } -pub enum SummaryInputExpr { - Constant(f64), Column(ColumnRef), Tuple(Vec), EntityIdentity(EntityIdentity), -} -pub enum EntityIdentity { PromqlLabelSet { excluding: Vec } } - -/// Readout of sketch state, carried by SummaryEstimate. -pub enum SketchStatistic { - FrequencyL2, FrequencyEntropy, - Quantile { q: f64 }, - PointCount { key: ColumnRef, value: Option }, - Cardinality, - TopK { k: usize }, -} - -/// Readout of a maintained population, carried by EvaluatePopulation. -pub enum PopulationStatistic { Quantile { q: f64 }, TopK { k: usize }, Sum, Count, Average } -pub struct MaintainedPopulation { - pub input: PopulationInput, - pub max_k: usize, // largest TopK it supports - pub quantiles: bool, // whether Quantile is supported -} -pub enum PopulationInput { - CurrentSeries(CurrentSeriesInput), // metric, matchers, grouping, without, lookback_ms - Rows { input: Rc, value_column: usize, grouping: GroupKeys }, -} -``` - -A finalized value's accuracy statement is `ResultGuarantee { metric, bound, failure_probability, provenance }` (`post_asap/guarantee.rs`). It is attached to readout and finalized nodes, never to raw state. - -### 6.4 ASAP operators (`crates/types/src/ir/asap.rs`) - -```rust -pub const UNIMPLEMENTED_ASAP_OP: &str = - "this ASAP operator is reserved: schema, accuracy, timing and export are not implemented"; - -pub enum ASAPOp { - SummaryAgg { - child: Rc, - family: FieldDataType, // never Plain - input: SummaryUpdate, - reduction: Reduction, // Reduce(GroupKeys) | PerEntity - grouping: GroupingStrategy, - filter: Option, // serde default None - }, - SummaryEstimate { summary_input: Rc, query: SketchStatistic }, - FinalizeExactAccumulator { child: Rc }, - MaintainPopulation { child: Rc, population: MaintainedPopulation }, - EvaluatePopulation { child: Rc, evaluation: PopulationStatistic }, - // Reserved on this branch; #560 implements SummaryMerge. - SummaryMerge { children: Vec> }, - SummarySubtract { left: Rc, right: Rc }, - SummaryDelete { summary_input: Rc, key: ColumnId }, - SummaryJoin { outer: Rc, inner: Rc, key: ColumnId, family: FieldDataType }, - Extension { child: Rc, name: String }, -} - -impl ASAPOp { - pub fn children(&self) -> Vec<&Rc>; // SummaryAgg includes its filter's subquery nodes - pub fn map_children(&self, f: impl FnMut(&Rc) -> Rc) -> Self; - pub fn kind_name(&self) -> &'static str; - /// Merge, Subtract, Delete, Join, Extension on this branch; #560 removes Merge. - pub fn is_unimplemented(&self) -> bool; - /// SummaryAgg/SummaryJoin `family`; #560 adds SummaryMerge (its inputs' state type). - pub fn produced_state(&self) -> Option<&FieldDataType>; - /// #560: merge_disjoint of the children's coverage; fails closed. - pub fn merged_coverage(&self) -> Result; - pub fn output_schema(&self) -> Result; - pub fn output_kind(&self) -> OperatorResultKind; - pub fn validate_inputs(&self) -> Result<(), SchemaDerivationError>; -} -``` diff --git a/docs/design_docs/proposals/decoupling_op_and_expr.md b/docs/design_docs/proposals/decoupling_op_and_expr.md index f850116bb..f87456a9f 100644 --- a/docs/design_docs/proposals/decoupling_op_and_expr.md +++ b/docs/design_docs/proposals/decoupling_op_and_expr.md @@ -62,7 +62,7 @@ operator inputs and scalar query-result references use `Rc`. read by expressions. Keep `ScalarExpr::Column(ColumnId)`: the ID selects a field for type checking and the corresponding input value for evaluation, independently of the executor's row/column storage layout. See the -[fields versus column references contract](asap-primitive-schema.md#21-consideration-1-the-schema-is-the-edge-between-two-nodes). +[fields versus column references contract](operator-sharing.md#21-one-schema-model-for-values-and-state). Names are resolved to `ColumnId` before constructing these nodes. Parsing and unresolved `ColumnRef` handling remain frontend concerns; no alternative generic diff --git a/docs/design_docs/proposals/operator-sharing.md b/docs/design_docs/proposals/operator-sharing.md index c9ab193a4..1851d1809 100644 --- a/docs/design_docs/proposals/operator-sharing.md +++ b/docs/design_docs/proposals/operator-sharing.md @@ -365,13 +365,155 @@ caching or mutation mechanism. ### 2.1 One schema model for values and state -Every operator output, before and after optimization, uses one `Schema` whose -`Field`s are typed by `FieldDataType`: `Plain(DataType)` for a readable value, or -the family, algorithm and parameters of summary or exact-accumulator state. -`OperatorResultKind` marks state outputs, and state becomes a value only through -an explicit readout. The schema model, `ColumnRef` versus `ColumnId`, the -validation entry points and the readout boundary are specified in -[Schema and physical data for ASAP primitives](asap-primitive-schema.md). +Use one `Schema` for operator outputs before and after optimization. Rename today's +`SummaryFamilyType` to `FieldDataType`: it types every field, and `Plain` is not a summary +family. Rename `Column` to `Field` and `Schema.columns` to `Schema.fields`: the struct +describes a column and holds none of its data. Retain the current `Schema` metadata. +The following is the proposed resolved interface; it is not the current Rust definition. + +```rust +struct Field { + name: String, + dtype: FieldDataType, + nullable: bool, + table: Option, +} + +struct Schema { + fields: Vec, + time_index: Option, + unique_keys: Vec>, + closed: bool, +} + +// Today's `SummaryFamilyType`, renamed; variants and payloads unchanged. +enum FieldDataType { + Plain(DataType), + ExactAggregate(ExactKind, ExactParams), + Sketch(SketchKind, GroupingStrategy), + Sample(SamplingKind, SamplingParams), + Wavelet(WaveletKind, WaveletParams), + StatModel(StatModelKind, StatModelParams), +} + +// Proposed derived output classification, separate from column types. +enum OperatorResultKind { + Relation, + InstantVector, + RangeVector, + State, +} + +impl Operator { + fn output_schema(&self) -> Result; + fn output_kind(&self) -> Result; + fn validate_inputs(&self) -> Result<(), QueryExprError>; +} + +impl OperatorNode { + fn validate_structure(&self) -> Result<(), QueryExprError>; + fn validate_execution_timing(&self) -> Result<(), QueryExprError>; +} + +impl ScalarExpr { + fn scalar_type(&self, input: &Schema) -> Result<(DataType, bool), QueryExprError>; +} +``` + +**Fields versus column references.** These names describe different roles, not +competing representations of the same object: + +| Name | Role | Holds runtime values? | +|---|---|---| +| `Schema` | Ordered `Field` metadata, plus key/time/closedness information | No | +| `Field` | Name, type, nullability and optional qualifier for one output column | No | +| `ColumnRef` | Unresolved logical reference: `Named`, `Qualified`, `SampleValue`, or `Wildcard` | No | +| `ColumnId = usize` | Resolved column position in a particular input/output schema | No | +| Runtime batch | Values conforming to a schema; storage layout is executor-specific | Yes | + +Keep `ColumnRef`, `ColumnId`, and `ScalarExpr::Column(ColumnId)`. Renaming the +metadata struct `Column` to `Field` does not rename column references to field +references. The same position identifies metadata during planning and values +during execution; it is not a stable field identity across projections or joins. +Schema `unique_keys` and `time_index` also use these column positions. + +For example, resolving `t.bytes` to position `1` produces `ColumnId = 1`. +`schema.fields[1]` supplies its type and nullability; evaluating +`ScalarExpr::Column(1)` reads the corresponding value. The native executor +currently reads `row[1]` from `Batch { schema, rows: Vec> }`. A columnar +executor would select array `1` instead. No physical `Column` container is +introduced by the metadata rename, and the old metadata `Column` struct is not +retained as a second type. + +**Relationship to current types.** `Field` is today's pre-ASAP `Column` with `dtype` +widened from `DataType` to `FieldDataType`. `FieldDataType` is today's `SummaryFamilyType` +under a name that also fits its `Plain` case. The proposed common `Schema` replaces +the separate operator-edge roles of pre-ASAP `Schema` and post-ASAP `SummarySchema` / +`SummaryField`; it does not rename `DataType`. A pre-ASAP value column becomes +`Plain(dtype)`. +Frontend validation permits only ordinary value columns, preserving the current +pre-ASAP restriction even though the common schema can also express state. + +| Field | Meaning and requirement | +|---|---| +| `fields` | Ordered named fields. `Plain(DataType)` is a readable value; other variants retain the identity and parameters of summary or exact-accumulator state. | +| `Field.nullable`, `Field.table` | Preserve SQL nullability and qualified column resolution. | +| `time_index` | Identifies the time column when present; it does not by itself distinguish an instant vector from a range vector. | +| `unique_keys` | Proven column combinations identifying rows; an empty list asserts no known key. Recompute these proofs when a rewrite changes identity. | +| `closed` | Whether `fields` completely describes the output. An open PromQL schema must retain unlisted labels through the existing complete-series-identity contract. | + +`OperatorResultKind` is derived from the operation and its inputs and retained as +`OperatorNode.result_kind`. `State` describes an output carrying unfinalized state; its +schema may also contain ordinary grouping keys. `SummaryEstimate`, +`FinalizeExactAccumulator` and other readouts derive the appropriate relation or +vector kind from their operation and input context. Matching numeric columns do +not make those kinds interchangeable. + +**Interface contracts.** `Operator::output_schema` and `output_kind` derive output +metadata from the payload and validated inputs. `validate_inputs` checks local +producer/consumer compatibility, such as vector inputs for `BinaryOp` or the +required state family for a summary readout. Scalar typing checks the input-kind +contract of `PromqlScalarFromVector` and other scalar plan reads. + +| Validation entry | Scope and stage | +|---|---| +| `OperatorNode::validate_structure()` | Walks the reachable operator DAG, including scalar plan references; checks input contracts, scalar typing and agreement between retained and derived output metadata. Valid for logical and physical plans; permits `timing = None`. | +| `OperatorNode::validate_execution_timing()` | Includes structural validation, then requires assigned timing on every executable operator and checks phase dependencies. Used for executable physical candidates. | +| Existing planner assessment and selection (#509) | Establishes guarantees using the existing accuracy models and checks them against request requirements and deployment capabilities. Neither node method re-proves a guarantee or decides request feasibility. | + +The two node methods need only the DAG and its annotations. Request requirements +and deployment models remain inputs to the existing planning/selection workflow, +not implicit globals of `validate_structure`. Passing the timing check alone does +not establish that a physical candidate satisfies the query's accuracy requirement. + +`Scan.schema` declares the source columns; `Values.schema` declares the constructed +row shape. `OperatorNode.schema` is the derived output for any operation. A scan's +predicates cannot change its declared output columns; a Values row must match the +declared arity, types and nullability. These leaf outputs retain the declaration's +column layout and time/identity information, with only justified metadata changes. +The declaration and derived output therefore have distinct roles, and structural +validation rejects disagreement rather than trusting two independent schemas. + +`scalar_type` keeps the existing method name and `(DataType, nullable)` result. +Its `input` is the applicable column scope: the child schema for a projection, +both input schemas for a join predicate, or aggregate outputs for `HAVING`. +Explicit subquery/conversion expressions validate their referenced producer using +the contracts above. Numeric expressions cannot consume state columns as numbers. +A standalone scalar expression is checked with an empty column scope and needs no fabricated +relation output schema. `QueryExprError` retains the existing error-type name; +result-kind, state-family, schema and execution-phase mismatches require +corresponding validation errors. + +For example, a KLL build outputs `State` with a +`Sketch(SketchKind, GroupingStrategy)` column identifying KLL and its parameters. +Its p99 readout outputs an ordinary `Plain(Float64)` column in the appropriate +relation/vector schema. A numeric predicate can use that readout, but not the KLL +state. Exact accumulator state similarly requires `FinalizeExactAccumulator`. +An ordinary operator may pass state through only where its input/output contract +permits it. A bare-column projection can preserve the field's `FieldDataType` +directly during `output_schema` derivation; `scalar_type` applies when that column +is used as a scalar value and rejects state. Copying a state column does not turn +it into a readable scalar. ### 2.2 Preserve existing accuracy semantics diff --git a/docs/design_docs/proposals/univmon-frequency-summary.md b/docs/design_docs/proposals/univmon-frequency-summary.md index a2d9f96c1..75cfd080b 100644 --- a/docs/design_docs/proposals/univmon-frequency-summary.md +++ b/docs/design_docs/proposals/univmon-frequency-summary.md @@ -35,8 +35,7 @@ cardinality alternatives, and exact count remains the cheaper first count candidate. All four readouts have the same unit-weight update, input sub-DAG, grouping, -window, parameter identity and state schema -([ASAP primitive schema](asap-primitive-schema.md)). Existing post-ASAP structural +window, parameter identity and state schema. Existing post-ASAP structural sharing can therefore intern their state producer while preserving distinct readout nodes. Sharing is only legal within the same execution/data scope. Precompute placement, SummaryCatalog installation, retention, and runtime diff --git a/docs/develop_docs/asap-aware-mapping-contracts.md b/docs/develop_docs/asap-aware-mapping-contracts.md index 011758143..447cb3823 100644 --- a/docs/develop_docs/asap-aware-mapping-contracts.md +++ b/docs/develop_docs/asap-aware-mapping-contracts.md @@ -325,12 +325,24 @@ backend inspection. Automatic selection skips those unproven ratios. Use ### Family, category, algorithm, and parameters -A summary's identity has four levels: family (`FieldDataType` variant), sketch -category (`SketchCategory`), algorithm (`SketchAlgorithm`), and the validated -committed choice (`SketchKind`). The levels and their validation are specified in -[Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#22-consideration-2-a-field-can-have-an-asap-primitive-type). +Sketches separate their query category from the concrete algorithm and its parameters: -Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `FieldDataType::Sketch(SketchKind, GroupingStrategy)`. +| Level | Type | Example | +| --- | --- | --- | +| **family** | `SummaryFamilyType` | `Sketch`, `Sample`, `Wavelet`, `StatModel`, `ExactAggregate` | +| **category** | `SketchCategory` | `Quantile`, `Cardinality`, `Frequency`, `TopK` | +| **algorithm** | `SketchAlgorithm` | `Kll` / `DDSketch` (both quantile); `Hll` (HyperLogLog) / `Theta` / `Kmv` (K-Minimum Values), all cardinality | +| **committed choice** | `SketchKind` | one validated category + algorithm + parameter combination | + +A `SketchKind` is a validated committed choice. Its public constructor, +`SketchKind::new(algorithm, params)`, verifies that the parameter variant belongs +to the selected algorithm and classifies the pair into its category. The public +`.category()`, `.algorithm()`, and `.params()` accessors expose the committed +values without permitting an invalid combination. + +Where this matters in practice: `CostModel::rank_candidates`, `CostModel::size_params`, and `SketchAlgorithmStrategy::replacements` operate at the **algorithm** level. `summary_candidates(intent)` returns a list of `SketchAlgorithm`s (`[Kll, DDSketch]` for a `Quantile` intent), never a bare `SketchKind` with nothing chosen underneath it. `SketchKind` appears after an algorithm has been selected and sized—on `Realization::Sketch(SketchKind)` and `SummaryFamilyType::Sketch(SketchKind)`. + +`Sample`, `Wavelet`, and `StatModel` each use a flat `(Kind, Params)` pair. `Sketch` needs the additional algorithm level because multiple algorithms can serve the same purpose—for example, KLL and DDSketch both answer quantile queries. --- @@ -366,7 +378,7 @@ The crate provides no default `Matcher` implementation because the answer depend Concretely, `explanation.rs` reports three candidate kinds from each `TargetSubDAGCandidates`: -- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `FieldDataType::Sketch(..)`, not just an exact/pass-through candidate. +- `ExplanationKind::SketchApproximation` — the set contains a `Replacement::Summary` that realizes `SummaryFamilyType::Sketch(..)`, not just an exact/pass-through candidate. - `ExplanationKind::CommonSubexpressionReuse` — `consumer_count >= 2` and the set contains `SharedSubDAGStrategy`'s "build once and share" candidate (the `Replacement::Rewrite` whose `Rc` is the set's `target`). - `ExplanationKind::ExactComposition` — the candidate set contains an exact operation diff --git a/docs/develop_docs/pre-asap-ir.md b/docs/develop_docs/pre-asap-ir.md index 2492e28d7..abf5dc50b 100644 --- a/docs/develop_docs/pre-asap-ir.md +++ b/docs/develop_docs/pre-asap-ir.md @@ -17,10 +17,26 @@ The pre-ASAP IR is defined using the `QueryExpr` enum. We discuss some of import ## Fields and column references -`Schema` holds `Field` metadata (name, type, nullability, qualifier) and no -values; an unresolved `ColumnRef` resolves to a positional `ColumnId` within one -schema. The design, including how the same position selects a runtime value, is -in [Schema and physical data for ASAP primitives](../design_docs/proposals/asap-primitive-schema.md#21-consideration-1-the-schema-is-the-edge-between-two-nodes). +`Schema` owns `Field` metadata: name, type, nullability, and an optional table +qualifier. A `Field` contains no runtime values. The former schema `Column` +struct served this same metadata role; it was renamed to `Field`, not retained +as a second data container. + +`ColumnRef` is an unresolved logical reference (`Named`, `Qualified`, +`SampleValue`, or `Wildcard`). Resolution binds a reference to `ColumnId`, a +`usize` position within a particular schema. `QueryExpr::Column(ColumnId)` +reads that column; the same position indexes `Schema::fields` for type checking +and a runtime row for its value. Group keys, unique keys, and `time_index` also +use these column positions. They are not stable identities across projections +or joins, so the positional reference remains `ColumnId`, not `FieldId`. + +The native runtime names shared ownership `SchemaRef = Arc` and stores +`Batch { schema: SchemaRef, rows: Vec> }`. `Schema` is the same metadata +model during planning and execution; the `Ref` suffix only distinguishes ownership. +It has no physical `Column`/array container. A column reference expresses what +to read independently of whether an executor stores its data as rows or arrays. +For example, resolving `t.bytes` to `ColumnId = 1` obtains its type from +`schema.fields[1]`; native execution reads `row[1]`. ## Node index From 056725a804cac0b4ad91318f530323c6c0160193 Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 6 Oct 2026 19:29:48 +0000 Subject: [PATCH 17/18] docs(ir): say coverage time is absolute and population is Utf8 equality Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/summary_coverage.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/crates/types/src/ir/summary_coverage.rs b/crates/types/src/ir/summary_coverage.rs index a95896eb8..3f3798a88 100644 --- a/crates/types/src/ir/summary_coverage.rs +++ b/crates/types/src/ir/summary_coverage.rs @@ -20,10 +20,15 @@ pub struct SummaryCoverage { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct CoverageRegion { - /// Half-open bounds on the source's time column, in milliseconds. `None` - /// means no time restriction, e.g. a source without a time column. + /// Half-open absolute bounds on the source's time column, in milliseconds. + /// Only a materialized state (for example an ingested pane) has absolute + /// bounds; a state in a logical plan that is not yet evaluated uses `None`, + /// as does a source without a time column. `None` means no time restriction. pub time_ms: Option>, - /// Conjunction of non-null equality predicates; empty means unrestricted. + /// Conjunction of `field = 'text'` equality predicates (PromQL label + /// matchers, SQL text columns), keyed by field name. Only Utf8 equality is + /// represented, so values are strings; any other restriction is not + /// recorded here. Empty means unrestricted. pub population: BTreeMap, } From 4750e711382fc097d353d761a3d17c34b15a8fdd Mon Sep 17 00:00:00 2001 From: zzylol <50204836+zzylol@users.noreply.github.com> Date: Tue, 6 Oct 2026 20:34:37 +0000 Subject: [PATCH 18/18] feat(ir): carry summary coverage through CSE and flat DAGs Coverage is part of a summary state's identity: two states over different observations are never shared, so CSE hashes and compares it, and a flat node keeps it. Co-Authored-By: Claude Opus 5.5 --- crates/types/src/ir/cse.rs | 3 +++ crates/types/src/ir/flat.rs | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/crates/types/src/ir/cse.rs b/crates/types/src/ir/cse.rs index 821da2395..b9046a1ab 100644 --- a/crates/types/src/ir/cse.rs +++ b/crates/types/src/ir/cse.rs @@ -112,6 +112,7 @@ pub fn structural_hash(node: &OperatorNode, cache: &mut HashCache) -> u64 { &node.schema, &node.guarantee, node.timing, + &node.coverage, ); serde_json::to_string(&own) .unwrap_or_default() @@ -167,6 +168,7 @@ fn same_node(left: &OperatorNode, right: &OperatorNode, memo: &mut EqMemo) -> bo && left.result_kind == right.result_kind && left.schema == right.schema && left.timing == right.timing + && left.coverage == right.coverage && same_value(&left.guarantee, &right.guarantee) && same_value(&own_fields(left), &own_fields(right)) } @@ -254,6 +256,7 @@ fn intern_bottom_up( schema: node.schema.clone(), guarantee: node.guarantee.clone(), timing: node.timing, + coverage: node.coverage.clone(), }; let interned = table.intern(rebuilt); visited.insert(Rc::as_ptr(node), (Rc::clone(node), Rc::clone(&interned))); diff --git a/crates/types/src/ir/flat.rs b/crates/types/src/ir/flat.rs index 434a3406f..830a294db 100644 --- a/crates/types/src/ir/flat.rs +++ b/crates/types/src/ir/flat.rs @@ -15,6 +15,7 @@ use serde::{Deserialize, Serialize}; use super::node::{Operator, OperatorNode, OperatorResultKind}; use super::query::QueryRoot; +use super::summary_coverage::SummaryCoverage; use crate::post_asap::execution_data_state::ExecutionTiming; use crate::post_asap::guarantee::ResultGuarantee; use crate::pre_asap::schema::Schema; @@ -30,6 +31,8 @@ pub struct FlatNode { pub schema: Schema, pub guarantee: Option, pub timing: Option, + #[serde(default)] + pub coverage: Option, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -77,6 +80,7 @@ fn visit( schema: node.schema.clone(), guarantee: node.guarantee.clone(), timing: node.timing, + coverage: node.coverage.clone(), }); originals.push(Rc::clone(node)); ids.insert(Rc::as_ptr(node), id);