diff --git a/crates/dunst-mcp/src/engine.rs b/crates/dunst-mcp/src/engine.rs index 7d1fd6b..760f726 100644 --- a/crates/dunst-mcp/src/engine.rs +++ b/crates/dunst-mcp/src/engine.rs @@ -17,7 +17,7 @@ use std::{ use dunst_core::{ ActionExecutor, ActionResult, AffordanceGraph, AuditEntry, Bbox, DunstError, GraphDiff, - Perceptor, RiskAssessment, RiskLevel, Role, SceneGraph, SceneNode, SemanticAction, + NodeChange, Perceptor, RiskAssessment, RiskLevel, Role, SceneGraph, SceneNode, SemanticAction, SessionIdentity, Target, WindowRef, }; use dunst_graph::{audit, derive_affordances, scene, RiskEngine}; @@ -29,6 +29,7 @@ mod app_ops; mod apps; mod browser_query; mod chart; +mod choices; mod element_actions; mod file_select; mod input; @@ -39,6 +40,7 @@ mod raw_input_gate; mod read; mod runtime_support; mod scene_query; +mod selections; mod types; mod window_geometry; mod window_ops; @@ -60,13 +62,15 @@ use input::{is_press_key_name, layout_sensitive_hotkey_message, parse_combo}; use query_support::*; use raw_input::page_scroll_target_id; use raw_input_gate::{ - is_synthetic_approval_target_id, raw_paste_text_target_id, raw_press_key_target_id, - raw_set_field_text_target_id, raw_type_keys_target_id, RawApprovalKey, + is_synthetic_approval_target_id, raw_apply_selections_target_id, raw_paste_text_target_id, + raw_press_key_target_id, raw_set_field_text_target_id, raw_type_keys_target_id, RawApprovalKey, }; use runtime_support::*; use scene_query::*; use window_geometry::*; +pub use choices::*; +pub use selections::*; pub use types::*; const READ_REFRESH_TTL: Duration = Duration::from_millis(500); @@ -95,6 +99,10 @@ pub struct Engine { /// rejects the action only because the operator is active, the grant is /// restored so the automatic retry path does not ask for approval again. raw_approval_inflight: BTreeMap, + /// Bounded synthetic approval context for a single `apply_selections` call + /// or internal survey-scroll sweep. Existing element/raw gates consult this + /// to avoid re-prompting for each constituent action. + active_batch: Option, /// IDs currently awaiting approval — the gated participants of the actions that /// returned `PendingApproval` since the last refresh. Lets [`approve`](Self::approve) /// accept an element whose danger is *contextual* (a destructive value typed into @@ -157,6 +165,7 @@ impl Engine { approvals: BTreeSet::new(), raw_approvals: BTreeMap::new(), raw_approval_inflight: BTreeMap::new(), + active_batch: None, pending_gate_ids: BTreeSet::new(), cached_window_rect: None, cached_menubar_root: None, @@ -241,6 +250,7 @@ impl Engine { self.approvals.clear(); self.raw_approvals.clear(); self.raw_approval_inflight.clear(); + self.active_batch = None; self.pending_gate_ids.clear(); self.target = Target { pid, window_id }; self.window = self.perceptor.window_ref(&self.target)?; diff --git a/crates/dunst-mcp/src/engine/action.rs b/crates/dunst-mcp/src/engine/action.rs index d6450d2..1258565 100644 --- a/crates/dunst-mcp/src/engine/action.rs +++ b/crates/dunst-mcp/src/engine/action.rs @@ -242,6 +242,9 @@ impl Engine { if !gate.effective.requires_approval || gate.approved { return None; } + if self.batch_context_allows_mutation() { + return None; + } for g in &gate.gated_ids { self.pending_gate_ids.insert(g.clone()); } diff --git a/crates/dunst-mcp/src/engine/choices.rs b/crates/dunst-mcp/src/engine/choices.rs new file mode 100644 index 0000000..1d40c3d --- /dev/null +++ b/crates/dunst-mcp/src/engine/choices.rs @@ -0,0 +1,618 @@ +use super::*; +use std::collections::{BTreeMap, BTreeSet}; + +#[derive(Clone, Copy, Debug)] +pub struct EnumerateOpts<'a> { + pub scope: &'a str, + pub include_latent: bool, + pub scroll_scan: bool, + pub max_scroll_pages: usize, + pub limit: usize, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ChoiceModel { + pub ui_epoch: String, + pub scope: String, + pub coverage: Coverage, + pub groups: Vec, + pub warnings: Vec, + pub scroll_plan: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ChoiceGroup { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + pub kind: GroupKind, + pub requirement: Requirement, + pub classification_confidence: f32, + pub choices: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct Choice { + pub id: String, + pub group_id: String, + pub label: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub value: Option, + pub state: SelectionState, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub safe_click: Option, + pub actuator: ActuatorHint, + pub risk: RiskAssessment, + pub source: String, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ScrollHint { + pub id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub direction: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub bbox: Option, + pub risk: RiskAssessment, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum GroupKind { + SingleSelect, + MultiSelect, + TextField, + Action, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SelectionState { + Selected, + Unselected, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Requirement { + Required, + Optional, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum Coverage { + Complete, + Partial, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ActuatorHint { + ClickElement, + PickOption, + ClickNearText, + SetFieldText, + Scroll, +} + +#[derive(Clone, Debug)] +struct GroupSeed { + id: String, + label: Option, + kind: GroupKind, + requirement: Requirement, + confidence: f32, +} + +impl Engine { + pub fn enumerate_choices( + &mut self, + opts: EnumerateOpts<'_>, + ) -> dunst_core::Result { + let opts = NormalizedEnumerateOpts::from(opts); + let (result, targets, warnings, scroll_scan_completed) = if opts.scroll_scan { + self.scroll_scan_choice_targets(&opts) + } else { + let result = self.hit_targets(opts.include_latent, opts.scope, opts.limit, None); + let targets = result.targets.clone(); + (result, targets, Vec::new(), false) + }; + Ok(self.choice_model_from_targets( + &result, + targets, + opts.scope, + opts.scroll_scan, + scroll_scan_completed, + warnings, + )) + } + + fn choice_model_from_targets( + &self, + result: &HitTargetsResult, + targets: Vec, + scope: &str, + scroll_scan: bool, + scroll_scan_completed: bool, + mut warnings: Vec, + ) -> ChoiceModel { + warnings.extend(result.supplemental_warnings.clone()); + let mut scroll_plan = Vec::new(); + let mut groups: BTreeMap = BTreeMap::new(); + let mut has_vision_only_choices = false; + + for target in targets { + if let Some(hint) = scroll_hint(&target) { + scroll_plan.push(hint); + continue; + } + let Some((kind, confidence)) = classify_choice_target(&target) else { + continue; + }; + if matches!(target.source.as_str(), "ocr" | "vision") { + has_vision_only_choices = true; + } + let seed = self.choice_group_seed(&target, kind, confidence); + let choice = Choice { + id: target.id.clone(), + group_id: seed.id.clone(), + label: choice_label(&target), + value: target.value.clone(), + state: selection_state(target.value.as_deref()), + bbox: target.bbox, + safe_click: target.safe_click.clone(), + actuator: actuator_hint(&target, kind), + risk: target.risk.clone(), + source: target.source.clone(), + }; + groups + .entry(seed.id.clone()) + .and_modify(|group| { + group.requirement = max_requirement(group.requirement, seed.requirement); + group.classification_confidence = + group.classification_confidence.max(seed.confidence); + if group.label.is_none() { + group.label = seed.label.clone(); + } + group.choices.push(choice.clone()); + }) + .or_insert_with(|| ChoiceGroup { + id: seed.id, + label: seed.label, + kind: seed.kind, + requirement: seed.requirement, + classification_confidence: seed.confidence, + choices: vec![choice], + }); + } + + let mut groups: Vec = groups.into_values().collect(); + for group in &mut groups { + group.choices.sort_by(choice_order); + enforce_single_select_cardinality(group, &mut warnings); + } + groups.sort_by(group_order); + + let coverage = if scroll_scan { + if scroll_scan_completed { + scroll_plan.clear(); + Coverage::Complete + } else if scroll_plan.is_empty() { + Coverage::Complete + } else { + Coverage::Partial + } + } else if has_vision_only_choices && !scroll_plan.is_empty() { + Coverage::Partial + } else { + scroll_plan.clear(); + Coverage::Complete + }; + + ChoiceModel { + ui_epoch: result.ui_epoch.fingerprint.clone(), + scope: scope.to_string(), + coverage, + groups, + warnings, + scroll_plan, + } + } + + fn choice_group_seed(&self, target: &HitTarget, kind: GroupKind, confidence: f32) -> GroupSeed { + if matches!(kind, GroupKind::TextField | GroupKind::Action) { + let label = target.label.clone(); + return GroupSeed { + id: format!("grp_{}", stable_group_token(&target.id)), + requirement: requirement_from_labels([label.as_deref(), target.value.as_deref()]), + label, + kind, + confidence, + }; + } + + let graph = self.scene_graph(); + let parent = graph + .get(&target.id) + .and_then(|node| node.parent.as_deref()) + .and_then(|parent| graph.get(parent)); + let parent_id = parent.map(|node| node.id.as_str()).unwrap_or("ungrouped"); + let parent_label = parent + .and_then(|node| { + if matches!(node.role, Role::Window | Role::Toolbar | Role::MenuBar) { + None + } else { + node.label.as_deref().or(node.value.as_deref()) + } + }) + .map(str::to_string); + let label = parent_label.clone(); + let requirement = requirement_from_labels([ + label.as_deref(), + target.label.as_deref(), + target.value.as_deref(), + ]); + GroupSeed { + id: format!( + "grp_{}_{}", + group_kind_token(kind), + stable_group_token(parent_id) + ), + label, + kind, + requirement, + confidence, + } + } + + fn scroll_scan_choice_targets( + &mut self, + opts: &NormalizedEnumerateOpts<'_>, + ) -> (HitTargetsResult, Vec, Vec, bool) { + let initial = self.hit_targets(opts.include_latent, opts.scope, opts.limit, None); + let targets = initial.targets.clone(); + let mut warnings = Vec::new(); + + #[cfg(test)] + { + warnings.push( + "scroll_scan requested under the mock test backend; returned the current choice surface" + .to_string(), + ); + (initial, targets, warnings, true) + } + + #[cfg(all(target_os = "macos", not(test)))] + { + let mut targets = targets; + let mut pages_scrolled = 0usize; + self.begin_internal_batch_context( + "batch@survey-scroll".to_string(), + opts.max_scroll_pages.saturating_mul(2).saturating_add(2), + initial.ui_epoch.fingerprint.clone(), + ); + for _ in 0..opts.max_scroll_pages { + match self.scroll("down", 1, None) { + Ok(entry) if entry.result == ActionResult::Success => { + pages_scrolled += 1; + let next = + self.hit_targets(opts.include_latent, opts.scope, opts.limit, None); + merge_hit_targets(&mut targets, next.targets); + } + Ok(entry) => { + warnings.push(format!( + "scroll_scan stopped after non-successful survey scroll: {:?}", + entry.result + )); + break; + } + Err(err) => { + warnings.push(format!("scroll_scan stopped: {err}")); + break; + } + } + } + for _ in 0..pages_scrolled { + if let Err(err) = self.scroll("up", 1, None) { + warnings.push(format!( + "scroll_scan could not restore the original scroll position exactly: {err}" + )); + break; + } + } + self.clear_internal_batch_context(); + (initial, targets, warnings, pages_scrolled > 0) + } + + #[cfg(all(not(target_os = "macos"), not(test)))] + { + warnings.push( + "scroll_scan requested, but survey scrolling requires the macOS backend; returned the current choice surface" + .to_string(), + ); + (initial, targets, warnings, true) + } + } + + #[cfg(test)] + pub(super) fn choice_model_from_targets_for_test( + &self, + result: &HitTargetsResult, + targets: Vec, + scope: &str, + ) -> ChoiceModel { + self.choice_model_from_targets(result, targets, scope, false, false, Vec::new()) + } +} + +#[derive(Clone, Copy)] +struct NormalizedEnumerateOpts<'a> { + scope: &'a str, + include_latent: bool, + scroll_scan: bool, + #[cfg_attr(test, allow(dead_code))] + max_scroll_pages: usize, + limit: usize, +} + +impl<'a> From> for NormalizedEnumerateOpts<'a> { + fn from(opts: EnumerateOpts<'a>) -> Self { + Self { + scope: match opts.scope { + "all" | "browser_chrome" => opts.scope, + _ => "page", + }, + include_latent: opts.include_latent, + scroll_scan: opts.scroll_scan, + max_scroll_pages: opts.max_scroll_pages.clamp(1, 12), + limit: opts.limit.clamp(1, 500), + } + } +} + +fn classify_choice_target(target: &HitTarget) -> Option<(GroupKind, f32)> { + let role = target.role; + if role == "radio" { + return Some((GroupKind::SingleSelect, 0.95)); + } + if matches!(role, "checkbox" | "switch") { + return Some((GroupKind::MultiSelect, 0.95)); + } + if matches!(role, "text_field" | "text_area" | "search_field") { + return Some((GroupKind::TextField, 0.95)); + } + if matches!(role, "popup_button" | "combobox" | "menu_button") { + return Some((GroupKind::SingleSelect, 0.85)); + } + if target + .action_modes + .iter() + .any(|mode| matches!(mode.action, SemanticAction::Pick | SemanticAction::OpenMenu)) + { + return Some((GroupKind::SingleSelect, 0.75)); + } + if target + .action_modes + .iter() + .any(|mode| mode.action == SemanticAction::Type) + { + return Some((GroupKind::TextField, 0.75)); + } + if role == "button" + && target + .action_modes + .iter() + .any(|mode| mode.action == SemanticAction::Click) + { + return Some((GroupKind::Action, 0.7)); + } + if target.source == "ocr" + && target + .action_modes + .iter() + .any(|mode| mode.action == SemanticAction::Click) + { + return Some((GroupKind::Action, 0.55)); + } + None +} + +fn actuator_hint(target: &HitTarget, kind: GroupKind) -> ActuatorHint { + if target.id.starts_with("page@scroll:") { + return ActuatorHint::Scroll; + } + if matches!(target.source.as_str(), "ocr" | "vision") { + return ActuatorHint::ClickNearText; + } + if kind == GroupKind::TextField { + return ActuatorHint::SetFieldText; + } + if target + .action_modes + .iter() + .any(|mode| matches!(mode.action, SemanticAction::Pick | SemanticAction::OpenMenu)) + { + return ActuatorHint::PickOption; + } + ActuatorHint::ClickElement +} + +fn scroll_hint(target: &HitTarget) -> Option { + let direction = target.id.strip_prefix("page@scroll:")?.to_string(); + Some(ScrollHint { + id: target.id.clone(), + direction: Some(direction), + bbox: target.bbox, + risk: target.risk.clone(), + }) +} + +fn choice_label(target: &HitTarget) -> String { + target + .label + .as_deref() + .or(target.value.as_deref()) + .filter(|label| !label.trim().is_empty()) + .unwrap_or(target.id.as_str()) + .trim() + .to_string() +} + +fn selection_state(value: Option<&str>) -> SelectionState { + let Some(value) = value.map(str::trim).filter(|value| !value.is_empty()) else { + return SelectionState::Unknown; + }; + match value.to_ascii_lowercase().as_str() { + "1" | "true" | "yes" | "on" | "selected" | "checked" | "selectionne" | "sélectionné" => { + SelectionState::Selected + } + "0" | "false" | "no" | "off" | "unselected" | "unchecked" | "deselected" + | "non selectionne" | "non sélectionné" => SelectionState::Unselected, + _ => SelectionState::Unknown, + } +} + +fn requirement_from_labels<'a>(labels: impl IntoIterator>) -> Requirement { + for label in labels.into_iter().flatten() { + let normalized = label.to_ascii_lowercase(); + if normalized.contains('*') + || normalized.contains("required") + || normalized.contains("obligatoire") + || normalized.contains("requis") + || normalized.contains("requise") + { + return Requirement::Required; + } + } + Requirement::Optional +} + +fn max_requirement(left: Requirement, right: Requirement) -> Requirement { + match (left, right) { + (Requirement::Required, _) | (_, Requirement::Required) => Requirement::Required, + (Requirement::Unknown, _) | (_, Requirement::Unknown) => Requirement::Unknown, + _ => Requirement::Optional, + } +} + +fn enforce_single_select_cardinality(group: &mut ChoiceGroup, warnings: &mut Vec) { + if group.kind != GroupKind::SingleSelect { + return; + } + let mut seen = BTreeSet::new(); + for choice in &mut group.choices { + if choice.state != SelectionState::Selected { + continue; + } + if seen.insert(group.id.clone()) { + continue; + } + warnings.push(format!( + "single-select group {} reported multiple selected choices; keeping later state unknown for {}", + group.id, choice.id + )); + choice.state = SelectionState::Unknown; + } +} + +#[cfg(all(target_os = "macos", not(test)))] +fn merge_hit_targets(targets: &mut Vec, incoming: Vec) { + for target in incoming { + if targets + .iter() + .any(|existing| same_hit_target(existing, &target)) + { + continue; + } + targets.push(target); + } +} + +#[cfg(all(target_os = "macos", not(test)))] +fn same_hit_target(left: &HitTarget, right: &HitTarget) -> bool { + if left.id == right.id { + return true; + } + match (left.bbox, right.bbox) { + (Some(a), Some(b)) => { + (a.x - b.x).abs() < 1.0 + && (a.y - b.y).abs() < 1.0 + && (a.w - b.w).abs() < 1.0 + && (a.h - b.h).abs() < 1.0 + && choice_label(left).eq_ignore_ascii_case(&choice_label(right)) + } + _ => false, + } +} + +fn group_order(left: &ChoiceGroup, right: &ChoiceGroup) -> std::cmp::Ordering { + group_min_y(left) + .partial_cmp(&group_min_y(right)) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + group_min_x(left) + .partial_cmp(&group_min_x(right)) + .unwrap_or(std::cmp::Ordering::Equal) + }) + .then_with(|| left.id.cmp(&right.id)) +} + +fn choice_order(left: &Choice, right: &Choice) -> std::cmp::Ordering { + let ly = left.bbox.map(|bbox| bbox.y).unwrap_or(f64::MAX); + let ry = right.bbox.map(|bbox| bbox.y).unwrap_or(f64::MAX); + ly.partial_cmp(&ry) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| { + let lx = left.bbox.map(|bbox| bbox.x).unwrap_or(f64::MAX); + let rx = right.bbox.map(|bbox| bbox.x).unwrap_or(f64::MAX); + lx.partial_cmp(&rx).unwrap_or(std::cmp::Ordering::Equal) + }) + .then_with(|| left.id.cmp(&right.id)) +} + +fn group_min_y(group: &ChoiceGroup) -> f64 { + group + .choices + .iter() + .filter_map(|choice| choice.bbox.map(|bbox| bbox.y)) + .fold(f64::MAX, f64::min) +} + +fn group_min_x(group: &ChoiceGroup) -> f64 { + group + .choices + .iter() + .filter_map(|choice| choice.bbox.map(|bbox| bbox.x)) + .fold(f64::MAX, f64::min) +} + +fn stable_group_token(value: &str) -> String { + value + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '_' + } + }) + .collect::() + .split('_') + .filter(|part| !part.is_empty()) + .collect::>() + .join("_") +} + +fn group_kind_token(kind: GroupKind) -> &'static str { + match kind { + GroupKind::SingleSelect => "single", + GroupKind::MultiSelect => "multi", + GroupKind::TextField => "text", + GroupKind::Action => "action", + } +} diff --git a/crates/dunst-mcp/src/engine/raw_input_gate.rs b/crates/dunst-mcp/src/engine/raw_input_gate.rs index bb5dc3a..a1f5430 100644 --- a/crates/dunst-mcp/src/engine/raw_input_gate.rs +++ b/crates/dunst-mcp/src/engine/raw_input_gate.rs @@ -229,6 +229,9 @@ impl Engine { reasoning: Option<&str>, risk: RiskAssessment, ) -> Option { + if self.batch_context_allows_mutation() { + return None; + } if self.consume_raw_approval(target_id) || self.approvals.contains(target_id) { return None; } @@ -335,6 +338,14 @@ impl Engine { if target_id.starts_with("file@") || target_id.starts_with("hover-reveal@") { return Ok(()); } + if target_id.starts_with("batch@selections:") { + if valid_batch_selection_target_id(target_id).is_some() { + return Ok(()); + } + return Err(DunstError::Execution(format!( + "{target_id} is not a valid batch selection approval target" + ))); + } Err(DunstError::Execution(format!( "{target_id} is not a recognised synthetic raw-input target" @@ -391,6 +402,15 @@ impl Engine { self.raw_approval_inflight.remove(target_id); } + pub(super) fn clear_raw_approval(&mut self, target_id: &str) { + self.raw_approval_inflight.remove(target_id); + for policy in raw_approval_policy(target_id) { + self.raw_approvals.remove(&policy.key); + } + self.pending_gate_ids.remove(target_id); + self.approvals.remove(target_id); + } + #[cfg(test)] pub(super) fn raw_approval_available_for_test(&mut self, target_id: &str) -> bool { let now = Instant::now(); @@ -489,6 +509,7 @@ pub(super) fn is_synthetic_approval_target_id(target_id: &str) -> bool { || target_id.starts_with("screen@") || target_id.starts_with("file@") || target_id.starts_with("hover-reveal@") + || target_id.starts_with("batch@selections:") } pub(super) fn raw_press_key_target_id(key: &str, repeat: usize) -> String { @@ -507,6 +528,13 @@ pub(super) fn raw_set_field_text_target_id(text: &str) -> String { raw_text_payload_target_id("set_field_text", text) } +pub(super) fn raw_apply_selections_target_id(hash: &str, count: usize) -> String { + format!( + "batch@selections:{hash}:{}", + count.clamp(1, MAX_BATCH_PICKS) + ) +} + fn raw_text_payload_target_id(action: &str, text: &str) -> String { let mut hash = 0xcbf2_9ce4_8422_2325_u64; for byte in text.as_bytes() { @@ -517,6 +545,15 @@ fn raw_text_payload_target_id(action: &str, text: &str) -> String { } fn raw_approval_policy(target_id: &str) -> Vec { + if let Some(count) = valid_batch_selection_target_id(target_id) { + return vec![RawApprovalPolicy { + key: RawApprovalKey { + scope: RawApprovalScope::Exact(target_id.to_string()), + }, + grant_events: count, + cost_events: 1, + }]; + } if let Some(rest) = target_id.strip_prefix("keyboard@press:") { let (key, cost_events) = parse_key_with_count(rest); return vec![RawApprovalPolicy { diff --git a/crates/dunst-mcp/src/engine/selections.rs b/crates/dunst-mcp/src/engine/selections.rs new file mode 100644 index 0000000..9857a07 --- /dev/null +++ b/crates/dunst-mcp/src/engine/selections.rs @@ -0,0 +1,709 @@ +use super::*; + +pub(super) const MAX_BATCH_PICKS: usize = 64; +const MAX_RESCANS: u32 = 8; + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct SelectionPlan { + pub steps: Vec, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct SelectionStep { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub group_id: Option, + pub choice_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label: Option, + pub op: SelectionOp, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected_after: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bbox: Option, +} + +#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)] +pub struct ExpectedState { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Deserialize, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum SelectionOp { + Select, + Deselect, + SetText, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct ApplyOutcome { + pub status: ApplyStatus, + pub batch_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub approval_hint: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub max_risk: Option, + #[serde(rename = "preview", skip_serializing_if = "Vec::is_empty")] + pub pending_preview: Vec, + #[serde(skip_serializing_if = "String::is_empty")] + pub ui_epoch: String, + pub rescans: u32, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub steps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub verify: Option, + #[serde(skip_serializing_if = "Vec::is_empty")] + pub remaining_steps: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ApplyStatus { + PendingApproval, + Applied, + PartiallyApplied, + Refused, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct BatchPreviewStep { + pub choice_id: String, + pub op: SelectionOp, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + pub risk: RiskLevel, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct StepResult { + pub choice_id: String, + pub op: SelectionOp, + pub result: StepResultStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub resolved_by: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub label: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum StepResultStatus { + Success, + Failed, + Skipped, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ResolvedBy { + Id, + LabelAfterRescan, + BboxAfterRescan, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct BatchVerify { + pub ok: bool, + pub checks: Vec, +} + +#[derive(Debug, Clone, serde::Serialize)] +pub struct VerifyCheck { + pub choice_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub actual: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub expected_value: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub actual_value: Option, + pub ok: bool, +} + +#[derive(Clone, Debug)] +pub(super) struct BatchApprovalContext { + pub(super) batch_id: String, + pub(super) remaining_budget: usize, + pub(super) expected_epoch: String, +} + +#[derive(Clone)] +struct ResolvedChoice { + choice: Choice, + resolved_by: ResolvedBy, +} + +impl Engine { + pub fn apply_selections( + &mut self, + plan: SelectionPlan, + expected_epoch: &str, + ) -> dunst_core::Result { + validate_plan(&plan, expected_epoch)?; + let current = self.current_ui_epoch_fingerprint(); + if current != expected_epoch { + return Err(DunstError::Execution( + "stale UI epoch: expected_epoch no longer matches the current target state; call enumerate_choices again before mutating" + .into(), + )); + } + + let batch_id = + raw_apply_selections_target_id(&selection_plan_hash(&plan), plan.steps.len()); + let initial_model = self.enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 500, + })?; + if plan.steps.iter().all(|step| { + resolve_step(step, &initial_model).is_none() + && step.label.as_deref().is_none_or(str::is_empty) + && step.bbox.is_none() + }) { + return Ok(ApplyOutcome { + status: ApplyStatus::Refused, + batch_id, + approval_hint: None, + max_risk: None, + pending_preview: Vec::new(), + ui_epoch: current, + rescans: 0, + steps: Vec::new(), + verify: None, + remaining_steps: plan.steps, + reason: Some("unresolvable_step".to_string()), + }); + } + let preview = self.preview_selection_plan(&plan, &initial_model); + let max_risk = preview + .iter() + .map(|step| step.risk) + .max() + .unwrap_or(RiskLevel::Low); + let batch_risk = RiskAssessment { + level: max_risk, + requires_approval: true, + reasons: vec![format!( + "batch choice selection requires one operator approval for {} step(s)", + plan.steps.len() + )], + }; + + if self + .gate_raw_input( + &batch_id, + SemanticAction::Click, + Some(format!("apply {} selection step(s)", plan.steps.len())), + Some("apply choice selections as one approved batch"), + batch_risk, + ) + .is_some() + { + return Ok(ApplyOutcome { + status: ApplyStatus::PendingApproval, + batch_id, + approval_hint: Some( + "operator must run approve(batch_id); approve authorizes the whole batch once" + .to_string(), + ), + max_risk: Some(max_risk), + pending_preview: preview, + ui_epoch: String::new(), + rescans: 0, + steps: Vec::new(), + verify: None, + remaining_steps: Vec::new(), + reason: None, + }); + } + + self.begin_internal_batch_context( + batch_id.clone(), + plan.steps.len().min(MAX_BATCH_PICKS), + expected_epoch.to_string(), + ); + let outcome = + self.execute_selection_batch(plan, batch_id.clone(), expected_epoch, initial_model); + self.clear_internal_batch_context(); + self.clear_raw_approval(&batch_id); + outcome + } + + pub(super) fn begin_internal_batch_context( + &mut self, + batch_id: String, + remaining_budget: usize, + expected_epoch: String, + ) { + self.active_batch = Some(BatchApprovalContext { + batch_id, + remaining_budget, + expected_epoch, + }); + } + + pub(super) fn clear_internal_batch_context(&mut self) { + self.active_batch = None; + } + + pub(super) fn batch_context_allows_mutation(&self) -> bool { + self.active_batch.as_ref().is_some_and(|ctx| { + ctx.remaining_budget > 0 && !ctx.batch_id.is_empty() && !ctx.expected_epoch.is_empty() + }) + } + + fn consume_batch_budget(&mut self) -> bool { + let Some(ctx) = &mut self.active_batch else { + return false; + }; + if ctx.remaining_budget == 0 { + return false; + } + ctx.remaining_budget -= 1; + true + } + + fn execute_selection_batch( + &mut self, + plan: SelectionPlan, + batch_id: String, + expected_epoch: &str, + mut model: ChoiceModel, + ) -> dunst_core::Result { + let mut pinned_epoch = expected_epoch.to_string(); + let mut rescans = 0u32; + let mut results = Vec::new(); + let mut remaining_steps = Vec::new(); + let mut stopped = false; + + for (idx, step) in plan.steps.iter().enumerate() { + if stopped { + remaining_steps.push(step.clone()); + continue; + } + let resolved = resolve_step(step, &model); + let Some(resolved) = resolved else { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Failed, + resolved_by: None, + label: step.label.clone(), + error: Some("unresolvable".to_string()), + }); + remaining_steps.push(step.clone()); + continue; + }; + + if step_already_satisfied(step, &resolved.choice) { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Success, + resolved_by: Some(resolved.resolved_by), + label: Some(resolved.choice.label), + error: None, + }); + continue; + } + + if !self.consume_batch_budget() { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Skipped, + resolved_by: Some(resolved.resolved_by), + label: Some(resolved.choice.label.clone()), + error: Some("batch budget exhausted".to_string()), + }); + remaining_steps.extend(plan.steps[idx..].iter().cloned()); + stopped = true; + continue; + } + + let label = resolved.choice.label.clone(); + let audit = self.execute_selection_step(step, &resolved.choice); + match audit { + Ok(Some(entry)) if entry.result == ActionResult::Success => { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Success, + resolved_by: Some(resolved.resolved_by), + label: Some(label), + error: None, + }); + if idx + 1 < plan.steps.len() { + let current = self.current_ui_epoch_fingerprint(); + if current != pinned_epoch { + if graph_diff_looks_like_reflow(&entry.graph_diff) { + if rescans >= MAX_RESCANS { + remaining_steps.extend(plan.steps[idx + 1..].iter().cloned()); + stopped = true; + } else { + rescans += 1; + model = self.enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 500, + })?; + pinned_epoch = model.ui_epoch.clone(); + } + } else { + pinned_epoch = current; + } + } + } + } + Ok(Some(entry)) => { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Failed, + resolved_by: Some(resolved.resolved_by), + label: Some(label), + error: Some(format!("actuator returned {:?}", entry.result)), + }); + remaining_steps.extend(plan.steps[idx + 1..].iter().cloned()); + stopped = true; + } + Ok(None) => { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Success, + resolved_by: Some(resolved.resolved_by), + label: Some(label), + error: None, + }); + } + Err(err) => { + results.push(StepResult { + choice_id: step.choice_id.clone(), + op: step.op, + result: StepResultStatus::Failed, + resolved_by: Some(resolved.resolved_by), + label: Some(label), + error: Some(err.to_string()), + }); + remaining_steps.extend(plan.steps[idx + 1..].iter().cloned()); + stopped = true; + } + } + } + + let verify = self.verify_selection_batch(&plan)?; + let success = !stopped + && remaining_steps.is_empty() + && results + .iter() + .all(|step| step.result == StepResultStatus::Success) + && verify.ok; + let status = if success { + ApplyStatus::Applied + } else { + ApplyStatus::PartiallyApplied + }; + Ok(ApplyOutcome { + status, + batch_id, + approval_hint: None, + max_risk: None, + pending_preview: Vec::new(), + ui_epoch: self.current_ui_epoch_fingerprint(), + rescans, + steps: results, + verify: Some(verify), + remaining_steps, + reason: None, + }) + } + + fn execute_selection_step( + &mut self, + step: &SelectionStep, + choice: &Choice, + ) -> dunst_core::Result> { + match step.op { + SelectionOp::Select | SelectionOp::Deselect => match choice.actuator { + ActuatorHint::PickOption => self + .pick_option(&choice.label, false, Some("batch choice pick")) + .map(|result| Some(result.audit)), + ActuatorHint::ClickNearText => self + .click_near_text( + &choice.label, + OcrClickOptions { + content_only: true, + accurate: true, + occurrence: 1, + expected_text: None, + reasoning: Some("batch OCR choice click"), + offset: (0.0, 0.0), + }, + ) + .map(|result| Some(result.audit)), + _ => self + .click_element(&choice.id, Some("batch choice click")) + .map(Some), + }, + SelectionOp::SetText => { + let value = step.value.as_deref().unwrap_or_default(); + if matches!(choice.source.as_str(), "accessibility" | "ax") { + self.type_into(&choice.id, value, Some("batch set choice text")) + .map(Some) + } else { + self.set_field_text(value).map(Some) + } + } + } + } + + fn preview_selection_plan( + &self, + plan: &SelectionPlan, + model: &ChoiceModel, + ) -> Vec { + plan.steps + .iter() + .map(|step| { + let resolved = resolve_step(step, model); + BatchPreviewStep { + choice_id: step.choice_id.clone(), + op: step.op, + label: resolved + .as_ref() + .map(|resolved| resolved.choice.label.clone()) + .or_else(|| step.label.clone()), + risk: resolved + .as_ref() + .map(|resolved| resolved.choice.risk.level) + .unwrap_or(RiskLevel::High), + } + }) + .collect() + } + + fn verify_selection_batch(&mut self, plan: &SelectionPlan) -> dunst_core::Result { + let model = self.enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 500, + })?; + let mut checks = Vec::new(); + for step in &plan.steps { + let resolved = resolve_step(step, &model); + let (actual_state, actual_value) = resolved + .as_ref() + .map(|resolved| (Some(resolved.choice.state), resolved.choice.value.clone())) + .unwrap_or((None, None)); + let expected_state = expected_state_for_step(step); + let expected_value = expected_value_for_step(step); + let state_ok = expected_state + .zip(actual_state) + .map(|(expected, actual)| expected == actual) + .unwrap_or(true); + let value_ok = expected_value + .as_ref() + .zip(actual_value.as_ref()) + .map(|(expected, actual)| expected == actual) + .unwrap_or_else(|| expected_value.is_none()); + checks.push(VerifyCheck { + choice_id: step.choice_id.clone(), + expected: expected_state, + actual: actual_state, + expected_value, + actual_value, + ok: resolved.is_some() && state_ok && value_ok, + }); + } + Ok(BatchVerify { + ok: checks.iter().all(|check| check.ok), + checks, + }) + } +} + +fn validate_plan(plan: &SelectionPlan, expected_epoch: &str) -> dunst_core::Result<()> { + if expected_epoch.trim().is_empty() { + return Err(DunstError::Execution( + "apply_selections requires non-empty expected_epoch".into(), + )); + } + if plan.steps.is_empty() { + return Err(DunstError::Execution( + "apply_selections requires at least one step".into(), + )); + } + if plan.steps.len() > MAX_BATCH_PICKS { + return Err(DunstError::Execution(format!( + "apply_selections accepts at most {MAX_BATCH_PICKS} steps" + ))); + } + for (idx, step) in plan.steps.iter().enumerate() { + if step.choice_id.trim().is_empty() { + return Err(DunstError::Execution(format!( + "plan step {idx} has empty choice_id" + ))); + } + if step.op == SelectionOp::SetText && step.value.is_none() { + return Err(DunstError::Execution(format!( + "plan step {idx} op=set_text requires value" + ))); + } + } + Ok(()) +} + +fn resolve_step(step: &SelectionStep, model: &ChoiceModel) -> Option { + if let Some(choice) = model + .groups + .iter() + .flat_map(|group| &group.choices) + .find(|choice| choice.id == step.choice_id) + { + return Some(ResolvedChoice { + choice: choice.clone(), + resolved_by: ResolvedBy::Id, + }); + } + + if let Some(label) = step.label.as_deref() { + let label_norm = normalize_match(label); + let mut label_matches: Vec<&Choice> = model + .groups + .iter() + .filter(|group| { + step.group_id + .as_deref() + .map(|group_id| group.id == group_id) + .unwrap_or(true) + }) + .flat_map(|group| &group.choices) + .filter(|choice| normalize_match(&choice.label) == label_norm) + .collect(); + if label_matches.is_empty() && step.group_id.is_some() { + label_matches = model + .groups + .iter() + .flat_map(|group| &group.choices) + .filter(|choice| normalize_match(&choice.label) == label_norm) + .collect(); + } + if label_matches.len() == 1 { + return Some(ResolvedChoice { + choice: label_matches[0].clone(), + resolved_by: ResolvedBy::LabelAfterRescan, + }); + } + } + + let bbox = step.bbox?; + nearest_bbox_choice(model, bbox).map(|choice| ResolvedChoice { + choice: choice.clone(), + resolved_by: ResolvedBy::BboxAfterRescan, + }) +} + +fn nearest_bbox_choice(model: &ChoiceModel, bbox: Bbox) -> Option<&Choice> { + let center = (bbox.x + bbox.w / 2.0, bbox.y + bbox.h / 2.0); + model + .groups + .iter() + .flat_map(|group| &group.choices) + .filter_map(|choice| { + let choice_bbox = choice.bbox?; + let choice_center = ( + choice_bbox.x + choice_bbox.w / 2.0, + choice_bbox.y + choice_bbox.h / 2.0, + ); + let dx = center.0 - choice_center.0; + let dy = center.1 - choice_center.1; + let distance = (dx * dx) + (dy * dy); + (distance <= 48.0 * 48.0).then_some((choice, distance)) + }) + .min_by(|(_, left), (_, right)| { + left.partial_cmp(right).unwrap_or(std::cmp::Ordering::Equal) + }) + .map(|(choice, _)| choice) +} + +fn step_already_satisfied(step: &SelectionStep, choice: &Choice) -> bool { + match step.op { + SelectionOp::Select => choice.state == SelectionState::Selected, + SelectionOp::Deselect => choice.state == SelectionState::Unselected, + SelectionOp::SetText => step + .value + .as_deref() + .zip(choice.value.as_deref()) + .is_some_and(|(expected, actual)| expected == actual), + } +} + +fn expected_state_for_step(step: &SelectionStep) -> Option { + step.expected_after + .as_ref() + .and_then(|expected| expected.state) + .or(match step.op { + SelectionOp::Select => Some(SelectionState::Selected), + SelectionOp::Deselect => Some(SelectionState::Unselected), + SelectionOp::SetText => None, + }) +} + +fn expected_value_for_step(step: &SelectionStep) -> Option { + step.expected_after + .as_ref() + .and_then(|expected| expected.value.clone()) + .or_else(|| { + if step.op == SelectionOp::SetText { + step.value.clone() + } else { + None + } + }) +} + +fn graph_diff_looks_like_reflow(diff: &GraphDiff) -> bool { + diff.changes.iter().any(|change| match change { + NodeChange::Added { .. } | NodeChange::Removed { .. } => true, + NodeChange::Changed { field, .. } => { + matches!(field.as_str(), "bbox" | "enabled" | "children" | "parent") + } + }) +} + +fn selection_plan_hash(plan: &SelectionPlan) -> String { + let mut hash = 0xcbf2_9ce4_8422_2325_u64; + let bytes = serde_json::to_vec(plan).unwrap_or_default(); + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x0000_0100_0000_01b3); + } + format!("{hash:016x}") +} + +pub(super) fn valid_batch_selection_target_id(target_id: &str) -> Option { + let rest = target_id.strip_prefix("batch@selections:")?; + let (hash, count) = rest.split_once(':')?; + if hash.len() != 16 || !hash.chars().all(|ch| ch.is_ascii_hexdigit()) { + return None; + } + let count = count.parse::().ok()?; + (1..=MAX_BATCH_PICKS).contains(&count).then_some(count) +} diff --git a/crates/dunst-mcp/src/engine/tests.rs b/crates/dunst-mcp/src/engine/tests.rs index 98b2d40..2b1403d 100644 --- a/crates/dunst-mcp/src/engine/tests.rs +++ b/crates/dunst-mcp/src/engine/tests.rs @@ -299,7 +299,9 @@ impl Perceptor for SequencePerceptor { } mod actions; +mod choices; mod drag_type; mod graph_views; mod page_text; mod raw_window; +mod selections; diff --git a/crates/dunst-mcp/src/engine/tests/choices.rs b/crates/dunst-mcp/src/engine/tests/choices.rs new file mode 100644 index 0000000..fa6174f --- /dev/null +++ b/crates/dunst-mcp/src/engine/tests/choices.rs @@ -0,0 +1,235 @@ +use super::*; + +fn choice_roots() -> Vec { + vec![raw_node( + "AXWindow", + Some("Checkout"), + None, + test_bbox(0.0, 0.0, 700.0, 500.0), + &[], + vec![ + raw_node( + "AXGroup", + Some("Delivery time *"), + None, + test_bbox(20.0, 40.0, 300.0, 120.0), + &[], + vec![ + raw_node( + "AXRadioButton", + Some("ASAP"), + Some("1"), + test_bbox(40.0, 70.0, 80.0, 24.0), + &["press"], + vec![], + ), + raw_node( + "AXRadioButton", + Some("Schedule"), + Some("0"), + test_bbox(40.0, 100.0, 120.0, 24.0), + &["press"], + vec![], + ), + ], + ), + raw_node( + "AXGroup", + Some("Extras"), + None, + test_bbox(360.0, 40.0, 260.0, 120.0), + &[], + vec![ + raw_node( + "AXCheckBox", + Some("Cutlery"), + Some("0"), + test_bbox(380.0, 70.0, 90.0, 24.0), + &["press"], + vec![], + ), + raw_node( + "AXCheckBox", + Some("Napkins"), + Some("1"), + test_bbox(380.0, 100.0, 90.0, 24.0), + &["press"], + vec![], + ), + ], + ), + raw_node( + "AXTextArea", + Some("Note to courier"), + Some(""), + test_bbox(40.0, 210.0, 300.0, 80.0), + &[], + vec![], + ), + ], + )] +} + +#[test] +fn enumerate_classifies_radios_as_single_select_and_checkboxes_as_multi() { + let (mut eng, _) = engine_from_roots(choice_roots(), "CheckoutApp", "Checkout"); + + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + + let single = model + .groups + .iter() + .find(|group| group.kind == GroupKind::SingleSelect) + .expect("radio group"); + assert_eq!(single.choices.len(), 2); + assert_eq!( + single + .choices + .iter() + .filter(|choice| choice.state == SelectionState::Selected) + .count(), + 1 + ); + + let multi = model + .groups + .iter() + .find(|group| group.kind == GroupKind::MultiSelect) + .expect("checkbox group"); + assert_eq!(multi.choices.len(), 2); + assert!(multi.choices.iter().any(|choice| choice.label == "Cutlery")); +} + +#[test] +fn enumerate_marks_required_group_from_label_markers() { + let (mut eng, _) = engine_from_roots(choice_roots(), "CheckoutApp", "Checkout"); + + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + + let delivery = model + .groups + .iter() + .find(|group| group.label.as_deref() == Some("Delivery time *")) + .expect("delivery group"); + assert_eq!(delivery.requirement, Requirement::Required); +} + +#[test] +fn enumerate_ax_latent_captures_offscreen_choices_without_scroll() { + let roots = vec![raw_node( + "AXWindow", + Some("Checkout"), + None, + test_bbox(0.0, 0.0, 700.0, 500.0), + &[], + vec![raw_node( + "AXGroup", + Some("Delivery time"), + None, + test_bbox(20.0, 40.0, 300.0, 120.0), + &[], + vec![raw_node( + "AXRadioButton", + Some("Tomorrow"), + Some("0"), + None, + &["press"], + vec![], + )], + )], + )]; + let (mut eng, _) = engine_from_roots(roots, "CheckoutApp", "Checkout"); + + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + + assert!(model + .groups + .iter() + .flat_map(|group| &group.choices) + .any(|choice| choice.label == "Tomorrow")); +} + +#[test] +fn enumerate_scroll_scan_restores_origin_and_sets_coverage_complete() { + let (mut eng, _) = engine_from_roots(choice_roots(), "CheckoutApp", "Checkout"); + let before = eng.current_ui_epoch_fingerprint(); + + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: true, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + + assert_eq!(model.coverage, Coverage::Complete); + assert_eq!(eng.current_ui_epoch_fingerprint(), before); +} + +#[test] +fn enumerate_partial_coverage_returns_scroll_plan() { + let (eng, _) = engine_from_roots(choice_roots(), "CheckoutApp", "Checkout"); + let mut result = eng.hit_targets(true, "page", 100, None); + let scroll = result + .targets + .iter() + .find(|target| target.id == "page@scroll:down") + .cloned() + .expect("scroll target"); + let ocr_choice = HitTarget { + id: "ocr_text_delivery_window".to_string(), + source: "ocr".to_string(), + role: "button", + label: Some("Delivery window".to_string()), + value: None, + bbox: Some(Bbox { + x: 80.0, + y: 320.0, + w: 120.0, + h: 24.0, + }), + safe_click: None, + confidence: 0.8, + action_modes: vec![HitActionMode { + action: SemanticAction::Click, + tool_hint: "click_near_text".to_string(), + target_id: Some("ocr_text_delivery_window".to_string()), + arguments: None, + drop_targets: Vec::new(), + risk: RiskAssessment::low(), + }], + risk: RiskAssessment::low(), + }; + result.targets = vec![scroll.clone(), ocr_choice.clone()]; + + let model = eng.choice_model_from_targets_for_test(&result, vec![scroll, ocr_choice], "page"); + + assert_eq!(model.coverage, Coverage::Partial); + assert_eq!(model.scroll_plan.len(), 1); +} diff --git a/crates/dunst-mcp/src/engine/tests/selections.rs b/crates/dunst-mcp/src/engine/tests/selections.rs new file mode 100644 index 0000000..6219661 --- /dev/null +++ b/crates/dunst-mcp/src/engine/tests/selections.rs @@ -0,0 +1,351 @@ +use super::*; + +fn checkbox_form(cutlery: &str, napkins: &str) -> Vec { + vec![raw_node( + "AXWindow", + Some("Checkout"), + None, + test_bbox(0.0, 0.0, 700.0, 500.0), + &[], + vec![raw_node( + "AXGroup", + Some("Extras"), + None, + test_bbox(20.0, 40.0, 300.0, 120.0), + &[], + vec![ + raw_node( + "AXCheckBox", + Some("Cutlery"), + Some(cutlery), + test_bbox(40.0, 70.0, 90.0, 24.0), + &["press"], + vec![], + ), + raw_node( + "AXCheckBox", + Some("Napkins"), + Some(napkins), + test_bbox(40.0, 100.0, 90.0, 24.0), + &["press"], + vec![], + ), + ], + )], + )] +} + +fn schedule_roots(time_visible: bool, time_selected: &str) -> Vec { + let schedule_value = if time_visible { "1" } else { "0" }; + let mut groups = vec![raw_node( + "AXGroup", + Some("Delivery time"), + None, + test_bbox(20.0, 40.0, 300.0, 100.0), + &[], + vec![raw_node( + "AXRadioButton", + Some("Schedule"), + Some(schedule_value), + test_bbox(40.0, 70.0, 120.0, 24.0), + &["press"], + vec![], + )], + )]; + if time_visible { + groups.push(raw_node( + "AXGroup", + Some("Time"), + None, + test_bbox(20.0, 150.0, 300.0, 100.0), + &[], + vec![raw_node( + "AXCheckBox", + Some("7 PM"), + Some(time_selected), + test_bbox(40.0, 180.0, 120.0, 24.0), + &["press"], + vec![], + )], + )); + } + vec![raw_node( + "AXWindow", + Some("Checkout"), + None, + test_bbox(0.0, 0.0, 700.0, 500.0), + &[], + groups, + )] +} + +fn plan_for(choice: &Choice, op: SelectionOp) -> SelectionPlan { + SelectionPlan { + steps: vec![step_for(choice, op)], + } +} + +fn step_for(choice: &Choice, op: SelectionOp) -> SelectionStep { + SelectionStep { + group_id: Some(choice.group_id.clone()), + choice_id: choice.id.clone(), + label: Some(choice.label.clone()), + op, + value: None, + expected_after: None, + bbox: choice.bbox, + } +} + +fn choice_by_label(model: &ChoiceModel, label: &str) -> Choice { + model + .groups + .iter() + .flat_map(|group| &group.choices) + .find(|choice| choice.label == label) + .unwrap_or_else(|| panic!("missing choice {label:?}")) + .clone() +} + +#[test] +fn apply_selections_first_call_is_pending_with_per_step_risk_preview() { + let (mut eng, _) = engine_from_roots(checkbox_form("0", "0"), "CheckoutApp", "Checkout"); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let cutlery = choice_by_label(&model, "Cutlery"); + + let outcome = eng + .apply_selections(plan_for(&cutlery, SelectionOp::Select), &model.ui_epoch) + .unwrap(); + + assert_eq!(outcome.status, ApplyStatus::PendingApproval); + assert!(outcome.batch_id.starts_with("batch@selections:")); + assert_eq!(outcome.pending_preview.len(), 1); + assert_eq!(outcome.pending_preview[0].risk, RiskLevel::Low); +} + +#[test] +fn apply_selections_single_approval_executes_whole_batch() { + let (mut eng, calls) = engine_from_sequence( + vec![checkbox_form("0", "0"), checkbox_form("1", "0")], + "CheckoutApp", + "Checkout", + ); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let cutlery = choice_by_label(&model, "Cutlery"); + let plan = plan_for(&cutlery, SelectionOp::Select); + let pending = eng.apply_selections(plan.clone(), &model.ui_epoch).unwrap(); + + eng.approve(&pending.batch_id).unwrap(); + let applied = eng.apply_selections(plan, &model.ui_epoch).unwrap(); + + assert_eq!(applied.status, ApplyStatus::Applied); + assert_eq!(calls.lock().unwrap().len(), 1); + assert_eq!(applied.steps[0].result, StepResultStatus::Success); + assert!(applied.verify.as_ref().unwrap().ok); +} + +#[test] +fn apply_selections_batch_grant_is_one_shot_resists_second_batch() { + let (mut eng, _) = engine_from_sequence( + vec![checkbox_form("0", "0"), checkbox_form("1", "0")], + "CheckoutApp", + "Checkout", + ); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let cutlery = choice_by_label(&model, "Cutlery"); + let plan = plan_for(&cutlery, SelectionOp::Select); + let pending = eng.apply_selections(plan.clone(), &model.ui_epoch).unwrap(); + eng.approve(&pending.batch_id).unwrap(); + assert_eq!( + eng.apply_selections(plan.clone(), &model.ui_epoch) + .unwrap() + .status, + ApplyStatus::Applied + ); + + let second = eng + .apply_selections(plan, &eng.current_ui_epoch_fingerprint()) + .unwrap(); + assert_eq!(second.status, ApplyStatus::PendingApproval); +} + +#[test] +fn apply_selections_rescans_only_when_fingerprint_changes() { + let (mut eng, _) = engine_from_sequence( + vec![ + checkbox_form("0", "0"), + checkbox_form("1", "0"), + checkbox_form("1", "1"), + ], + "CheckoutApp", + "Checkout", + ); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let cutlery = choice_by_label(&model, "Cutlery"); + let napkins = choice_by_label(&model, "Napkins"); + let plan = SelectionPlan { + steps: vec![ + step_for(&cutlery, SelectionOp::Select), + step_for(&napkins, SelectionOp::Select), + ], + }; + let pending = eng.apply_selections(plan.clone(), &model.ui_epoch).unwrap(); + eng.approve(&pending.batch_id).unwrap(); + + let applied = eng.apply_selections(plan, &model.ui_epoch).unwrap(); + + assert_eq!(applied.status, ApplyStatus::Applied); + assert_eq!(applied.rescans, 0, "value-only changes are not reflow"); +} + +#[test] +fn apply_selections_reflow_reresolves_remaining_steps_by_label() { + let (mut eng, _) = engine_from_sequence( + vec![ + schedule_roots(false, "0"), + schedule_roots(true, "0"), + schedule_roots(true, "1"), + ], + "CheckoutApp", + "Checkout", + ); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let schedule = choice_by_label(&model, "Schedule"); + let plan = SelectionPlan { + steps: vec![ + step_for(&schedule, SelectionOp::Select), + SelectionStep { + group_id: Some(schedule.group_id.clone()), + choice_id: "stale_time_choice".to_string(), + label: Some("7 PM".to_string()), + op: SelectionOp::Select, + value: None, + expected_after: None, + bbox: None, + }, + ], + }; + let pending = eng.apply_selections(plan.clone(), &model.ui_epoch).unwrap(); + eng.approve(&pending.batch_id).unwrap(); + + let applied = eng.apply_selections(plan, &model.ui_epoch).unwrap(); + + assert_eq!(applied.status, ApplyStatus::Applied); + assert_eq!(applied.rescans, 1); + assert_eq!( + applied.steps[1].resolved_by, + Some(ResolvedBy::LabelAfterRescan) + ); +} + +#[test] +fn apply_selections_budget_exhaustion_degrades_to_partial() { + let (mut eng, _) = engine_from_roots(checkbox_form("0", "0"), "CheckoutApp", "Checkout"); + let epoch = eng.current_ui_epoch_fingerprint(); + let plan = SelectionPlan { + steps: vec![SelectionStep { + group_id: None, + choice_id: "future_choice".to_string(), + label: Some("Future choice".to_string()), + op: SelectionOp::Select, + value: None, + expected_after: None, + bbox: None, + }], + }; + let pending = eng.apply_selections(plan.clone(), &epoch).unwrap(); + eng.approve(&pending.batch_id).unwrap(); + + let partial = eng.apply_selections(plan, &epoch).unwrap(); + + assert_eq!(partial.status, ApplyStatus::PartiallyApplied); + assert_eq!(partial.remaining_steps.len(), 1); +} + +#[test] +fn apply_selections_single_consolidated_verify_reports_per_check() { + let (mut eng, _) = engine_from_sequence( + vec![ + checkbox_form("0", "0"), + checkbox_form("1", "0"), + checkbox_form("1", "1"), + ], + "CheckoutApp", + "Checkout", + ); + let model = eng + .enumerate_choices(EnumerateOpts { + scope: "page", + include_latent: true, + scroll_scan: false, + max_scroll_pages: 1, + limit: 100, + }) + .unwrap(); + let cutlery = choice_by_label(&model, "Cutlery"); + let napkins = choice_by_label(&model, "Napkins"); + let plan = SelectionPlan { + steps: vec![ + step_for(&cutlery, SelectionOp::Select), + step_for(&napkins, SelectionOp::Select), + ], + }; + let pending = eng.apply_selections(plan.clone(), &model.ui_epoch).unwrap(); + eng.approve(&pending.batch_id).unwrap(); + + let applied = eng.apply_selections(plan, &model.ui_epoch).unwrap(); + + let verify = applied.verify.expect("verify result"); + assert_eq!(verify.checks.len(), 2); +} + +#[test] +fn apply_selections_rejects_forged_batch_id_via_validate_synthetic() { + let (mut eng, _) = engine_from_roots(checkbox_form("0", "0"), "CheckoutApp", "Checkout"); + + let err = eng + .approve("batch@selections:not-a-hex-value:2") + .unwrap_err(); + + assert!(err.to_string().contains("valid batch selection")); +} diff --git a/crates/dunst-mcp/src/serve.rs b/crates/dunst-mcp/src/serve.rs index dc8cceb..0e0b999 100644 --- a/crates/dunst-mcp/src/serve.rs +++ b/crates/dunst-mcp/src/serve.rs @@ -601,6 +601,7 @@ fn tool_accepts_mutation_preconditions(name: &str) -> bool { | "hover_probe" | "drag_element" | "select_file" + | "apply_selections" | "click_at" | "click_near_text" | "dismiss_modal" diff --git a/crates/dunst-mcp/src/serve/dispatch.rs b/crates/dunst-mcp/src/serve/dispatch.rs index 1b9288b..2c50a42 100644 --- a/crates/dunst-mcp/src/serve/dispatch.rs +++ b/crates/dunst-mcp/src/serve/dispatch.rs @@ -3,6 +3,7 @@ use super::*; use crate::serve::registry::{tool_route, ToolRoute}; mod args; +mod batch_tools; mod element_tools; mod raw_tools; mod read_tools; @@ -77,6 +78,7 @@ pub(super) fn handle_tool_call(engine: &mut Engine, id: Value, req: &Value) -> V let outcome = match route { ToolRoute::Read => read_tools::dispatch(engine, name, &args), ToolRoute::Element => element_tools::dispatch(engine, name, &args), + ToolRoute::Batch => batch_tools::dispatch(engine, name, &args), ToolRoute::Raw => raw_tools::dispatch(engine, name, &args), ToolRoute::WindowApp => window_app_tools::dispatch(engine, name, &args), ToolRoute::Screenshot => unreachable!("handled above"), @@ -178,9 +180,11 @@ fn tool_requires_mutation_coordination(route: ToolRoute, name: &str, args: &Valu ToolRoute::Read => match name { "read_at" | "read_series" => arg_bool(args, "borrow_cursor").unwrap_or(false), "scan_chart" => true, + "enumerate_choices" => arg_bool(args, "scroll_scan").unwrap_or(false), _ => false, }, ToolRoute::Element => !matches!(name, "approve" | "verify_state"), + ToolRoute::Batch => true, ToolRoute::Raw => !matches!(name, "hover_at" | "unstick_cursor"), ToolRoute::WindowApp => matches!( name, diff --git a/crates/dunst-mcp/src/serve/dispatch/batch_tools.rs b/crates/dunst-mcp/src/serve/dispatch/batch_tools.rs new file mode 100644 index 0000000..e7f07e5 --- /dev/null +++ b/crates/dunst-mcp/src/serve/dispatch/batch_tools.rs @@ -0,0 +1,32 @@ +use super::*; + +pub(super) fn dispatch( + engine: &mut Engine, + name: &str, + args: &Value, +) -> Option> { + Some(match name { + "apply_selections" => { + let expected_epoch = match arg(args, "expected_epoch") { + Some(epoch) => epoch, + None => return Some(Err("apply_selections requires 'expected_epoch'".into())), + }; + let plan = match args.get("plan") { + Some(plan) => { + match serde_json::from_value::(plan.clone()) { + Ok(plan) => plan, + Err(err) => { + return Some(Err(format!("apply_selections has invalid 'plan': {err}"))) + } + } + } + None => return Some(Err("apply_selections requires 'plan'".into())), + }; + engine + .apply_selections(plan, &expected_epoch) + .map(|outcome| serde_json::to_value(outcome).unwrap_or(Value::Null)) + .map_err(|e| e.to_string()) + } + _ => return None, + }) +} diff --git a/crates/dunst-mcp/src/serve/dispatch/read_tools.rs b/crates/dunst-mcp/src/serve/dispatch/read_tools.rs index 9394d56..4782022 100644 --- a/crates/dunst-mcp/src/serve/dispatch/read_tools.rs +++ b/crates/dunst-mcp/src/serve/dispatch/read_tools.rs @@ -126,6 +126,30 @@ fn dispatch_snapshot_tools( .unwrap_or(Value::Null)) } } + "enumerate_choices" => { + if let Err(err) = ensure_recent_graph( + engine, + arg_bool(args, "fresh").unwrap_or(true), + arg_bool(args, "force_refresh").unwrap_or(false), + ) { + Err(err) + } else { + let scope = arg(args, "scope").unwrap_or_else(|| "page".to_string()); + engine + .enumerate_choices(crate::engine::EnumerateOpts { + scope: scope.as_str(), + include_latent: arg_bool(args, "include_latent").unwrap_or(true), + scroll_scan: arg_bool(args, "scroll_scan").unwrap_or(false), + max_scroll_pages: args + .get("max_scroll_pages") + .and_then(Value::as_u64) + .unwrap_or(6) as usize, + limit: args.get("limit").and_then(Value::as_u64).unwrap_or(200) as usize, + }) + .map(|model| serde_json::to_value(model).unwrap_or(Value::Null)) + .map_err(|e| e.to_string()) + } + } "window_view" => { if let Err(err) = ensure_recent_graph( engine, diff --git a/crates/dunst-mcp/src/serve/registry.rs b/crates/dunst-mcp/src/serve/registry.rs index b018469..826567d 100644 --- a/crates/dunst-mcp/src/serve/registry.rs +++ b/crates/dunst-mcp/src/serve/registry.rs @@ -2,6 +2,7 @@ pub(super) enum ToolRoute { Read, Element, + Batch, Raw, WindowApp, Screenshot, @@ -39,6 +40,7 @@ pub(super) const TOOL_REGISTRY: &[RegisteredTool] = &[ tool("detect_modal", ToolRoute::Read), tool("extract_ocr_cards", ToolRoute::Read), tool("query_affordances", ToolRoute::Read), + tool("enumerate_choices", ToolRoute::Read), tool("read_at", ToolRoute::Read), tool("read_series", ToolRoute::Read), tool("scan_chart", ToolRoute::Read), @@ -53,6 +55,7 @@ pub(super) const TOOL_REGISTRY: &[RegisteredTool] = &[ tool("select_file", ToolRoute::Element), tool("approve", ToolRoute::Element), tool("verify_state", ToolRoute::Element), + tool("apply_selections", ToolRoute::Batch), tool("click_at", ToolRoute::Raw), tool("click_near_text", ToolRoute::Raw), tool("dismiss_modal", ToolRoute::Raw), diff --git a/crates/dunst-mcp/src/serve/tests/catalog.rs b/crates/dunst-mcp/src/serve/tests/catalog.rs index 7025fa5..9c0f1d6 100644 --- a/crates/dunst-mcp/src/serve/tests/catalog.rs +++ b/crates/dunst-mcp/src/serve/tests/catalog.rs @@ -5,7 +5,7 @@ use crate::serve::registry::TOOL_REGISTRY; fn tools_list_exposes_read_text_with_object_schema() { std::env::remove_var("DUNST_MCP_ENABLE_APPROVE_TOOL"); let tools = tools_list(); - assert_eq!(tools.len(), 70, "tool count"); + assert_eq!(tools.len(), 72, "tool count"); // Every tool must declare a JSON-Schema object input (the type:object fix). for t in &tools { assert_eq!( @@ -91,6 +91,32 @@ fn tools_list_exposes_read_text_with_object_schema() { tools.iter().any(|t| t["name"] == "get_hit_targets"), "semantic hit target tool present" ); + let enumerate_choices = tools + .iter() + .find(|t| t["name"] == "enumerate_choices") + .expect("choice enumeration tool present"); + assert_eq!( + enumerate_choices["inputSchema"]["properties"]["scroll_scan"]["type"], + "boolean" + ); + assert!( + enumerate_choices["inputSchema"]["properties"] + .get("expected_epoch") + .is_none(), + "enumerate_choices is read-only by default and must not advertise mutation preconditions" + ); + let apply_selections = tools + .iter() + .find(|t| t["name"] == "apply_selections") + .expect("batch selection tool present"); + assert_eq!( + apply_selections["inputSchema"]["required"], + json!(["expected_epoch", "plan"]) + ); + assert_eq!( + apply_selections["inputSchema"]["properties"]["fencing_token"]["type"], + "string" + ); assert!( tools.iter().any(|t| t["name"] == "visual_change_probe"), "visual change probe tool present" diff --git a/crates/dunst-mcp/src/serve/tests/dispatcher.rs b/crates/dunst-mcp/src/serve/tests/dispatcher.rs index f11e6d3..c913002 100644 --- a/crates/dunst-mcp/src/serve/tests/dispatcher.rs +++ b/crates/dunst-mcp/src/serve/tests/dispatcher.rs @@ -160,6 +160,36 @@ fn mutating_tool_rejects_stale_expected_epoch() { ); } +#[test] +fn stale_expected_epoch_refuses_apply_selections() { + set_test_coordination_dir(); + let mut e = engine_with_window(unique_window_id()); + e.set_session_identity(test_session("batch-epoch-a")); + + let resp = call( + &mut e, + "apply_selections", + json!({ + "expected_epoch": "stale-epoch", + "plan": { + "steps": [ + { "choice_id": "chk_cutlery", "op": "select", "label": "Cutlery" } + ] + } + }), + ); + + assert!( + is_error(&resp), + "stale epoch must refuse apply_selections: {resp}" + ); + assert!(text(&resp).contains("stale UI epoch")); + assert_eq!( + resp["result"]["_meta"]["dunst"]["coordination"]["epoch"]["status"], + "stale" + ); +} + #[test] fn mutating_tool_adds_window_lease_and_fencing_meta() { set_test_coordination_dir(); diff --git a/crates/dunst-mcp/src/serve/tools.rs b/crates/dunst-mcp/src/serve/tools.rs index e39bb1e..3227276 100644 --- a/crates/dunst-mcp/src/serve/tools.rs +++ b/crates/dunst-mcp/src/serve/tools.rs @@ -5,6 +5,7 @@ pub(super) fn tools_list() -> Vec { tools.extend(state_tools()); tools.extend(query_tools()); tools.extend(element_tools()); + tools.extend(batch_tools()); tools.extend(pointer_and_chart_tools()); tools.extend(window_app_tools()); tools.extend(keyboard_menu_tools()); @@ -306,6 +307,22 @@ fn query_tools() -> Vec { &["action"], ), ), + tool( + "enumerate_choices", + "Survey the whole choice surface once and return a structured option model: groups (single-select vs multi-select vs text field), required/optional, and per-choice {id, label, coords, current state}. Default mode captures off-screen AX choices in a single pass (no scroll). scroll_scan=true performs a position-restoring scroll sweep to also assemble OCR/vision choices on virtualized or AX-sparse surfaces; it surveys without operator approval and restores the original scroll position. Pass the returned ui_epoch to apply_selections as expected_epoch.", + schema( + json!({ + "scope": { "type": "string", "enum": ["page", "all", "browser_chrome"], "description": "target surface (default page)" }, + "include_latent": { "type": "boolean", "description": "include off-screen AX choices (default true)" }, + "scroll_scan": { "type": "boolean", "description": "position-restoring scroll sweep for OCR-only surfaces (default false)" }, + "max_scroll_pages": { "type": "integer", "description": "scroll-sweep bound, 1-12 (default 6)" }, + "limit": { "type": "integer", "description": "max choices, 1-500 (default 200)" }, + "fresh": { "type": "boolean", "description": "ensure a recent graph first (default true)" }, + "force_refresh": { "type": "boolean", "description": "force AX refresh even inside the short TTL (default false)" } + }), + &[], + ), + ), ] } @@ -413,6 +430,47 @@ fn element_tools() -> Vec { ] } +fn batch_tools() -> Vec { + vec![tool( + "apply_selections", + "Apply a whole choice plan as ONE batch behind a single operator approval. Build the plan from enumerate_choices and pass that ui_epoch as expected_epoch. The batch refuses a stale plan up front, re-scans ONLY when the UI epoch fingerprint changes mid-batch (progressive disclosure / reflow) and re-resolves the remaining steps, then runs a single consolidated verify. First call returns status=pending_approval with a per-step preview incl. risk; an operator approves the returned batch_id once, then re-call with the same plan to execute.", + schema( + json!({ + "expected_epoch": { "type": "string", "description": "ui_epoch.fingerprint the plan was built from (required)" }, + "plan": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "group_id": { "type": "string" }, + "choice_id": { "type": "string", "description": "Choice.id from enumerate_choices" }, + "label": { "type": "string", "description": "re-resolution fallback after reflow" }, + "op": { "type": "string", "enum": ["select", "deselect", "set_text"] }, + "value": { "type": "string", "description": "text for set_text" }, + "expected_after": { + "type": "object", + "properties": { + "state": { "type": "string", "enum": ["selected", "unselected"] }, + "value": { "type": "string" } + } + } + }, + "required": ["choice_id", "op"] + } + } + }, + "required": ["steps"] + }, + "include_diff": { "type": "boolean" } + }), + &["expected_epoch", "plan"], + ), + )] +} + fn pointer_and_chart_tools() -> Vec { vec![ tool( diff --git a/docs/AGENT_GUIDE.md b/docs/AGENT_GUIDE.md index dc8630f..06de04b 100644 --- a/docs/AGENT_GUIDE.md +++ b/docs/AGENT_GUIDE.md @@ -62,6 +62,25 @@ Use the safest available interaction layer first: Always re-read the field or page with OCR/AX before saving or submitting after a raw mutation. +## Batch A Multi-Field Choice Page + +Use `enumerate_choices` before filling choice-heavy forms, modals, or checkout +pages: + +1. Call `enumerate_choices` with default `include_latent=true`. Use + `scroll_scan=true` only for virtualized or AX-sparse surfaces that need an OCR + survey; it is mutation-coordinated, not approval-gated. +2. Build one `apply_selections.plan.steps[]` from returned `Choice.id` values. + Include `label` for reflow fallback; use `op: "select"`, `"deselect"`, or + `"set_text"`. +3. Pass `enumerate_choices.ui_epoch` as `expected_epoch`. +4. The first `apply_selections` call returns `status: "pending_approval"` and a + single `batch_id`; surface the preview to the operator and approve that id + once. +5. Re-call `apply_selections` with the same plan. Inspect `steps`, `rescans`, + and the consolidated `verify` block. If the result is `partially_applied`, + re-run `enumerate_choices` and apply only the remaining choices. + ## Firefox And Sparse AX Firefox can expose only window chrome while the page itself is readable by OCR. diff --git a/docs/CONTRACTS.md b/docs/CONTRACTS.md index d2ed73b..a59d132 100644 --- a/docs/CONTRACTS.md +++ b/docs/CONTRACTS.md @@ -45,6 +45,26 @@ the same change. Crates: `dunst-core`, `-graph`, `-mcp`, `-vision`. `engine::tests::raw_key_approval_allows_short_repeated_same_key_burst`, `engine::tests::raw_scroll_approval_covers_same_direction_count_change`, `engine::tests::attach_clears_raw_approval_grants`. +- **Batch selections are approved as one unit.** `apply_selections` records + exactly one `PendingApproval` for a `batch@selections::` target whose + preview carries per-step risk and an aggregate `max_risk`; a single operator + `approve(batch_id)` authorizes the whole batch, the grant is one-shot, and the + `BatchApprovalContext` is always cleared on exit so no later single action is + silently authorized. + — `engine::tests::apply_selections_first_call_is_pending_with_per_step_risk_preview`, + `engine::tests::apply_selections_batch_grant_is_one_shot_resists_second_batch`. +- **Batch execution is epoch-guarded.** `apply_selections` refuses a plan whose + `expected_epoch` no longer matches before mutation and re-scans only when a + mid-batch structural reflow changes the UI fingerprint, re-resolving remaining + steps by id then label/bbox; execution is bounded by `MAX_RESCANS` and the step + budget. + — `engine::tests::apply_selections_rescans_only_when_fingerprint_changes`, + `serve::tests::stale_expected_epoch_refuses_apply_selections`. +- **Enumeration is read-only or survey-only.** `enumerate_choices` mutates no + application data: default mode does not scroll; `scroll_scan` is + mutation-coordinated but not operator-approval-gated and restores the original + scroll position on live backends. + — `engine::tests::enumerate_scroll_scan_restores_origin_and_sets_coverage_complete`. - **Approval transport boundary.** `approve` is an operator-side interlock, not a default agent affordance. The MCP server does not advertise or execute the `approve` tool unless `DUNST_MCP_ENABLE_APPROVE_TOOL=1` is set for a controlled