diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 87814d4..98070ea 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -12,7 +12,7 @@ "name": "computer-use-linux", "source": "./plugins/computer-use-linux", "description": "Native Linux desktop control: semantic AT-SPI accessibility trees, screenshots, compositor window targeting (GNOME, KDE, Hyprland, niri, COSMIC, i3, X11), Wayland/X11 input, and an on-screen activity indicator.", - "version": "0.7.22", + "version": "0.7.23", "category": "productivity" } ] diff --git a/CHANGELOG.md b/CHANGELOG.md index 7677f43..b6e0048 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.7.23] - 2026-10-24 + ### Fixed - Report failed desktop actions and intentional refusals as MCP tool errors with `isError: true`, while preserving successful results when only post-action observation fails. Thanks @OnlyHa1bara. (#280) +- Read stable machine-readable AT-SPI action names so Qt buttons use native click/press/toggle activation instead of pointer fallback. Thanks @sinantux for the detailed reports. (#281) +- Map KWin element clicks using coherent frame, client, output and scale geometry. Reject ambiguous window ownership, unknown desktop visibility, mixed output scales and stale pointer dimensions before sending input. (#281) +- Keep KWin Save/Open dialogs, confirmation prompts and verified interactive transients targetable, including dialogs that skip the taskbar. (#286) +- Refresh stale semantic AT-SPI targets once before action dispatch, preserving the original app scope and never replaying numeric-index or already-dispatched actions. (#282) +- Scope `get_app_state` by short or reverse-DNS `app_id` values, reject ambiguous aliases, and avoid whole-desktop fallback for unresolved explicit app IDs. (#283) +- Verify editable-text `set_value` results with exact full-content readback, report mismatches or verification failures instead of false success, and mark password fields unverifiable. (#284) +- Reject unsupported X11 PointerRoot keyboard monitoring before typing, preserve one xdotool dispatch with an explicit monitoring warning, and never replay the text. (#272) ### Security -- Update Tokio to 1.53.3 and register the GNOME indicator socket with the I/O-safe `AsyncFd::register` API, addressing RUSTSEC-2026-0386. (#280) +- Update Tokio to 1.53.3 and register the GNOME indicator socket with the I/O-safe `AsyncFd::register` API, addressing RUSTSEC-2026-0386. Thanks @OnlyHa1bara. (#280) +- Update Pi and schema-check dependencies, including patched Undici, Vitest, Zod, body-parser, fast-uri and ip-address. Regenerate the bundled MCP client. (#285) ## [0.7.22] - 2026-10-20 @@ -857,7 +867,8 @@ pages; also bumps the MCP server's advertised version string to match. - Validated against GNOME 50.1 on Wayland (Ubuntu 25.10). - KDE / Sway / Hyprland untested — see README support matrix. -[Unreleased]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.22...HEAD +[Unreleased]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.23...HEAD +[0.7.23]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.22...v0.7.23 [0.7.22]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.21...v0.7.22 [0.7.21]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.20...v0.7.21 [0.7.20]: https://github.com/agent-sh/computer-use-linux/compare/v0.7.19...v0.7.20 diff --git a/Cargo.lock b/Cargo.lock index 75f4807..075d735 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -378,7 +378,7 @@ dependencies = [ [[package]] name = "computer-use-linux" -version = "0.7.22" +version = "0.7.23" dependencies = [ "anyhow", "atspi", diff --git a/Cargo.toml b/Cargo.toml index d3b5078..0911162 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "computer-use-linux" -version = "0.7.22" +version = "0.7.23" edition = "2021" authors = ["Avi Fenesh "] license = "MIT" diff --git a/README.md b/README.md index e0b1e60..c23dbdd 100644 --- a/README.md +++ b/README.md @@ -106,16 +106,22 @@ The preference does not substitute an arbitrary action name for a coordinate click. Explicit `x`/`y`, right clicks, and double/multiple clicks retain pointer semantics. Re-check application state after either kind of activation. +On KDE Wayland, element clicks that need pointer dispatch match the cached AT-SPI frame to a coherent KWin window/output snapshot, then map the target separately to physical capture pixels (uinput) and logical desktop coordinates (portal). Client-local and decorated frames, actual decoration offsets, and nonzero or negative output origins are supported when the geometry and uniform output scale can be verified. Ambiguous windows, mixed or unknown scales, overlapping outputs, or stale pointer geometry fail before input; explicit coordinates keep their existing behavior. + For coordinate `click` or `scroll` with `relative: true`, select a target window and measure from its clipped screenshot crop origin. Divide preview `x` and `y` by the returned screenshot `scale` before passing them. These are not raw GDK surface or widget-local coordinates; decorations and clipping can change the origin. A missing window target is rejected. -**X11 text fidelity:** `xdotool`/XTEST can silently substitute symbols under non-US XKB layouts (reported with Brazilian `br`/ABNT2) or changing keymap state, even when `type_text` returns `ok: true`. Every successful xdotool text dispatch includes a layout warning. XKB layout replacement events, persistent mapping/group changes, and keyboard focus/grab transitions detected across a call return an error after dispatch, without replaying the text. Keyboard grabs can divert characters without changing the polled focused-window ID. If monitoring cannot start, xdotool still dispatches and the response explicitly warns that layout-change and keyboard interception detection were unavailable. This also detects a `setxkbmap` layout changed and restored during one call. Temporary raw remaps or group switches restored within a call and dropped text with a stable map remain unverified. Re-observe the actual text before pressing Enter, submitting it, or treating it as delivered. Use `include_tree: true` for AT-SPI readback when the application exposes the text, or `screenshot_after: true` to inspect it. These observations do not automatically compare the field to the requested text. Do not retry blindly. +**X11 text fidelity:** `xdotool`/XTEST can silently substitute symbols under non-US XKB layouts (reported with Brazilian `br`/ABNT2) or changing keymap state, even when `type_text` returns `ok: true`. Every successful xdotool text dispatch includes a layout warning. XKB layout replacement events, persistent mapping/group changes, and keyboard focus/grab transitions detected across a call return an error after dispatch, without replaying the text. Keyboard grabs can divert characters without changing the polled focused-window ID. If monitoring cannot start, xdotool still dispatches and the response explicitly warns that layout-change and keyboard interception detection were unavailable. PointerRoot focus is unsupported because pointer motion can redirect input without a detectable focus event. This also detects a `setxkbmap` layout changed and restored during one call. Temporary raw remaps or group switches restored within a call and dropped text with a stable map remain unverified. Re-observe the actual text before pressing Enter, submitting it, or treating it as delivered. Use `include_tree: true` for AT-SPI readback when the application exposes the text, or `screenshot_after: true` to inspect it. These observations do not automatically compare the field to the requested text. Do not retry blindly. Targeted `press_key`/`type_text` results append focused-element feedback from AT-SPI (role, name, editable) and warn when a concrete non-terminal focus is not editable. Terminals accept keyboard input without an EditableText interface; re-observe the contents to confirm input. Click/screenshot/input results warn when the target window or coordinate is partially or fully off-screen. `get_app_state` returns a compact readiness block by default; pass `verbose: true` for the full diagnostics report. It also reports `tree_scoped` (false when no app target narrowed the AT-SPI tree, with a warning in `message`) and `accessibility_tree_truncated` (true when the node, depth, or read budget stopped traversal with unread elements left). +An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. + +`set_value` on editable text reads the full contents back after dispatch. `verified: true` means an exact match; mismatches and readback failures return `ok: false` without replaying the mutation. Password fields return `verified: false` because masked content cannot prove the write. Re-observe after any mismatch; the contents may have changed. + On X11, text input sends LF and CRLF line endings as Return, with one newline per CRLF pair. On GNOME, a read-only keymap preflight rejects the whole string before input if a character is absent or requires a modifier level Mutter cannot synthesize. The background reader cannot establish the active layout group, so every configured group must support the text. This can reject text that works in one of several groups; use `set_value` on an editable field instead. The check requires `libxkbcommon.so.0` at runtime and stops after a two-second keymap read deadline. It does not cover held modifiers or a keymap changed concurrently with typing. diff --git a/package.json b/package.json index be88e0a..df5be79 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@agent-sh/computer-use-linux", - "version": "0.7.22", + "version": "0.7.23", "description": "Native Linux desktop control for AI agents (MCP server, Pi tools, and CLI): semantic AT-SPI accessibility trees, screenshots, compositor window targeting, Wayland/X11 input, and an on-screen activity indicator.", "license": "MIT", "type": "commonjs", diff --git a/pi/extension/generated-tools.ts b/pi/extension/generated-tools.ts index d875ed7..af21f0c 100644 --- a/pi/extension/generated-tools.ts +++ b/pi/extension/generated-tools.ts @@ -7,9 +7,9 @@ export interface GeneratedMcpToolDefinition { annotations: Record; } -export const GENERATED_SERVER_VERSION = "0.7.22"; -export const GENERATED_TOOL_CATALOG_HASH = "b9a6ef2736cd165b97982ec6ccf1e6cfb47712f0d4a8ae778ecac47e211b15ec"; -export const GENERATED_SHELL_TOOL_CATALOG_HASH = "113fc01364844db492475ec7ad00f1d03fe8e2d7acedc8a0aa3b48d245f99105"; +export const GENERATED_SERVER_VERSION = "0.7.23"; +export const GENERATED_TOOL_CATALOG_HASH = "3e2d6cbbf31957fcdc8d05e66f959ad1c3b277685dfa30ed3c3b2f4df451eec6"; +export const GENERATED_SHELL_TOOL_CATALOG_HASH = "052c430ada325bd74fd764fc3f9d1139f4e6e401b6a5dbc6ef8a3de538214241"; export const GENERATED_MCP_TOOLS = [ { @@ -1794,7 +1794,7 @@ export const GENERATED_MCP_TOOLS = "openWorldHint": true, "readOnlyHint": false }, - "description": "Set the value of a settable accessibility element selected by index, identifier, or semantic selector. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.", + "description": "Set the value of a settable accessibility element selected by index, identifier, or semantic selector. Editable-text writes are verified by exact readback; password fields are explicitly unverifiable. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.", "inputSchema": { "$defs": { "ScreenshotOutputFormat": { diff --git a/plugins/computer-use-linux/.claude-plugin/plugin.json b/plugins/computer-use-linux/.claude-plugin/plugin.json index d98820c..c195a5e 100644 --- a/plugins/computer-use-linux/.claude-plugin/plugin.json +++ b/plugins/computer-use-linux/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "computer-use-linux", - "version": "0.7.22", + "version": "0.7.23", "description": "Native Linux desktop control: semantic AT-SPI accessibility trees, screenshots, compositor window targeting (GNOME, KDE, Hyprland, niri, COSMIC, i3, X11), Wayland/X11 input, and an on-screen activity indicator.", "author": { "name": "agent-sh", diff --git a/plugins/computer-use-linux/.codex-plugin/plugin.json b/plugins/computer-use-linux/.codex-plugin/plugin.json index 59581af..fa338fc 100644 --- a/plugins/computer-use-linux/.codex-plugin/plugin.json +++ b/plugins/computer-use-linux/.codex-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "computer-use-linux", - "version": "0.7.22", + "version": "0.7.23", "description": "Native Linux desktop control: semantic AT-SPI accessibility trees, screenshots, compositor window targeting (GNOME, KDE, Hyprland, niri, COSMIC, i3, X11), Wayland/X11 input, and an on-screen activity indicator.", "author": { "name": "agent-sh", diff --git a/plugins/computer-use-linux/bin/computer-use-linux b/plugins/computer-use-linux/bin/computer-use-linux index 928b8e0..52ac6f2 100755 --- a/plugins/computer-use-linux/bin/computer-use-linux +++ b/plugins/computer-use-linux/bin/computer-use-linux @@ -5,7 +5,7 @@ # message goes to stderr. set -eu -version=0.7.22 +version=0.7.23 say() { echo "computer-use-linux plugin: $*" >&2 diff --git a/plugins/computer-use-linux/skills/computer-use-linux/SKILL.md b/plugins/computer-use-linux/skills/computer-use-linux/SKILL.md index 6c94019..5a33aa4 100644 --- a/plugins/computer-use-linux/skills/computer-use-linux/SKILL.md +++ b/plugins/computer-use-linux/skills/computer-use-linux/SKILL.md @@ -134,7 +134,7 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set ## Procedure 1. In Pi, call `computer_use_linux_tools` with the exact tools or capability you need. Enabled tools use the `computer_use_linux_` prefix, appear starting on the next model turn, and remain active for the session. -2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 2000 and 64) rather than lowering them. The compact readiness block identifies missing setup. +2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 2000 and 64) rather than lowering them. The compact readiness block identifies missing setup. 3. Use `doctor` only when you need the full diagnostic report. 4. If `can_build_accessibility_tree` is false, run `setup_accessibility` and restart the target app. 5. If `can_query_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload. @@ -143,6 +143,7 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set 8. Use coordinates only when the UI surface has no useful accessibility tree. 9. For text input, prefer `type_text` with a target selector (`window_id`, `pid`, `app_id`, `wm_class`, `title`, `tty`, `terminal_pid`, `terminal_command`, or `terminal_cwd`) rather than relying on current focus. 10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback. +11. For `set_value`, `verified: true` is the only exact-content confirmation. `verified: false` requires fresh observation; password fields are deliberately unverifiable. The field is omitted for numeric values and early failures. On Hyprland, `type_text` passes one literal wtype argument after `--` for a classified nonsecret field, avoiding stdin keymap-refresh corruption in Electron. diff --git a/scripts/x11_type_guard_test.py b/scripts/x11_type_guard_test.py index 398a344..ac86b24 100644 --- a/scripts/x11_type_guard_test.py +++ b/scripts/x11_type_guard_test.py @@ -2,7 +2,8 @@ """Opt-in Xvfb/MCP regression for issue #272, isolated from the real desktop. Requires Xvfb, xterm, xdotool, xkbcomp, setxkbmap and dbus-run-session. Checks byte-exact -stable input, restored layout changes, keyboard grabs and restored focus changes. +stable input, restored layout changes, keyboard grabs, restored focus changes, and +PointerRoot/no-fixed-focus dispatch behavior. """ import json import os @@ -100,6 +101,12 @@ def bind(name, result, args): def focus(): value=W(); revert=I();get_focus(d,ctypes.byref(value),ctypes.byref(revert));return value.value before=focus() +if sys.argv[1] in ("pointer-root", "focus-none"): + focus_id = 1 if sys.argv[1] == "pointer-root" else 0 + set_focus(d,focus_id,1,0);sync(d,0) + assert focus() == focus_id + close(d) + sys.exit(0) w=create(d,root(d),0,0,10,10,0,0,0); map_window(d,w);sync(d,0) if sys.argv[1] == "grab": assert grab(d,w,0,1,1,0) == 0 else: set_focus(d,w,1,0);sync(d,0) @@ -130,6 +137,8 @@ def focus(): ("layout-restored-3", "TESTE:;'\"[]{}<>~^\\|!@#%&*()_+=?/abcXYZ0198" * 3, "layout"), ("keyboard-grab-stable-layout-and-focus", "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" * 2, "grab"), ("focus-restored", "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" * 2, "focus"), + ("focus-none", "No fixed focus still dispatches\n", "focus-none"), + ("pointer-root", "Pointer root still dispatches\n", "pointer-root"), ]): capture = Path(scratch) / f"{index}.txt" marker = Path(scratch) / f"{index}.dispatch" @@ -157,8 +166,12 @@ def focus(): "protocolVersion": "2025-11-25", "capabilities": {}, "clientInfo": {"name": "x11-type-guard-test", "version": "1"}}}) rpc(mcp, {"jsonrpc": "2.0", "method": "notifications/initialized"}) + focus_mode = disturbance in ("pointer-root", "focus-none") + if focus_mode: + subprocess.run([sys.executable, str(interrupt), disturbance], + env=env, check=True, timeout=5) before_map = subprocess.check_output(["xkbcomp", "-xkb", env["DISPLAY"], "-"], env=env, stderr=subprocess.DEVNULL, timeout=5) - if disturbance: + if disturbance and not focus_mode: def replace_layout(): try: if not select.select([notify], [], [], 15)[0]: @@ -188,7 +201,15 @@ def replace_layout(): output = result.get("structuredContent") or json.loads( next(item["text"] for item in result["content"] if item["type"] == "text")) assert marker.read_text().splitlines() == ["type"], "text was replayed" - if disturbance: + if focus_mode: + assert output["ok"], output + assert not result.get("isError", False), result + assert "Action sent through xdotool (X11 XTEST)." in output["message"], output + if disturbance == "pointer-root": + assert "X11 keyboard monitoring does not support PointerRoot" in output["message"], output + else: + assert "layout-change detection was unavailable" not in output["message"], output + elif disturbance: assert not output["ok"], output if disturbance == "layout": assert "layout changed during type_text" in json.dumps(output), output diff --git a/skills/computer-use-linux/SKILL.md b/skills/computer-use-linux/SKILL.md index 6c94019..5a33aa4 100644 --- a/skills/computer-use-linux/SKILL.md +++ b/skills/computer-use-linux/SKILL.md @@ -134,7 +134,7 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set ## Procedure 1. In Pi, call `computer_use_linux_tools` with the exact tools or capability you need. Enabled tools use the `computer_use_linux_` prefix, appear starting on the next model turn, and remain active for the session. -2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 2000 and 64) rather than lowering them. The compact readiness block identifies missing setup. +2. Begin every desktop-control turn with `get_app_state`, scoped to the app you are working in: pass `app_name_or_bundle_identifier` or a window target (`window_id`, `pid`, `app_id`, `wm_class`, `title`). An `app_id` may be a full desktop ID or its final dot-separated segment; matching is case-insensitive, ambiguous aliases are rejected, and an unresolved explicit `app_id` returns an empty scoped tree rather than the desktop. Without a target the result is the whole desktop AT-SPI tree, `tree_scoped` is `false`, and `message` warns; that can flood context. Use `include_screenshot: false` when the accessibility tree is sufficient. If `accessibility_tree_truncated` is `true`, the tree is incomplete: scope to a narrower target and raise `max_nodes` or `max_depth` (hard caps 2000 and 64) rather than lowering them. The compact readiness block identifies missing setup. 3. Use `doctor` only when you need the full diagnostic report. 4. If `can_build_accessibility_tree` is false, run `setup_accessibility` and restart the target app. 5. If `can_query_windows` is false on GNOME Wayland, run `setup_window_targeting` and ask the user to log out and back in if setup says the shell extension needs a reload. @@ -143,6 +143,7 @@ Pi native tools skip this MCP `command` config; see [Pi setup](references/pi-set 8. Use coordinates only when the UI surface has no useful accessibility tree. 9. For text input, prefer `type_text` with a target selector (`window_id`, `pid`, `app_id`, `wm_class`, `title`, `tty`, `terminal_pid`, `terminal_command`, or `terminal_cwd`) rather than relying on current focus. 10. After mutating actions (`click`, `drag`, `scroll`, `press_key`, `type_text`, `perform_action`, `set_value`, `activate_window`, `move_window`, `resize_window`), either pass `screenshot_after: true | "screen" | "window"` (with optional `format`, `quality`, `scale`, `max_width`, `max_height`, `max_bytes`) and/or `include_tree: true` on the action call itself to observe the result in the same tool call, or re-check state with `get_app_state`, `focused_window`, or an app-specific readback. +11. For `set_value`, `verified: true` is the only exact-content confirmation. `verified: false` requires fresh observation; password fields are deliberately unverifiable. The field is omitted for numeric values and early failures. On Hyprland, `type_text` passes one literal wtype argument after `--` for a classified nonsecret field, avoiding stdin keymap-refresh corruption in Electron. diff --git a/src/abs_pointer.rs b/src/abs_pointer.rs index 3aa57ff..761b9b4 100644 --- a/src/abs_pointer.rs +++ b/src/abs_pointer.rs @@ -28,7 +28,7 @@ pub(crate) struct PointerLanding { pub(crate) emitted: (i32, i32), } -#[derive(Clone, Copy)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] struct AbsPointerGeometry { max_x: i32, max_y: i32, @@ -46,6 +46,10 @@ impl AbsPointerGeometry { (self.max_x, self.max_y) } + fn matches_dimensions(self, width: i32, height: i32) -> bool { + self == Self::from_dimensions(width, height) + } + fn clamp_coordinates(self, x: i32, y: i32) -> (i32, i32) { (x.clamp(0, self.max_x), y.clamp(0, self.max_y)) } @@ -97,6 +101,10 @@ impl AbsPointer { Ok(Self { device, geometry }) } + pub fn matches_dimensions(&self, width: i32, height: i32) -> bool { + self.geometry.matches_dimensions(width, height) + } + /// Move the pointer to capture-pixel coordinates `(x, y)` and report /// both the requested point and the values emitted after edge clamping. pub fn move_to(&mut self, x: i32, y: i32) -> Result { @@ -198,6 +206,15 @@ mod tests { assert_eq!(geometry.axis_maxima(), (1919, 1079)); } + #[test] + fn stale_capture_geometry_is_rejected_before_pointer_reuse() { + let cached = AbsPointerGeometry::from_dimensions(3840, 2160); + + assert!(cached.matches_dimensions(3840, 2160)); + assert!(!cached.matches_dimensions(2560, 1440)); + assert!(!cached.matches_dimensions(3840, 1080)); + } + #[test] fn pointer_landing_preserves_the_request_and_emitted_coordinates() { let geometry = AbsPointerGeometry::from_dimensions(1920, 1080); diff --git a/src/atspi_tree.rs b/src/atspi_tree.rs index 7ec3239..8290442 100644 --- a/src/atspi_tree.rs +++ b/src/atspi_tree.rs @@ -4,7 +4,9 @@ use atspi::{ proxy::{ accessible::{AccessibleProxy, ObjectRefExt}, action::ActionProxy, - proxy_ext::ProxyExt, + editable_text::EditableTextProxy, + proxy_ext::{Proxies, ProxyExt}, + text::TextProxy, }, CoordType, ObjectRef, ObjectRefOwned, StateSet, }; @@ -15,7 +17,7 @@ use futures_util::{stream, StreamExt}; use schemars::JsonSchema; use serde::Serialize; use std::{collections::VecDeque, future::Future, time::Duration}; -use tokio::time::timeout; +use tokio::time::{sleep_until, timeout, timeout_at, Instant}; use zbus::{ fdo::DBusProxy, names::{BusName, UniqueName}, @@ -100,10 +102,22 @@ pub struct ActionInvocation { #[derive(Debug, Clone)] pub enum ValueSetInvocation { Numeric { value: f64 }, - EditableText, + EditableText { outcome: EditableTextSetOutcome }, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum EditableTextSetOutcome { + Verified, + MismatchAfterDeadline, + ReadbackFailed, + ReadbackTimedOut, + PasswordUnverifiable, } const MAX_TEXT_READBACK_CHARS: i32 = 4096; +const EDITABLE_TEXT_READ_CHUNK_UNITS: usize = 4096; +const EDITABLE_TEXT_VERIFY_TIMEOUT: Duration = Duration::from_secs(1); +const EDITABLE_TEXT_VERIFY_INTERVAL: Duration = Duration::from_millis(50); const MAX_TEXT_SELECTIONS: i32 = 8; const DEFAULT_SNAPSHOT_MAX_NODES: usize = 1_000; const HARD_SNAPSHOT_MAX_NODES: usize = 2_000; @@ -302,6 +316,10 @@ pub(crate) struct AccessibilitySnapshot { /// when they were matched by app name alone or not filtered at all: those /// roots can belong to any app, so no single pid describes the tree. pub root_pid: Option, + /// True when selected roots satisfy the supplied app/name filter, or no + /// filter was supplied. Normal snapshots retain their PID-only fallback; + /// action recovery requires this evidence before invoking a fresh node. + pub app_filter_matched: bool, /// True when max_nodes, max_depth, or the child read budget stopped /// traversal with unread elements left. Failed element reads do not count. pub truncated: bool, @@ -366,6 +384,7 @@ async fn snapshot_tree_inner( .await; let scoped = selected_roots.scoped; let root_pid = selected_roots.pid; + let app_filter_matched = selected_roots.app_filter_matched; let mut nodes = Vec::new(); let mut truncated = false; let mut traversal = BoundedTraversal::new(max_nodes); @@ -415,6 +434,7 @@ async fn snapshot_tree_inner( nodes, scoped, root_pid, + app_filter_matched, truncated, }) } @@ -647,7 +667,14 @@ pub async fn set_element_value(object_ref_id: &str, value: &str) -> Result, + proxies: &Proxies<'_>, + value: &str, +) -> Result { if let Ok(numeric_value) = value.parse::() { if let Ok(value_proxy) = proxies.value().await { value_proxy @@ -663,14 +690,7 @@ pub async fn set_element_value(object_ref_id: &str, value: &str) -> Result().is_err() && proxies.value().await.is_ok() { @@ -684,6 +704,159 @@ pub async fn set_element_value(object_ref_id: &str, value: &str) -> Result, + proxies: &Proxies<'_>, + editable_text: &EditableTextProxy<'_>, + value: &str, +) -> Result { + let ok = editable_text + .set_text_contents(value) + .await + .context("failed to set AT-SPI editable text contents")?; + if !ok { + return Err(anyhow!("AT-SPI EditableText rejected the new contents")); + } + + let outcome = verify_editable_text_contents(proxy, proxies, value).await; + Ok(ValueSetInvocation::EditableText { outcome }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum EditableTextReadbackError { + Failed, + TimedOut, +} + +async fn readback_within( + deadline: Instant, + read: impl Future>, +) -> std::result::Result { + timeout_at(deadline, read) + .await + .map_err(|_| EditableTextReadbackError::TimedOut)? + .map_err(|_| EditableTextReadbackError::Failed) +} + +async fn text_readback_matches( + text: &TextProxy<'_>, + requested: &str, + deadline: Instant, +) -> std::result::Result { + let character_count = readback_within(deadline, text.character_count()).await?; + if character_count < 0 { + return Err(EditableTextReadbackError::Failed); + } + if character_count as usize == requested.chars().count() { + return unicode_scalar_text_readback_matches(text, requested, deadline).await; + } + + let requested_utf16 = requested.encode_utf16().collect::>(); + if character_count as usize != requested_utf16.len() { + return Ok(false); + } + utf16_text_readback_matches(text, &requested_utf16, deadline).await +} + +async fn unicode_scalar_text_readback_matches( + text: &TextProxy<'_>, + requested: &str, + deadline: Instant, +) -> std::result::Result { + let requested_chars = requested.chars().collect::>(); + let mut offset = 0; + while offset < requested_chars.len() { + let end = offset + .saturating_add(EDITABLE_TEXT_READ_CHUNK_UNITS) + .min(requested_chars.len()); + let chunk = readback_within(deadline, text.get_text(offset as i32, end as i32)).await?; + if !chunk + .chars() + .eq(requested_chars[offset..end].iter().copied()) + { + return Ok(false); + } + offset = end; + } + + Ok(true) +} + +async fn utf16_text_readback_matches( + text: &TextProxy<'_>, + requested: &[u16], + deadline: Instant, +) -> std::result::Result { + let mut offset = 0; + while offset < requested.len() { + let mut end = offset + .saturating_add(EDITABLE_TEXT_READ_CHUNK_UNITS) + .min(requested.len()); + if end < requested.len() && (0xD800..=0xDBFF).contains(&requested[end - 1]) { + end -= 1; + } + let chunk = readback_within(deadline, text.get_text(offset as i32, end as i32)).await?; + if !chunk + .encode_utf16() + .eq(requested[offset..end].iter().copied()) + { + return Ok(false); + } + offset = end; + } + + Ok(true) +} + +async fn verify_editable_text_contents( + proxy: &AccessibleProxy<'_>, + proxies: &Proxies<'_>, + requested: &str, +) -> EditableTextSetOutcome { + let deadline = Instant::now() + EDITABLE_TEXT_VERIFY_TIMEOUT; + match readback_within(deadline, proxy.get_role()).await { + Ok(atspi::Role::PasswordText) => return EditableTextSetOutcome::PasswordUnverifiable, + Ok(_) => {} + Err(EditableTextReadbackError::Failed) => { + return EditableTextSetOutcome::ReadbackFailed; + } + Err(EditableTextReadbackError::TimedOut) => { + return EditableTextSetOutcome::ReadbackTimedOut; + } + } + let text = match readback_within(deadline, proxies.text()).await { + Ok(text) => text, + Err(EditableTextReadbackError::Failed) => { + return EditableTextSetOutcome::ReadbackFailed; + } + Err(EditableTextReadbackError::TimedOut) => { + return EditableTextSetOutcome::ReadbackTimedOut; + } + }; + + loop { + match text_readback_matches(&text, requested, deadline).await { + Ok(true) => return EditableTextSetOutcome::Verified, + Ok(false) => {} + Err(EditableTextReadbackError::Failed) => { + return EditableTextSetOutcome::ReadbackFailed; + } + Err(EditableTextReadbackError::TimedOut) => { + return EditableTextSetOutcome::ReadbackTimedOut; + } + } + + let now = Instant::now(); + if now >= deadline { + return EditableTextSetOutcome::MismatchAfterDeadline; + } + sleep_until((now + EDITABLE_TEXT_VERIFY_INTERVAL).min(deadline)).await; + if Instant::now() >= deadline { + return EditableTextSetOutcome::MismatchAfterDeadline; + } + } +} + async fn connect() -> Result { hydrate_session_bus_env(); AccessibilityConnection::new() @@ -735,6 +908,7 @@ struct SelectedRoots { scoped: bool, /// Set only when the roots were chosen because they belong to this pid. pid: Option, + app_filter_matched: bool, } /// Normalized app-name filter, or `None` when the caller passed nothing usable. @@ -783,6 +957,7 @@ async fn select_roots( roots: pid_and_filter_matches, scoped: true, pid: Some(target_pid), + app_filter_matched: true, }; } if !pid_matches.is_empty() { @@ -790,6 +965,7 @@ async fn select_roots( roots: pid_matches, scoped: true, pid: Some(target_pid), + app_filter_matched: needle.is_none(), }; } @@ -801,6 +977,7 @@ async fn select_roots( roots: remaining, scoped: false, pid: None, + app_filter_matched: true, }; }; @@ -811,10 +988,12 @@ async fn select_roots( } } + let app_filter_matched = !selected.is_empty(); SelectedRoots { roots: selected, scoped: true, pid: None, + app_filter_matched, } } @@ -1015,7 +1194,7 @@ async fn read_actions(proxy: &ActionProxy<'_>) -> Result> { Ok(stream::iter(0..count) .map(|index| async move { let (name, description, keybinding) = tokio::join!( - proxy.get_localized_name(index), + proxy.get_name(index), proxy.get_description(index), proxy.get_key_binding(index), ); @@ -1193,7 +1372,7 @@ mod tests { process::{Child, Command, Stdio}, sync::{ atomic::{AtomicUsize, Ordering}, - Arc, + Arc, Mutex, }, }; @@ -1223,6 +1402,295 @@ mod tests { } } + const EDITABLE_FIXTURE_PATH: &str = "/editable"; + + #[derive(Default)] + struct FakeEditableState { + contents: String, + mutations: usize, + character_count_reads: usize, + get_text_reads: usize, + numeric_value: f64, + numeric_writes: usize, + } + + #[derive(Clone, Copy)] + enum FakeMutation { + Apply, + NoOp, + Delayed, + Normalize, + Mask, + Reject, + Fail, + } + + #[derive(Clone, Copy, Debug)] + enum FakeTextOffsetModel { + UnicodeScalars, + Utf16CodeUnits, + } + + #[derive(Clone, Copy)] + enum FakeTextBehavior { + Normal, + CharacterCountError, + GetTextError, + NegativeCount, + StallCharacterCount, + } + + #[derive(Clone, Copy)] + struct FakeEditableConfig { + mutation: FakeMutation, + text: FakeTextBehavior, + text_offset_model: FakeTextOffsetModel, + role: atspi::Role, + has_text: bool, + has_value: bool, + } + + impl Default for FakeEditableConfig { + fn default() -> Self { + Self { + mutation: FakeMutation::Apply, + text: FakeTextBehavior::Normal, + text_offset_model: FakeTextOffsetModel::Utf16CodeUnits, + role: atspi::Role::Text, + has_text: true, + has_value: false, + } + } + } + + struct FakeEditableAccessible(FakeEditableConfig); + + #[zbus::interface(name = "org.a11y.atspi.Accessible")] + impl FakeEditableAccessible { + #[zbus(property)] + fn child_count(&self) -> i32 { + 0 + } + + fn get_interfaces(&self) -> Vec<&str> { + let mut interfaces = vec!["org.a11y.atspi.Accessible", "org.a11y.atspi.EditableText"]; + if self.0.has_text { + interfaces.push("org.a11y.atspi.Text"); + } + if self.0.has_value { + interfaces.push("org.a11y.atspi.Value"); + } + interfaces + } + + fn get_role(&self) -> atspi::Role { + self.0.role + } + } + + struct FakeEditableText { + state: Arc>, + mutation: FakeMutation, + } + + #[zbus::interface(name = "org.a11y.atspi.EditableText")] + impl FakeEditableText { + fn set_text_contents(&self, new_contents: &str) -> zbus::fdo::Result { + self.state.lock().unwrap().mutations += 1; + match self.mutation { + FakeMutation::Apply => { + self.state.lock().unwrap().contents = new_contents.to_string(); + } + FakeMutation::NoOp => {} + FakeMutation::Delayed => { + let state = Arc::clone(&self.state); + let new_contents = new_contents.to_string(); + tokio::spawn(async move { + tokio::time::sleep(Duration::from_millis(150)).await; + state.lock().unwrap().contents = new_contents; + }); + } + FakeMutation::Normalize => { + self.state.lock().unwrap().contents = new_contents.replace("\r\n", "\n"); + } + FakeMutation::Mask => { + self.state.lock().unwrap().contents = "•".repeat(new_contents.chars().count()); + } + FakeMutation::Reject => return Ok(false), + FakeMutation::Fail => { + return Err(zbus::fdo::Error::Failed( + "injected SetTextContents failure".into(), + )); + } + } + Ok(true) + } + } + + fn unicode_scalar_offset_to_byte_index(contents: &str, offset: usize) -> Option { + contents + .char_indices() + .map(|(byte_index, _)| byte_index) + .chain(std::iter::once(contents.len())) + .nth(offset) + } + + fn utf16_offset_to_byte_index(contents: &str, offset: usize) -> Option { + let mut utf16_offset = 0; + for (byte_index, character) in contents.char_indices() { + if utf16_offset == offset { + return Some(byte_index); + } + utf16_offset += character.len_utf16(); + } + (utf16_offset == offset).then_some(contents.len()) + } + + struct FakeText { + state: Arc>, + behavior: FakeTextBehavior, + text_offset_model: FakeTextOffsetModel, + } + + #[zbus::interface(name = "org.a11y.atspi.Text")] + impl FakeText { + #[zbus(property)] + async fn character_count(&self) -> zbus::fdo::Result { + self.state.lock().unwrap().character_count_reads += 1; + match self.behavior { + FakeTextBehavior::CharacterCountError => Err(zbus::fdo::Error::Failed( + "injected CharacterCount failure".into(), + )), + FakeTextBehavior::NegativeCount => Ok(-1), + FakeTextBehavior::StallCharacterCount => std::future::pending().await, + _ => { + let state = self.state.lock().unwrap(); + let count = match self.text_offset_model { + FakeTextOffsetModel::UnicodeScalars => state.contents.chars().count(), + FakeTextOffsetModel::Utf16CodeUnits => { + state.contents.encode_utf16().count() + } + }; + Ok(count as i32) + } + } + } + + fn get_text(&self, start_offset: i32, end_offset: i32) -> zbus::fdo::Result { + let mut state = self.state.lock().unwrap(); + state.get_text_reads += 1; + if matches!(self.behavior, FakeTextBehavior::GetTextError) { + return Err(zbus::fdo::Error::Failed("injected GetText failure".into())); + } + if start_offset < 0 || end_offset < start_offset { + return Err(zbus::fdo::Error::InvalidArgs("invalid text offsets".into())); + } + let offset_to_byte_index = match self.text_offset_model { + FakeTextOffsetModel::UnicodeScalars => unicode_scalar_offset_to_byte_index, + FakeTextOffsetModel::Utf16CodeUnits => utf16_offset_to_byte_index, + }; + let start = offset_to_byte_index(&state.contents, start_offset as usize) + .ok_or_else(|| zbus::fdo::Error::InvalidArgs("invalid text offsets".into()))?; + let end = offset_to_byte_index(&state.contents, end_offset as usize) + .ok_or_else(|| zbus::fdo::Error::InvalidArgs("invalid text offsets".into()))?; + Ok(state.contents[start..end].to_string()) + } + } + + struct FakeValue(Arc>); + + #[zbus::interface(name = "org.a11y.atspi.Value")] + impl FakeValue { + #[zbus(property)] + fn current_value(&self) -> f64 { + self.0.lock().unwrap().numeric_value + } + + #[zbus(property)] + fn set_current_value(&self, value: f64) { + let mut state = self.0.lock().unwrap(); + state.numeric_value = value; + state.numeric_writes += 1; + } + } + + async fn start_editable_fixture( + initial_contents: &str, + config: FakeEditableConfig, + ) -> ( + PrivateBus, + zbus::Connection, + zbus::Connection, + Arc>, + ) { + let (bus, address) = PrivateBus::start(); + let state = Arc::new(Mutex::new(FakeEditableState { + contents: initial_contents.to_string(), + ..Default::default() + })); + let mut builder = zbus::connection::Builder::address(address.as_str()) + .unwrap() + .serve_at(EDITABLE_FIXTURE_PATH, FakeEditableAccessible(config)) + .unwrap() + .serve_at( + EDITABLE_FIXTURE_PATH, + FakeEditableText { + state: Arc::clone(&state), + mutation: config.mutation, + }, + ) + .unwrap(); + if config.has_text { + builder = builder + .serve_at( + EDITABLE_FIXTURE_PATH, + FakeText { + state: Arc::clone(&state), + behavior: config.text, + text_offset_model: config.text_offset_model, + }, + ) + .unwrap(); + } + if config.has_value { + builder = builder + .serve_at(EDITABLE_FIXTURE_PATH, FakeValue(Arc::clone(&state))) + .unwrap(); + } + let peer = builder.build().await.unwrap(); + let client = zbus::connection::Builder::address(address.as_str()) + .unwrap() + .build() + .await + .unwrap(); + (bus, peer, client, state) + } + + async fn dispatch_editable_fixture( + client: &zbus::Connection, + peer: &zbus::Connection, + value: &str, + ) -> Result { + let proxy = AccessibleProxy::builder(client) + .destination(peer.unique_name().unwrap().clone()) + .unwrap() + .path(EDITABLE_FIXTURE_PATH) + .unwrap() + .cache_properties(zbus::proxy::CacheProperties::No) + .build() + .await + .unwrap(); + let proxies = proxy.proxies().await.unwrap(); + set_element_value_with_proxies(&proxy, &proxies, value).await + } + + fn editable_outcome(invocation: ValueSetInvocation) -> EditableTextSetOutcome { + match invocation { + ValueSetInvocation::EditableText { outcome } => outcome, + ValueSetInvocation::Numeric { .. } => panic!("expected editable-text dispatch"), + } + } + struct FakeAccessible { name: String, children: Vec<(String, OwnedObjectPath)>, @@ -1288,17 +1756,17 @@ mod tests { self.count } - fn get_name(&self, index: i32) -> &str { + async fn get_name(&self, index: i32) -> &str { assert!((0..self.count).contains(&index)); + self.name_reads.fetch_add(1, Ordering::SeqCst); + if self.stall_indexed_reads { + std::future::pending::<()>().await; + } "click" } async fn get_localized_name(&self, index: i32) -> &str { assert!((0..self.count).contains(&index)); - self.name_reads.fetch_add(1, Ordering::SeqCst); - if self.stall_indexed_reads { - std::future::pending::<()>().await; - } "Activate" } @@ -1317,6 +1785,429 @@ mod tests { } } + struct FakeQtAction; + + #[zbus::interface(name = "org.a11y.atspi.Action")] + impl FakeQtAction { + #[zbus(property, name = "NActions")] + fn nactions(&self) -> i32 { + 2 + } + + fn get_name(&self, index: i32) -> &str { + match index { + 0 => "Press", + 1 => "Set Focus", + _ => panic!("invalid action index"), + } + } + + fn get_description(&self, index: i32) -> &str { + match index { + 0 => "Eylemi tetikler", + 1 => "Odağı ayarlar", + _ => panic!("invalid action index"), + } + } + + fn get_key_binding(&self, _index: i32) -> &str { + "" + } + } + + #[tokio::test] + async fn editable_text_true_without_publication_is_mismatch_and_mutates_once() { + let (_bus, peer, client, state) = start_editable_fixture( + "original", + FakeEditableConfig { + mutation: FakeMutation::NoOp, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "modified") + .await + .unwrap(), + ); + + // A last poll can cross the same fixed deadline. Both outcomes mean + // an unverified write; neither permits verification or replay. + assert!( + matches!( + outcome, + EditableTextSetOutcome::MismatchAfterDeadline + | EditableTextSetOutcome::ReadbackTimedOut + ), + "{outcome:?}" + ); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert!(state.character_count_reads > 1); + assert!(state.get_text_reads > 1); + assert_eq!(state.contents, "original"); + } + + #[tokio::test] + async fn editable_text_exact_readback_accepts_empty_whitespace_and_trailing_newline() { + let (_bus, peer, client, state) = + start_editable_fixture("initial", FakeEditableConfig::default()).await; + let values = [ + "", + " \t\n", + " leading and trailing\t \n", + "trailing newline\n", + "Case and unicode € 内容 😀🦀𐐷", + ]; + + for value in values { + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, value) + .await + .unwrap(), + ); + assert_eq!(outcome, EditableTextSetOutcome::Verified); + assert_eq!(state.lock().unwrap().contents, value); + } + assert_eq!(state.lock().unwrap().mutations, values.len()); + } + + #[tokio::test] + async fn editable_text_non_bmp_readback_supports_scalar_and_utf16_peers() { + let requested = "🦀 GTK verified "; + for text_offset_model in [ + FakeTextOffsetModel::UnicodeScalars, + FakeTextOffsetModel::Utf16CodeUnits, + ] { + let (_bus, peer, client, state) = start_editable_fixture( + "", + FakeEditableConfig { + text_offset_model, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, requested) + .await + .unwrap(), + ); + + assert_eq!( + outcome, + EditableTextSetOutcome::Verified, + "{text_offset_model:?}" + ); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.contents, requested); + } + } + + #[tokio::test] + async fn editable_text_non_bmp_matching_count_does_not_accept_different_content() { + let requested = "🦀 GTK verified "; + let published = "🦀 GTK verified!"; + assert_eq!(requested.chars().count(), published.chars().count()); + assert_eq!( + requested.encode_utf16().count(), + published.encode_utf16().count() + ); + for text_offset_model in [ + FakeTextOffsetModel::UnicodeScalars, + FakeTextOffsetModel::Utf16CodeUnits, + ] { + let (_bus, peer, client, state) = start_editable_fixture( + published, + FakeEditableConfig { + mutation: FakeMutation::NoOp, + text_offset_model, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, requested) + .await + .unwrap(), + ); + + // A last poll can cross the same fixed deadline. Both outcomes + // leave the write unverified without extending its deadline. + assert!( + matches!( + outcome, + EditableTextSetOutcome::MismatchAfterDeadline + | EditableTextSetOutcome::ReadbackTimedOut + ), + "{text_offset_model:?}: {outcome:?}" + ); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert!(state.get_text_reads > 0, "{text_offset_model:?}"); + assert_eq!(state.contents, published); + } + } + + #[tokio::test] + async fn editable_text_readback_chunks_utf16_without_splitting_non_bmp() { + let (_bus, peer, client, state) = start_editable_fixture( + "", + FakeEditableConfig { + text_offset_model: FakeTextOffsetModel::Utf16CodeUnits, + ..Default::default() + }, + ) + .await; + let mut expected = "a".repeat(EDITABLE_TEXT_READ_CHUNK_UNITS - 1); + // The crab's surrogate pair straddles the first 4096-unit read boundary. + expected.push_str("🦀é"); + expected.push_str(&"内容\n".repeat(1_200)); + assert!(expected.encode_utf16().count() > MAX_TEXT_READBACK_CHARS as usize); + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, &expected) + .await + .unwrap(), + ); + + assert_eq!(outcome, EditableTextSetOutcome::Verified); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert!(state.get_text_reads >= 2); + assert_eq!(state.contents, expected); + } + + #[tokio::test] + async fn editable_text_readback_waits_for_async_publication() { + let (_bus, peer, client, state) = start_editable_fixture( + "pending", + FakeEditableConfig { + mutation: FakeMutation::Delayed, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "published asynchronously") + .await + .unwrap(), + ); + + assert_eq!(outcome, EditableTextSetOutcome::Verified); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert!(state.character_count_reads > 1); + assert_eq!(state.contents, "published asynchronously"); + } + + #[tokio::test] + async fn editable_text_normalization_mismatch_is_not_verified() { + let (_bus, peer, client, state) = start_editable_fixture( + "", + FakeEditableConfig { + mutation: FakeMutation::Normalize, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "first\r\nsecond\n") + .await + .unwrap(), + ); + + assert_eq!(outcome, EditableTextSetOutcome::MismatchAfterDeadline); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert!(state.character_count_reads > 1); + assert_eq!(state.contents, "first\nsecond\n"); + } + + #[tokio::test] + async fn password_text_is_unverifiable_even_when_requested_text_is_bullets() { + let (_bus, peer, client, state) = start_editable_fixture( + "", + FakeEditableConfig { + mutation: FakeMutation::Mask, + role: atspi::Role::PasswordText, + ..Default::default() + }, + ) + .await; + + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "••••••••") + .await + .unwrap(), + ); + + assert_eq!(outcome, EditableTextSetOutcome::PasswordUnverifiable); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.character_count_reads, 0); + assert_eq!(state.contents, "••••••••"); + } + + #[tokio::test] + async fn editable_text_missing_text_and_read_errors_are_structured_outcomes() { + let configs = [ + FakeEditableConfig { + has_text: false, + ..Default::default() + }, + FakeEditableConfig { + text: FakeTextBehavior::CharacterCountError, + ..Default::default() + }, + FakeEditableConfig { + text: FakeTextBehavior::GetTextError, + ..Default::default() + }, + ]; + + for config in configs { + let (_bus, peer, client, state) = start_editable_fixture("before", config).await; + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "readback may have changed") + .await + .unwrap(), + ); + assert_eq!(outcome, EditableTextSetOutcome::ReadbackFailed); + assert_eq!(state.lock().unwrap().mutations, 1); + } + } + + #[tokio::test] + async fn editable_text_negative_counts_and_stalled_readback_are_structured_outcomes() { + let cases = [ + ( + FakeTextBehavior::NegativeCount, + EditableTextSetOutcome::ReadbackFailed, + ), + ( + FakeTextBehavior::StallCharacterCount, + EditableTextSetOutcome::ReadbackTimedOut, + ), + ]; + + for (text, expected) in cases { + let (_bus, peer, client, state) = start_editable_fixture( + "before", + FakeEditableConfig { + text, + ..Default::default() + }, + ) + .await; + let outcome = editable_outcome( + dispatch_editable_fixture(&client, &peer, "after") + .await + .unwrap(), + ); + assert_eq!(outcome, expected); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.get_text_reads, 0); + } + } + + #[tokio::test] + async fn editable_text_rejection_and_mutation_errors_remain_errors() { + let cases = [ + (FakeMutation::Reject, "rejected"), + (FakeMutation::Fail, "failed to set"), + ]; + + for (mutation, expected_error) in cases { + let (_bus, peer, client, state) = start_editable_fixture( + "before", + FakeEditableConfig { + mutation, + ..Default::default() + }, + ) + .await; + let error = dispatch_editable_fixture(&client, &peer, "after") + .await + .unwrap_err(); + assert!(error.to_string().contains(expected_error)); + let state = state.lock().unwrap(); + assert_eq!(state.mutations, 1); + assert_eq!(state.character_count_reads, 0); + } + } + + #[tokio::test] + async fn numeric_value_dispatch_remains_isolated_from_editable_text() { + let (_bus, peer, client, state) = start_editable_fixture( + "unchanged", + FakeEditableConfig { + has_value: true, + ..Default::default() + }, + ) + .await; + + let invocation = dispatch_editable_fixture(&client, &peer, "42.5") + .await + .unwrap(); + + match invocation { + ValueSetInvocation::Numeric { value } => assert_eq!(value, 42.5), + ValueSetInvocation::EditableText { .. } => panic!("expected numeric dispatch"), + } + let state = state.lock().unwrap(); + assert_eq!(state.numeric_value, 42.5); + assert_eq!(state.numeric_writes, 1); + assert_eq!(state.mutations, 0); + assert_eq!(state.character_count_reads, 0); + assert_eq!(state.contents, "unchanged"); + } + + #[tokio::test] + async fn qt_action_names_do_not_depend_on_localized_names() { + let (_bus, address) = PrivateBus::start(); + let peer = zbus::connection::Builder::address(address.as_str()) + .unwrap() + .serve_at("/actions/qt", FakeQtAction) + .unwrap() + .build() + .await + .unwrap(); + let client = zbus::connection::Builder::address(address.as_str()) + .unwrap() + .build() + .await + .unwrap(); + let action = ActionProxy::builder(&client) + .destination(peer.unique_name().unwrap().clone()) + .unwrap() + .path("/actions/qt") + .unwrap() + .cache_properties(zbus::proxy::CacheProperties::No) + .build() + .await + .unwrap(); + + let actions = timeout(Duration::from_secs(1), read_actions(&action)) + .await + .expect("Qt action reads must not require GetLocalizedName") + .unwrap(); + + assert_eq!(actions.len(), 2); + assert_eq!(actions[0].name, "Press"); + assert_eq!(actions[0].description, "Eylemi tetikler"); + assert_eq!(actions[1].name, "Set Focus"); + assert_eq!(actions[1].description, "Odağı ayarlar"); + assert_eq!(select_named_action_index(&actions, "press").unwrap(), 0); + } + #[tokio::test] async fn indexed_children_resolve_webkit_names_on_the_accessibility_bus() { let (_bus, address) = PrivateBus::start(); @@ -1408,10 +2299,10 @@ mod tests { .expect("indexed action reads must not wait for GetActions") .unwrap(); assert_eq!(actions.len(), 1); - assert_eq!(actions[0].name, "Activate"); + assert_eq!(actions[0].name, "click"); assert_eq!(actions[0].description, "Activate the control"); assert_eq!(actions[0].keybinding, "Return"); - assert_eq!(select_named_action_index(&actions, "Activate").unwrap(), 0); + assert_eq!(select_named_action_index(&actions, "click").unwrap(), 0); assert!(action.do_action(0).await.unwrap()); } diff --git a/src/lib.rs b/src/lib.rs index ea94d41..5446dd3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -26,7 +26,8 @@ pub mod atspi_tree { accessibility_chrome_context, focused_element_summary_in_app, list_accessible_apps, object_ref_owner_pid, perform_action, perform_named_action, probe_focused_element, set_element_value, snapshot_accessibility_tree, snapshot_limits, AccessibleAppSummary, - FocusProbe, FocusedElementSummary, ValueSetInvocation, UNKNOWN_ROLE, + EditableTextSetOutcome, FocusProbe, FocusedElementSummary, ValueSetInvocation, + UNKNOWN_ROLE, }; pub use crate::atspi_tree_impl::{ snapshot_tree, AccessibilityAction, AccessibilityNode, AccessibilityText, diff --git a/src/server.rs b/src/server.rs index 55a06c0..57764a6 100644 --- a/src/server.rs +++ b/src/server.rs @@ -2,8 +2,8 @@ use crate::atspi_tree::{ accessibility_chrome_context, focused_element_summary_in_app, list_accessible_apps, object_ref_owner_pid, perform_action as invoke_accessibility_action, perform_named_action, probe_focused_element, set_element_value, snapshot_accessibility_tree, AccessibilityAction, - AccessibilityNode, AccessibleAppSummary, Bounds, FocusProbe, FocusedElementSummary, - ValueSetInvocation, + AccessibilityNode, AccessibleAppSummary, Bounds, EditableTextSetOutcome, FocusProbe, + FocusedElementSummary, ValueSetInvocation, }; use crate::diagnostics::{doctor_report, setup_accessibility_report, DoctorReport, SetupReport}; use crate::gnome_extension::{setup_window_targeting_report, WindowTargetingSetupReport}; @@ -18,6 +18,9 @@ use crate::screenshot::{ ScreenshotOutputFormat, ScreenshotPayloadOptions, }; use crate::terminal::{terminal_paste_shortcut, TerminalPasteShortcut}; +use crate::windowing::backends::kwin::{ + KwinElementClickPoint, KwinElementClickTarget, KwinElementFrame, +}; use crate::windowing::registry; use crate::windows::{ focus_window_target, focused_window, list_windows, resolve_window_target, @@ -72,12 +75,29 @@ const SHELL_MAX_CWD_BYTES: usize = 4096; const SHELL_MAX_ENV_ENTRIES: usize = 64; const SHELL_MAX_ENV_BYTES: usize = 64 * 1024; const SHELL_RESPONSE_STREAM_BYTES: usize = 512 * 1024; +const UNMATCHABLE_ACCESSIBILITY_APP_FILTER: &str = "\0"; -/// Nodes and their app ownership must come from the same snapshot. +/// Nodes, their app/window ownership, and their recovery scope must come from +/// the same snapshot. #[derive(Default)] struct CachedAccessibilitySnapshot { nodes: Vec, pid: Option, + window_id: Option, + window_backend: Option, + recovery_scope: Option, +} + +#[derive(Clone)] +struct AccessibilityRecoveryScope { + params: GetAppStateParams, + target_pid: Option, +} + +struct RefreshedAccessibilitySnapshot { + snapshot: crate::atspi_tree_impl::AccessibilitySnapshot, + target_pid: Option, + window: Option, } #[derive(Clone, Default)] @@ -387,7 +407,7 @@ impl ComputerUseLinux { let mut diagnostics = tokio::task::spawn_blocking(doctor_report) .await .expect("diagnostics task panicked"); - let (window_context, window_error, window_permissions_hint) = + let (window_context, window_error, window_permissions_hint, window_enumerated) = self.resolve_window_context(¶ms).await; let (max_nodes, max_depth) = crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth); @@ -403,7 +423,7 @@ impl ComputerUseLinux { .as_deref() .is_some_and(|name| !name.trim().is_empty()); let app_filter = self - .resolve_accessibility_app_filter(¶ms, window_context.as_ref()) + .resolve_accessibility_app_filter(¶ms, window_context.as_ref(), window_enumerated) .await; let (screenshot, screenshot_error) = if include_screenshot { let result: Result = async { @@ -445,7 +465,10 @@ impl ComputerUseLinux { let mut shell_only_note = None; let (accessibility_tree, accessibility_tree_raw_count, accessibility_error) = if diagnostics.readiness.can_build_accessibility_tree { - let target_pid = window_context.as_ref().and_then(|window| window.pid); + let target_pid = accessibility_snapshot_target_pid( + app_filter.as_deref(), + window_context.as_ref().and_then(|window| window.pid), + ); match snapshot_accessibility_tree( app_filter.as_deref(), target_pid, @@ -485,8 +508,14 @@ impl ComputerUseLinux { // match that can be another app entirely; recording the pid then // would let that app's index pass the target check instead of // taking the per-node owner lookup. - self.commit_snapshot(&accessibility_tree, tree_root_pid) - .await; + self.commit_app_state_snapshot( + &accessibility_tree, + tree_root_pid, + window_context.as_ref(), + ¶ms, + tree_scoped, + ) + .await; } else { self.commit_snapshot(&[], None).await; } @@ -723,6 +752,74 @@ impl ComputerUseLinux { } } + async fn ensure_abs_pointer_for_kwin_capture( + &self, + width: u32, + height: u32, + ) -> std::result::Result { + if env_flag_enabled("CU_DISABLE_ABS_POINTER") + || env_flag_enabled("COMPUTER_USE_LINUX_FORCE_YDOTOOL_POINTER") + || (env_flag_enabled("COMPUTER_USE_LINUX_FORCE_PORTAL_POINTER") + && self.is_wayland_session()) + { + return Ok(false); + } + let stale = { + let mut guard = self + .abs_pointer + .lock() + .map_err(|_| "absolute pointer lock was poisoned".to_string())?; + match guard.as_ref() { + Some(pointer) if pointer.matches_dimensions(width as i32, height as i32) => { + return Ok(true) + } + Some(_) => { + *guard = None; + true + } + None => false, + } + }; + self.create_abs_pointer_for_kwin_capture_with( + width, + height, + stale, + crate::abs_pointer::AbsPointer::create, + ) + .await + } + + async fn create_abs_pointer_for_kwin_capture_with( + &self, + width: u32, + height: u32, + stale: bool, + create: F, + ) -> std::result::Result + where + F: FnOnce(i32, i32) -> Result + Send + 'static, + { + let created = + tokio::task::spawn_blocking(move || create(width as i32, height as i32)).await; + match created { + Ok(Ok(pointer)) => { + let mut guard = self + .abs_pointer + .lock() + .map_err(|_| "absolute pointer lock was poisoned".to_string())?; + *guard = Some(pointer); + Ok(true) + } + Ok(Err(error)) if stale => Err(format!( + "cached absolute-pointer geometry was stale for the current raw capture and the device could not be recreated: {error:#}" + )), + Err(error) if stale => Err(format!( + "cached absolute-pointer geometry was stale for the current raw capture and the device recreation task failed: {error}" + )), + _ => Ok(false), + } + } + #[tool( name = "click", output_schema = rmcp::handler::server::tool::schema_for_output::(), @@ -749,6 +846,7 @@ impl ComputerUseLinux { let received = Some(serde_json::json!(params.clone())); let input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await; let mut portal_target_point = None; + let mut input_focus = None; let mut target_pid = params.pid; // Raise the target window first (if specified) so the click lands on the // intended app rather than whatever is stacked on top at that pixel. @@ -819,6 +917,7 @@ impl ComputerUseLinux { .zip(params.y) .and_then(|(x, y)| coordinate_map.portal_point(x, y)); } + input_focus = focus; } if params.x.zip(params.y).is_none() { if let Some(node) = self.cached_node_for( @@ -899,8 +998,28 @@ impl ComputerUseLinux { }), }; } - let ClickTarget::Coordinates(x, y) = target else { - unreachable!("click target must resolve to coordinates or an AT-SPI action"); + let (x, y, kwin_click) = match target { + ClickTarget::Coordinates(x, y) => (x, y, None), + ClickTarget::ElementBounds { node, center } => { + if self.should_map_kwin_element_click(input_focus.as_ref()) { + match self + .prepare_kwin_element_click(&node, center, input_focus.as_ref()) + .await + { + Ok(prepared) => { + portal_target_point = Some(prepared.portal_point()); + let (x, y) = prepared.uinput_point(); + (x, y, Some(prepared)) + } + Err(message) => { + return self.kwin_element_click_failure(node.index, &message, received); + } + } + } else { + (center.0, center.1, None) + } + } + ClickTarget::PrimaryAction { .. } => unreachable!(), }; self.indicator .pointer("click", (x, y), self.indicator_space()) @@ -916,7 +1035,27 @@ impl ComputerUseLinux { if let Some(abs_button) = crate::abs_pointer::PointerButton::from_name(params.button.as_deref()) { - if self.ensure_abs_pointer().await { + let abs_pointer_ready = if let Some(kwin_click) = kwin_click { + match self + .ensure_abs_pointer_for_kwin_capture( + kwin_click.capture_size.0, + kwin_click.capture_size.1, + ) + .await + { + Ok(ready) => ready, + Err(message) => { + return self.kwin_element_click_failure( + kwin_click.element_index, + &message, + received, + ); + } + } + } else { + self.ensure_abs_pointer().await + }; + if abs_pointer_ready { let pointer = Arc::clone(&self.abs_pointer); let count = params.click_count.unwrap_or(1).clamp(1, 10); let (input_guard, result) = run_cancellation_safe_input(input_guard, async move { @@ -927,7 +1066,11 @@ impl ComputerUseLinux { }) .await; let _input_guard = input_guard; - return Json(abs_pointer_result("click", result, received)); + return Json(kwin_element_click_dispatch_output( + abs_pointer_result("click", result, received), + kwin_click, + "the uinput absolute pointer", + )); } } let off_screen_note = self.off_screen_note_for_point(x, y).await; @@ -951,15 +1094,20 @@ impl ComputerUseLinux { .await { Ok(()) => { - return Json(with_notes( - ActionOutput { - ok: true, - implemented: true, - action: "click".to_string(), - message: "Action sent through the remote desktop portal.".to_string(), - received, - }, - off_screen_note.clone(), + return Json(kwin_element_click_dispatch_output( + with_notes( + ActionOutput { + ok: true, + implemented: true, + action: "click".to_string(), + message: "Action sent through the remote desktop portal." + .to_string(), + received, + }, + off_screen_note.clone(), + ), + kwin_click, + "the remote desktop portal", )); } Err(error) => { @@ -992,16 +1140,20 @@ impl ComputerUseLinux { .await { Ok(()) => { - return Json(with_notes( - ActionOutput { - ok: true, - implemented: true, - action: "click".to_string(), - message: "Action sent through the remote desktop portal." - .to_string(), - received, - }, - off_screen_note.clone(), + return Json(kwin_element_click_dispatch_output( + with_notes( + ActionOutput { + ok: true, + implemented: true, + action: "click".to_string(), + message: "Action sent through the remote desktop portal." + .to_string(), + received, + }, + off_screen_note.clone(), + ), + kwin_click, + "the remote desktop portal", )); } Err(error) => { @@ -1100,8 +1252,8 @@ impl ComputerUseLinux { #[tool( name = "set_value", - output_schema = rmcp::handler::server::tool::schema_for_output::(), - description = "Set the value of a settable accessibility element selected by index, identifier, or semantic selector. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.", + output_schema = rmcp::handler::server::tool::schema_for_output::(), + description = "Set the value of a settable accessibility element selected by index, identifier, or semantic selector. Editable-text writes are verified by exact readback; password fields are explicitly unverifiable. Pass screenshot_after (true, \"screen\", or \"window\") or include_tree=true to return post-action observation in the same result.", annotations( read_only_hint = false, destructive_hint = true, @@ -1114,11 +1266,12 @@ impl ComputerUseLinux { Parameters(params): Parameters, ) -> Result { let capture = params.capture.clone(); - let Json(output) = self.dispatch_set_value(params).await; - self.finish_action_output(output, &capture, None).await + let (output, verified) = self.dispatch_set_value(params).await; + self.finish_set_value_output(output, verified, &capture) + .await } - async fn dispatch_set_value(&self, params: SetValueParams) -> Json { + async fn dispatch_set_value(&self, params: SetValueParams) -> (ActionOutput, Option) { let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await; let received = Some(serde_json::json!(params.clone())); let object_ref = match self.resolve_object_ref( @@ -1129,39 +1282,44 @@ impl ComputerUseLinux { ) { Ok(object_ref) => object_ref, Err(message) => { - return Json(ActionOutput { - ok: false, - implemented: true, - action: "set_value".to_string(), - message, - received, - }); + return ( + ActionOutput { + ok: false, + implemented: true, + action: "set_value".to_string(), + message, + received, + }, + None, + ); } }; self.indicator.text("set_value", ¶ms.value, false).await; match set_element_value(&object_ref, ¶ms.value).await { - Ok(ValueSetInvocation::Numeric { value }) => Json(ActionOutput { - ok: true, - implemented: true, - action: "set_value".to_string(), - message: format!("AT-SPI numeric value set to {value}."), - received, - }), - Ok(ValueSetInvocation::EditableText) => Json(ActionOutput { - ok: true, - implemented: true, - action: "set_value".to_string(), - message: "AT-SPI editable text contents set.".to_string(), - received, - }), - Err(error) => Json(ActionOutput { - ok: false, - implemented: true, - action: "set_value".to_string(), - message: error.to_string(), - received, - }), + Ok(ValueSetInvocation::Numeric { value }) => ( + ActionOutput { + ok: true, + implemented: true, + action: "set_value".to_string(), + message: format!("AT-SPI numeric value set to {value}."), + received, + }, + None, + ), + Ok(ValueSetInvocation::EditableText { outcome }) => { + editable_text_action_result(outcome, received) + } + Err(error) => ( + ActionOutput { + ok: false, + implemented: true, + action: "set_value".to_string(), + message: error.to_string(), + received, + }, + None, + ), } } @@ -2156,7 +2314,7 @@ impl ComputerUseLinux { // The rmcp tool_handler macro only accepts a string literal here, so this // can't be env!("CARGO_PKG_VERSION"); the MCP safety check (CI) fails the // build if it drifts from the Cargo version. - version = "0.7.22", + version = "0.7.23", instructions = "Begin every turn that uses Computer Use by calling get_app_state. If diagnostics report disabled GNOME accessibility, call setup_accessibility before asking the user to retry. Use list_windows/focused_window before targeted keyboard input. If diagnostics report windowing.can_list_windows=false on GNOME, call setup_window_targeting to install the optional GNOME Shell extension backend, then ask the user to log out and back in if the setup report says a shell reload is required. This Linux backend can capture size-bounded screenshots through GNOME Shell or XDG Desktop Portal, read AT-SPI trees with action/value metadata, invoke native AT-SPI actions, set AT-SPI values or editable text, list/focus compositor windows through registered Linux window backends when the session permits it, attach best-effort terminal tty/process metadata to terminal windows, send coordinate or element-targeted click/scroll/drag input through the Wayland remote desktop portal when available, and send literal type_text through KDE clipboard integration on Plasma Wayland, wtype on compatible Wayland compositors, or portal keysyms on other Wayland sessions. Portal text startup or conversion failures return an error without replaying through ydotool. GNOME preflights the complete text against every configured keyboard layout group because the background reader cannot establish the active group; rejected text can use set_value on an editable field. Raw ydotool text is limited to printable ASCII, tab and newline, uses US physical key positions, and requires checking the resulting field contents under the active layout. Screenshot results include width/height for the returned image plus coordinate_width/coordinate_height and scale for desktop coordinate conversion; request more detail with max_width, max_height, max_bytes, format=jpeg, quality, or a smaller target/crop instead of relying on unbounded screenshots. Mutating tools (click, type_text, press_key, scroll, drag, perform_action, set_value, activate_window, move_window, resize_window) accept optional screenshot_after (true, \"screen\", or \"window\") with format/quality/scale/max_width/max_height/max_bytes and optional include_tree=true to return post-action observation in the same tool call. Tools with readOnlyHint=false may mutate local desktop or application state; hosts should require approval for actions that can submit, delete, send, purchase, or overwrite data. For element-targeted actions, prefer element_index from the latest get_app_state result; click, perform_action, and set_value can also use semantic role/name/text/states selectors when the target is unique. type_text and press_key accept optional window_id, pid, app_id, wm_class, title, tty, terminal_pid, terminal_command, or terminal_cwd selectors and refuse targeted input if focus cannot be verified. After targeted keyboard input, results append focused-element feedback from AT-SPI (role, name, editable) and warn when a concrete non-terminal focus is not editable. Exact window focus alone does not suppress a missing-focus warning; only a complete scoped tree containing window/menu chrome without content makes it informational. Terminals accept keyboard input without an EditableText interface. On Hyprland, wtype uses a single literal argument after -- to avoid stdin keymap-refresh corruption; text is visible in process arguments, so confirmed password fields are rejected (use set_value). Terminals and ambiguous fields, with or without AT-SPI, use private stdin for at most 100 Unicode characters per call; split larger input into smaller calls or expose accessibility. Input classification has its own 1500 ms budget, independent of the 250 ms cosmetic indicator timeout. Other compatible compositors retain stdin. The wtype path rejects NUL and more than 131071 UTF-8 bytes before focus or indicators. X11 xdotool can silently substitute symbols under non-US XKB layouts (reported with br/ABNT2) or changing keymap state; inspect the text before pressing Enter or submitting it. A successful type_text result confirms dispatch only; every text backend leaves field contents unverified. Re-observe the target before retrying. Screenshot, click, and input results warn when the target window or coordinate is partially or fully off-screen; use move_window/resize_window (GNOME Shell extension backend) to bring a window fully on-screen before retrying. scroll accepts the same window targeting and relative coordinates as click. get_app_state returns a compact readiness block by default; pass verbose=true for the full diagnostics dump. Scope get_app_state with app_name_or_bundle_identifier or a window target (window_id, pid, app_id, wm_class, title); without one it returns the whole desktop AT-SPI tree, reports tree_scoped=false, and warns in message, which can flood context. accessibility_tree_truncated=true means the node, depth, or read budget stopped traversal with unread elements left; recover by scoping to a narrower app or window target and raising max_nodes or max_depth (hard caps 2000 and 64), not by lowering max_nodes. Electron apps with only window/menu chrome may need --force-renderer-accessibility to expose renderer content." )] impl ServerHandler for ComputerUseLinux {} @@ -2561,7 +2719,7 @@ struct PostActionObservation { /// Post-action screenshot metadata when `screenshot_after` was requested; the image itself is the first `image` content block. #[serde(skip_serializing_if = "Option::is_none")] screenshot: Option, - /// Post-action screenshot capture error when the action succeeded but `screenshot_after` failed. + /// Post-action screenshot capture error when a requested `screenshot_after` capture failed. #[serde(skip_serializing_if = "Option::is_none")] screenshot_error: Option, /// Post-action compacted AT-SPI accessibility tree when `include_tree=true` was requested. @@ -2576,7 +2734,7 @@ struct PostActionObservation { /// True when the post-action accessibility tree hit traversal limits. #[serde(skip_serializing_if = "Option::is_none")] accessibility_tree_truncated: Option, - /// Post-action AT-SPI error when the action succeeded but `include_tree=true` failed. + /// Post-action AT-SPI error when a requested `include_tree=true` capture failed. #[serde(skip_serializing_if = "Option::is_none")] accessibility_error: Option, } @@ -3336,6 +3494,60 @@ struct ActionToolOutput { observation: PostActionObservation, } +#[derive(Debug, Clone, Serialize, JsonSchema)] +struct SetValueToolOutput { + #[serde(flatten)] + action: ActionOutput, + /// Exact editable-text readback result; omitted for numeric values and early failures. + #[serde(skip_serializing_if = "Option::is_none")] + verified: Option, + #[serde(flatten)] + observation: PostActionObservation, +} + +fn editable_text_action_result( + outcome: EditableTextSetOutcome, + received: Option, +) -> (ActionOutput, Option) { + let (ok, verified, message) = match outcome { + EditableTextSetOutcome::Verified => ( + true, + Some(true), + "AT-SPI editable text contents set and verified.", + ), + EditableTextSetOutcome::MismatchAfterDeadline => ( + false, + Some(false), + "AT-SPI EditableText returned success, but the resulting contents did not match the requested value. The contents may be unchanged, normalized, or partially changed; re-observe the element before retrying.", + ), + EditableTextSetOutcome::ReadbackFailed => ( + false, + Some(false), + "AT-SPI EditableText returned success, but content verification failed. The value may have changed; re-observe the element before retrying.", + ), + EditableTextSetOutcome::ReadbackTimedOut => ( + false, + Some(false), + "AT-SPI EditableText returned success, but content verification timed out. The value may have changed; re-observe the element before retrying.", + ), + EditableTextSetOutcome::PasswordUnverifiable => ( + true, + Some(false), + "AT-SPI editable text contents dispatched to a password field; exact contents cannot be verified.", + ), + }; + ( + ActionOutput { + ok, + implemented: true, + action: "set_value".to_string(), + message: message.to_string(), + received, + }, + verified, + ) +} + impl ComputerUseLinux { fn is_wayland_session(&self) -> bool { crate::diagnostics::hydrate_session_bus_env(); @@ -3566,21 +3778,21 @@ impl ComputerUseLinux { async fn resolve_window_context( &self, params: &GetAppStateParams, - ) -> (Option, Option, Option) { + ) -> (Option, Option, Option, bool) { let target = params.window_target(); if !target.has_target() { - return (None, None, None); + return (None, None, None, false); } match list_windows().await { Ok(windows) => match resolve_window_target(&windows, &target) { - Ok(window) => (Some(window.clone()), None, None), - Err(error) => (None, Some(format!("{error:#}")), None), + Ok(window) => (Some(window.clone()), None, None, true), + Err(error) => (None, Some(format!("{error:#}")), None, true), }, Err(error) => { let error = format!("{error:#}"); let hint = window_permission_hint(&error); - (None, Some(error), hint) + (None, Some(error), hint, false) } } } @@ -3899,6 +4111,36 @@ impl ComputerUseLinux { ) } + async fn finish_set_value_output( + &self, + mut output: ActionOutput, + verified: Option, + capture: &PostActionCaptureParams, + ) -> Result { + if !output.ok && verified != Some(false) { + return structured_tool_result( + SetValueToolOutput { + action: output, + verified, + observation: PostActionObservation::default(), + }, + None, + ); + } + let (observation, screenshot, notes) = self + .capture_post_action_observation(capture, None, None) + .await; + output = with_notes(output, notes); + structured_tool_result( + SetValueToolOutput { + action: output, + verified, + observation, + }, + screenshot, + ) + } + async fn finish_activate_window_output( &self, output: ActivateWindowOutput, @@ -4104,24 +4346,84 @@ impl ComputerUseLinux { &self, params: &GetAppStateParams, window_context: Option<&WindowInfo>, + window_enumerated: bool, ) -> Option { + self.resolve_accessibility_app_filter_with( + params, + window_context, + window_enumerated, + || list_accessible_apps(200), + ) + .await + } + + async fn resolve_accessibility_app_filter_with( + &self, + params: &GetAppStateParams, + window_context: Option<&WindowInfo>, + window_enumerated: bool, + list_apps: F, + ) -> Option + where + F: FnOnce() -> Fut, + Fut: Future>>, + { + if let Some(app_id) = trimmed_nonempty(params.app_id.as_deref()) { + // A failed lookup against a successfully enumerated desktop is + // authoritative. Only an unavailable backend permits an AT-SPI-only + // fallback. Higher-priority window selectors must not contradict + // the explicit app identity either. + let rejected = window_enumerated && window_context.is_none(); + let contradictory = window_context.is_some_and(|window| { + !window.app_id.as_deref().is_some_and(|actual| { + actual.eq_ignore_ascii_case(app_id) + || (!app_id.contains('.') + && crate::windows::app_id_leaf(actual).eq_ignore_ascii_case(app_id)) + }) + }); + if rejected || contradictory { + return Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string()); + } + } + if let Some(explicit) = trimmed_nonempty(params.app_name_or_bundle_identifier.as_deref()) { return Some(explicit.to_string()); } let target_pid = window_context.and_then(|window| window.pid).or(params.pid); let candidates = accessibility_filter_candidates(window_context); + let app_id = trimmed_nonempty(params.app_id.as_deref()).map(|requested| { + window_context + .and_then(|window| trimmed_nonempty(window.app_id.as_deref())) + .unwrap_or(requested) + }); + let needs_app_id_evidence = app_id.is_some_and(|app_id| { + !app_id.starts_with("window:") && !crate::windows::app_id_leaf(app_id).is_empty() + }); + let apps = if target_pid.is_some() || needs_app_id_evidence { + list_apps().await.ok() + } else { + None + }; if let Some(target_pid) = target_pid { - if let Ok(apps) = list_accessible_apps(200).await { - if let Some(object_ref) = - select_accessibility_object_ref(&apps, target_pid, &candidates) - { - return Some(object_ref); - } + if let Some(object_ref) = apps + .as_deref() + .and_then(|apps| select_accessibility_object_ref(apps, target_pid, &candidates)) + { + return Some(object_ref); } } + if let Some(app_id) = app_id { + if !app_id.starts_with("window:") { + return Some(select_accessibility_app_id_filter(apps.as_deref(), app_id)); + } + } + + if candidates.is_empty() && trimmed_nonempty(params.app_id.as_deref()).is_some() { + return Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string()); + } candidates.into_iter().next() } @@ -4456,11 +4758,49 @@ impl ComputerUseLinux { self.cache_snapshot(nodes, target_pid); } + async fn commit_app_state_snapshot( + &self, + nodes: &[AccessibilityNode], + target_pid: Option, + window: Option<&WindowInfo>, + params: &GetAppStateParams, + scoped: bool, + ) { + let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await; + let recovery_scope = scoped.then(|| AccessibilityRecoveryScope { + params: params.clone(), + target_pid, + }); + self.cache_snapshot_with_window(nodes, target_pid, recovery_scope, window); + } + fn cache_snapshot(&self, nodes: &[AccessibilityNode], target_pid: Option) { + self.cache_snapshot_with_scope(nodes, target_pid, None); + } + + fn cache_snapshot_with_scope( + &self, + nodes: &[AccessibilityNode], + target_pid: Option, + recovery_scope: Option, + ) { + self.cache_snapshot_with_window(nodes, target_pid, recovery_scope, None); + } + + fn cache_snapshot_with_window( + &self, + nodes: &[AccessibilityNode], + target_pid: Option, + recovery_scope: Option, + window: Option<&WindowInfo>, + ) { if let Ok(mut cached) = self.last_snapshot.lock() { *cached = CachedAccessibilitySnapshot { nodes: nodes.to_vec(), pid: target_pid, + window_id: window.map(|window| window.window_id), + window_backend: window.map(|window| window.backend.clone()), + recovery_scope, }; } } @@ -4535,15 +4875,142 @@ impl ComputerUseLinux { } } - fn resolve_click_target( + fn should_map_kwin_element_click(&self, focus: Option<&WindowFocusResult>) -> bool { + let cached_backend = self + .last_snapshot + .lock() + .ok() + .and_then(|snapshot| snapshot.window_backend.clone()); + kwin_element_adapter_required( + self.is_wayland_session(), + self.is_kde_wayland_session(), + cached_backend.as_deref(), + focus.map(|focus| focus.requested_window.backend.as_str()), + ) + } + + async fn prepare_kwin_element_click( &self, - params: &ClickParams, - ) -> std::result::Result { - if let Some((x, y)) = params.x.zip(params.y) { - return Ok(ClickTarget::Coordinates(x, y)); + node: &AccessibilityNode, + center: (i32, i32), + focus: Option<&WindowFocusResult>, + ) -> std::result::Result { + let mut target = self.cached_kwin_element_target(node, center, focus)?; + if target.pid.is_none() { + target.pid = Some( + object_ref_owner_pid(&node.object_ref) + .await + .map_err(|error| format!("could not identify the cached element's root PID: {error:#}"))? + .ok_or_else(|| { + "cached element's AT-SPI root PID is unavailable, so its KWin window identity cannot be verified; retry get_app_state with window_id".to_string() + })?, + ); + } + let snapshot = crate::windowing::backends::kwin::element_click_snapshot() + .await + .map_err(|error| format!("could not obtain a coherent KWin snapshot: {error:#}"))?; + let raw = capture_screenshot_raw().await.map_err(|error| { + format!("could not obtain a current raw desktop capture: {error:#}") + })?; + if raw.width == 0 + || raw.height == 0 + || raw.width > i32::MAX as u32 + || raw.height > i32::MAX as u32 + { + return Err(format!( + "current raw desktop capture has unusable dimensions {}x{}", + raw.width, raw.height + )); } + let point = snapshot + .map_element_click(&target, raw.width, raw.height) + .map_err(|error| format!("{error:#}"))?; + self.cache_desktop_size(raw.width, raw.height); + Ok(PreparedKwinElementClick { + element_index: node.index, + point, + capture_size: (raw.width, raw.height), + }) + } - let selector = params.selector(); + fn cached_kwin_element_target( + &self, + node: &AccessibilityNode, + center: (i32, i32), + focus: Option<&WindowFocusResult>, + ) -> std::result::Result { + let cached = self + .last_snapshot + .lock() + .map_err(|_| "cached accessibility snapshot is unavailable".to_string())?; + let window_id = if let Some(focus) = focus { + let requested_id = focus.requested_window.window_id; + let exact = focus.exact_window_focused + && focus + .focused_window + .as_ref() + .is_some_and(|window| window.window_id == requested_id); + if !exact { + return Err( + "explicit window targeting requires exact KWin window focus, not app-only focus" + .to_string(), + ); + } + if let Some(cached_window_id) = cached.window_id { + if cached_window_id != requested_id { + return Err(format!( + "cached element belongs to window_id {cached_window_id}, not the explicitly targeted window_id {requested_id}" + )); + } + } + Some(requested_id) + } else { + cached.window_id + }; + let frame = depth_one_kwin_element_frame(&cached.nodes, node, center)?; + Ok(KwinElementClickTarget { + window_id, + pid: cached.pid, + frame, + center, + require_exact_focus: focus.is_some(), + }) + } + + fn kwin_element_click_failure( + &self, + element_index: u32, + detail: &str, + received: Option, + ) -> Json { + let cached_window_id = self + .last_snapshot + .lock() + .ok() + .and_then(|snapshot| snapshot.window_id); + let window = cached_window_id + .map(|window_id| format!("window_id {window_id}")) + .unwrap_or_else(|| "window_id unavailable".to_string()); + Json(ActionOutput { + ok: false, + implemented: true, + action: "click".to_string(), + message: format!( + "click was not sent: KWin element coordinate mapping failed for element_index {element_index} ({window}): {detail}. No pointer event was emitted. Call get_app_state with the intended window_id and a full-desktop screenshot, then retry with the refreshed element_index, or pass explicit screenshot x/y coordinates." + ), + received, + }) + } + + fn resolve_click_target( + &self, + params: &ClickParams, + ) -> std::result::Result { + if let Some((x, y)) = params.x.zip(params.y) { + return Ok(ClickTarget::Coordinates(x, y)); + } + + let selector = params.selector(); let node = self.resolve_cached_node( params.element_index, &selector, @@ -4565,8 +5032,11 @@ impl ComputerUseLinux { } } - if let Some((x, y)) = node.bounds.as_ref().and_then(bounds_center) { - return Ok(ClickTarget::Coordinates(x, y)); + if let Some(center) = node.bounds.as_ref().and_then(bounds_center) { + return Ok(ClickTarget::ElementBounds { + node: Box::new(node), + center, + }); } if !is_plain_left_click(params.button.as_deref(), params.click_count) { @@ -4641,11 +5111,64 @@ impl ComputerUseLinux { resolve_semantic_node(cached.nodes.as_slice(), selector, purpose) } + async fn refresh_action_snapshot( + &self, + scope: &AccessibilityRecoveryScope, + ) -> Result { + let params = &scope.params; + let (window_context, _, _, window_enumerated) = self.resolve_window_context(params).await; + let app_filter = self + .resolve_accessibility_app_filter(params, window_context.as_ref(), window_enumerated) + .await; + let target_pid = action_recovery_target_pid( + scope, + window_context.as_ref().and_then(|window| window.pid), + ); + let (max_nodes, max_depth) = + crate::atspi_tree::snapshot_limits(params.max_nodes, params.max_depth); + let snapshot = snapshot_accessibility_tree( + app_filter.as_deref(), + accessibility_snapshot_target_pid(app_filter.as_deref(), target_pid), + max_nodes, + max_depth, + ) + .await?; + Ok(RefreshedAccessibilitySnapshot { + snapshot, + target_pid, + window: window_context, + }) + } + async fn perform_element_action( &self, params: &ActionParams, requested_action: Option<&str>, ) -> Json { + self.perform_element_action_with( + params, + requested_action, + |object_ref, requested_action| async move { + invoke_accessibility_action(&object_ref, requested_action.as_deref()).await + }, + |scope| async move { self.refresh_action_snapshot(&scope).await }, + ) + .await + } + + async fn perform_element_action_with( + &self, + params: &ActionParams, + requested_action: Option<&str>, + mut invoke: I, + refresh: R, + ) -> Json + where + I: FnMut(String, Option) -> IFut, + IFut: Future>, + R: FnOnce(AccessibilityRecoveryScope) -> RFut, + RFut: Future>, + { let _input_guard = Arc::clone(&self.input_operation_lock).lock_owned().await; let received = Some(serde_json::json!(params.clone())); let object_ref = match self.resolve_object_ref( @@ -4655,60 +5178,307 @@ impl ComputerUseLinux { ElementResolvePurpose::Action, ) { Ok(object_ref) => object_ref, - Err(message) => { - return Json(ActionOutput { - ok: false, - implemented: true, - action: "perform_action".to_string(), - message, - received, - }); - } + Err(message) => return action_error_output(message, &received), }; self.indicator.action("perform_action").await; - match invoke_accessibility_action(&object_ref, requested_action).await { - Ok(invocation) => Json(ActionOutput { - ok: invocation.ok, - implemented: true, - action: "perform_action".to_string(), - message: if invocation.ok { + let error = match invoke(object_ref.clone(), requested_action.map(str::to_string)).await { + Ok(invocation) => return action_invocation_output(invocation, &received), + Err(error) => error, + }; + + let semantic_recovery = params.element_index.is_none() + && trimmed_nonempty(params.element_identifier.as_deref()).is_none() + && !params.selector().is_empty(); + if !semantic_recovery || !is_recoverable_action_query_error(&error) { + return action_error_output(error.to_string(), &received); + } + + let original_error = format!("{error:#}"); + let recovery_scope = self + .last_snapshot + .lock() + .ok() + .and_then(|snapshot| snapshot.recovery_scope.clone()); + let Some(recovery_scope) = recovery_scope else { + return action_error_output( + format!( + "{original_error}. The cached snapshot has no stable scoped recovery target. Call get_app_state for the target app, then retry perform_action." + ), + &received, + ); + }; + + let refreshed = match refresh(recovery_scope.clone()).await { + Ok(refreshed) => refreshed, + Err(refresh_error) => { + return action_error_output( format!( - "AT-SPI action {} ({}) invoked.", - invocation.action_index, - invocation - .action_name - .as_deref() - .filter(|name| !name.is_empty()) - .unwrap_or("unnamed") - ) - } else { + "{original_error}. Automatic AT-SPI target recovery could not refresh the original scoped snapshot: {refresh_error:#}. Call get_app_state for the target app, then retry perform_action." + ), + &received, + ); + } + }; + let snapshot = refreshed.snapshot; + let pid_scope_matches = refreshed + .target_pid + .is_none_or(|pid| snapshot.root_pid == Some(pid)); + let window_scope_matches = if recovery_scope.params.window_id.is_some() { + let original_window_id = self + .last_snapshot + .lock() + .ok() + .and_then(|cached| cached.window_id); + // The resolver accepts JSON-rounded IDs. Compare the actual + // observed identity when available, not the raw requested number. + refreshed.window.as_ref().is_some_and(|window| { + original_window_id.is_none_or(|window_id| window.window_id == window_id) + }) + } else { + true + }; + if !snapshot.scoped + || !snapshot.app_filter_matched + || !pid_scope_matches + || !window_scope_matches + { + return action_error_output( + format!( + "{original_error}. Automatic AT-SPI target recovery could not prove that the refreshed tree has the original app scope and any explicit window_id target. Call get_app_state for the target app, then retry perform_action." + ), + &received, + ); + } + + let root_pid = snapshot.root_pid; + let (nodes, _) = compact_accessibility_tree_with_note( + snapshot.nodes, + snapshot.scoped, + snapshot.truncated, + ); + let mut recovery_scope = recovery_scope; + recovery_scope.target_pid = root_pid; + self.cache_snapshot_with_window( + &nodes, + root_pid, + Some(recovery_scope), + refreshed.window.as_ref(), + ); + let recovered = match self.resolve_cached_node( + None, + ¶ms.selector(), + ElementResolvePurpose::Action, + ) { + Ok(node) => node, + Err(message) => { + return action_error_output( format!( - "AT-SPI action {} ({}) returned false.", - invocation.action_index, - invocation - .action_name - .as_deref() - .filter(|name| !name.is_empty()) - .unwrap_or("unnamed") - ) - }, - received, - }), - Err(error) => Json(ActionOutput { - ok: false, - implemented: true, - action: "perform_action".to_string(), - message: error.to_string(), - received, - }), + "{original_error}. AT-SPI target recovery refreshed the scoped snapshot but could not re-resolve the semantic selector: {message}. Call get_app_state for the target app and choose the element again." + ), + &received, + ); + } + }; + if recovered.object_ref == object_ref { + return action_error_output( + format!( + "{original_error}. AT-SPI target recovery resolved the same dead object identity. Call get_app_state for the target app, then retry perform_action." + ), + &received, + ); + } + + match invoke(recovered.object_ref, requested_action.map(str::to_string)).await { + Ok(invocation) => action_invocation_output(invocation, &received), + Err(error) => action_error_output(error.to_string(), &received), + } + } +} + +fn action_recovery_target_pid( + scope: &AccessibilityRecoveryScope, + resolved_window_pid: Option, +) -> Option { + resolved_window_pid + .or(scope.params.pid) + .or(scope.target_pid) +} + +fn is_recoverable_action_query_error(error: &anyhow::Error) -> bool { + if error.chain().any(|cause| { + cause + .to_string() + .starts_with("failed to invoke AT-SPI action ") + }) { + return false; + } + + error.chain().any(|cause| { + if let Some(zbus::Error::MethodError(name, _, _)) = cause.downcast_ref::() { + if matches!( + name.as_str(), + "org.freedesktop.DBus.Error.ServiceUnknown" + | "org.freedesktop.DBus.Error.NameHasNoOwner" + ) { + return true; + } + } + if matches!( + cause.downcast_ref::(), + Some(zbus::fdo::Error::ServiceUnknown(_) | zbus::fdo::Error::NameHasNoOwner(_)) + ) { + return true; + } + let message = cause.to_string(); + message.contains("ServiceUnknown") || message.contains("NameHasNoOwner") + }) +} + +fn action_error_output( + message: String, + received: &Option, +) -> Json { + Json(ActionOutput { + ok: false, + implemented: true, + action: "perform_action".to_string(), + message, + received: received.clone(), + }) +} + +fn action_invocation_output( + invocation: crate::atspi_tree_impl::ActionInvocation, + received: &Option, +) -> Json { + Json(ActionOutput { + ok: invocation.ok, + implemented: true, + action: "perform_action".to_string(), + message: if invocation.ok { + format!( + "AT-SPI action {} ({}) invoked.", + invocation.action_index, + invocation + .action_name + .as_deref() + .filter(|name| !name.is_empty()) + .unwrap_or("unnamed") + ) + } else { + format!( + "AT-SPI action {} ({}) returned false.", + invocation.action_index, + invocation + .action_name + .as_deref() + .filter(|name| !name.is_empty()) + .unwrap_or("unnamed") + ) + }, + received: received.clone(), + }) +} + +#[derive(Debug, Clone, Copy)] +struct PreparedKwinElementClick { + element_index: u32, + point: KwinElementClickPoint, + capture_size: (u32, u32), +} + +impl PreparedKwinElementClick { + fn uinput_point(self) -> (i32, i32) { + self.point.capture + } + + fn portal_point(self) -> (i32, i32) { + self.point.logical + } +} + +fn kwin_element_adapter_required( + is_wayland: bool, + is_kde_wayland: bool, + cached_backend: Option<&str>, + explicit_backend: Option<&str>, +) -> bool { + if !is_wayland { + return false; + } + if let Some(backend) = explicit_backend { + return backend == KWIN_BACKEND; + } + cached_backend.map_or(is_kde_wayland, |backend| backend == KWIN_BACKEND) +} + +fn depth_one_kwin_element_frame( + nodes: &[AccessibilityNode], + node: &AccessibilityNode, + center: (i32, i32), +) -> std::result::Result { + let mut parent_index = node.parent_index; + let mut visited = Vec::new(); + while let Some(index) = parent_index { + if visited.contains(&index) { + return Err("cached element ancestry contains a cycle".to_string()); + } + visited.push(index); + let parent = nodes + .iter() + .find(|candidate| candidate.index == index) + .ok_or_else(|| { + "cached element is missing its depth-1 frame/window ancestry".to_string() + })?; + if parent.depth == 1 { + if !matches!( + parent.role.trim().to_ascii_lowercase().as_str(), + "frame" | "window" | "dialog" | "alert" + ) { + return Err( + "cached element's depth-1 ancestor is not a frame/window/dialog/alert" + .to_string(), + ); + } + let bounds = parent + .bounds + .as_ref() + .ok_or_else(|| "cached element's depth-1 frame/window has no bounds".to_string())?; + bounds_center(bounds).ok_or_else(|| { + "cached element's depth-1 frame/window bounds are unusable".to_string() + })?; + let right = i64::from(bounds.x) + i64::from(bounds.width); + let bottom = i64::from(bounds.y) + i64::from(bounds.height); + if i64::from(center.0) < i64::from(bounds.x) + || i64::from(center.0) >= right + || i64::from(center.1) < i64::from(bounds.y) + || i64::from(center.1) >= bottom + { + return Err("element center is outside its depth-1 AT-SPI frame/window".to_string()); + } + return Ok(KwinElementFrame { + x: bounds.x, + y: bounds.y, + width: bounds.width, + height: bounds.height, + }); + } + if parent.depth == 0 { + break; } + parent_index = parent.parent_index; } + Err("cached element has no depth-1 frame/window ancestor".to_string()) } #[derive(Debug)] enum ClickTarget { Coordinates(i32, i32), + ElementBounds { + node: Box, + center: (i32, i32), + }, PrimaryAction { object_ref: String, action_name: Option, @@ -5113,6 +5883,41 @@ fn is_sentinel_or_missing_bounds(bounds: Option<&Bounds>) -> bool { bounds.is_none() } +fn select_accessibility_app_id_filter( + apps: Option<&[AccessibleAppSummary]>, + app_id: &str, +) -> String { + let leaf = crate::windows::app_id_leaf(app_id); + let Some(apps) = apps.filter(|_| !leaf.is_empty()) else { + return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string(); + }; + let mut matches = apps.iter().filter(|app| { + app.name + .as_deref() + .is_some_and(|name| crate::windows::app_id_leaf(name).eq_ignore_ascii_case(leaf)) + }); + let Some(first) = matches.next() else { + return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string(); + }; + if matches.next().is_some() { + return UNMATCHABLE_ACCESSIBILITY_APP_FILTER.to_string(); + } + first.object_ref.clone() +} + +fn accessibility_snapshot_target_pid( + app_filter: Option<&str>, + target_pid: Option, +) -> Option { + // select_roots normally falls back to PID-only matches. A rejected app ID + // must not regain that window's tree through this fallback. + if app_filter == Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER) { + None + } else { + target_pid + } +} + fn select_accessibility_object_ref( apps: &[AccessibleAppSummary], target_pid: u32, @@ -5163,6 +5968,7 @@ fn accessibility_filter_candidates(window_context: Option<&WindowInfo>) -> Vec, + backend: &str, +) -> ActionOutput { + if output.ok { + if let Some(kwin_click) = kwin_click { + output.message = format!( + "Click dispatched through {backend} using KWin element coordinate mapping: capture {},{}; portal logical {},{}. Dispatch only: UI activation was not verified.", + kwin_click.point.capture.0, + kwin_click.point.capture.1, + kwin_click.point.logical.0, + kwin_click.point.logical.1 + ); + } + } + output +} + fn portal_action_error( action: &str, error: anyhow::Error, @@ -8176,46 +9001,234 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" } #[test] - fn select_accessibility_object_ref_prefers_exact_pid_match() { + fn accessibility_filter_candidates_include_reverse_dns_leaf() { + let window = window_info( + 42, + Some("Untitled — KWrite"), + Some("org.kde.kwrite"), + Some("kwrite"), + None, + ); + + let candidates = accessibility_filter_candidates(Some(&window)); + + assert!(candidates.contains(&"kwrite".to_string())); + assert!(!candidates.contains(&"desktop".to_string())); + } + + fn accessible_app(object_ref: &str, name: &str) -> AccessibleAppSummary { + AccessibleAppSummary { + object_ref: object_ref.to_string(), + name: Some(name.to_string()), + pid: None, + role: "application".to_string(), + child_count: 1, + bounds: None, + } + } + + #[test] + fn explicit_app_id_unique_root_uses_current_object_ref() { let apps = vec![ - AccessibleAppSummary { - object_ref: ":1.31/org/a11y/atspi/accessible/root".to_string(), - name: Some("electron".to_string()), - pid: Some(2774076), - role: "application".to_string(), - child_count: 1, - bounds: None, - }, - AccessibleAppSummary { - object_ref: ":1.64/org/a11y/atspi/accessible/root".to_string(), - name: Some("cu_atspi_gtk_test.py".to_string()), - pid: Some(2914326), - role: "application".to_string(), - child_count: 1, - bounds: None, - }, + accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"), + accessible_app(":1.41/org/a11y/atspi/accessible/root", "Kate"), ]; - let object_ref = select_accessibility_object_ref( - &apps, - 2914326, - &[ - "CU ATSPI GTK Test".to_string(), - "cu_atspi_gtk_test.py".to_string(), - ], - ) - .unwrap(); - - assert_eq!(object_ref, ":1.64/org/a11y/atspi/accessible/root"); + for app_id in ["kwrite", "org.kde.kwrite"] { + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), app_id), + ":1.40/org/a11y/atspi/accessible/root" + ); + } } #[test] - fn compact_accessibility_tree_reparents_actionable_descendants() { - let nodes = vec![ - AccessibilityNode { - index: 0, - parent_index: None, - depth: 0, + fn explicit_app_id_missing_root_or_list_failure_is_unmatchable() { + let apps = vec![accessible_app( + ":1.41/org/a11y/atspi/accessible/root", + "Kate", + )]; + + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), "kwrite"), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + assert_eq!( + select_accessibility_app_id_filter(None, "kwrite"), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + for app_id in ["window:46", ".desktop"] { + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), app_id), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + } + } + + #[test] + fn explicit_app_id_ambiguous_leaf_is_unmatchable() { + let apps = vec![ + accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"), + accessible_app(":1.42/org/a11y/atspi/accessible/root", "com.example.kwrite"), + ]; + + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), "kwrite"), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + } + + #[test] + fn explicit_full_app_id_cannot_prove_ambiguous_root_namespaces() { + let apps = vec![ + accessible_app(":1.40/org/a11y/atspi/accessible/root", "KWrite"), + accessible_app(":1.42/org/a11y/atspi/accessible/root", "kwrite"), + ]; + + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), "org.kde.kwrite"), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + } + + #[tokio::test] + async fn app_id_authoritative_window_rejection_cannot_use_accessibility_fallback() { + let backend = ComputerUseLinux::default(); + let windows = vec![ + window_info(1, Some("KWrite"), Some("org.kde.kwrite"), None, None), + window_info(2, Some("Other"), Some("com.example.kwrite"), None, None), + ]; + for app_id in ["kwrite", "org.missing.kwrite"] { + let params: GetAppStateParams = + serde_json::from_value(serde_json::json!({ "app_id": app_id })).unwrap(); + let target = params.window_target(); + assert!(resolve_window_target(&windows, &target).is_err()); + let filter = backend + .resolve_accessibility_app_filter_with(¶ms, None, true, || async { + Ok(vec![accessible_app(":1.40/root", "KWrite")]) + }) + .await; + assert_eq!( + filter.as_deref(), + Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER), + "{app_id}" + ); + } + } + + #[tokio::test] + async fn app_id_unavailable_window_backend_preserves_exact_accessibility_fallback() { + let backend = ComputerUseLinux::default(); + for app_id in ["KWRITE", "org.kde.kwrite"] { + let params: GetAppStateParams = + serde_json::from_value(serde_json::json!({ "app_id": app_id })).unwrap(); + let filter = backend + .resolve_accessibility_app_filter_with(¶ms, None, false, || async { + Ok(vec![accessible_app(":1.40/root", "KWrite")]) + }) + .await; + assert_eq!(filter.as_deref(), Some(":1.40/root")); + } + } + + #[tokio::test] + async fn app_id_conflicting_resolved_window_cannot_use_pid_or_explicit_name() { + let backend = ComputerUseLinux::default(); + let window = window_info(1, Some("Kate"), Some("org.kde.kate"), None, Some(123)); + for explicit_name in [None, Some("Kate")] { + let params: GetAppStateParams = serde_json::from_value(serde_json::json!({ + "window_id": 1, + "app_id": "org.kde.kwrite", + "app_name_or_bundle_identifier": explicit_name, + })) + .unwrap(); + assert!( + resolve_window_target(std::slice::from_ref(&window), ¶ms.window_target()) + .is_ok() + ); + let filter = backend + .resolve_accessibility_app_filter_with(¶ms, Some(&window), true, || async { + let mut app = accessible_app(":1.41/root", "Kate"); + app.pid = Some(123); + Ok(vec![app]) + }) + .await; + assert_eq!( + filter.as_deref(), + Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER) + ); + } + } + + #[test] + fn app_id_accessibility_fallback_rejects_arbitrary_name_substrings() { + let apps = vec![accessible_app(":1.40/root", "KWrite")]; + assert_eq!( + select_accessibility_app_id_filter(Some(&apps), "write"), + UNMATCHABLE_ACCESSIBILITY_APP_FILTER + ); + } + + #[test] + fn app_id_rejection_cannot_regain_a_pid_only_snapshot() { + assert_eq!( + accessibility_snapshot_target_pid( + Some(UNMATCHABLE_ACCESSIBILITY_APP_FILTER), + Some(123) + ), + None + ); + assert_eq!( + accessibility_snapshot_target_pid(Some(":1.40/root"), Some(123)), + Some(123) + ); + assert_eq!( + accessibility_snapshot_target_pid(None, Some(123)), + Some(123) + ); + } + + #[test] + fn select_accessibility_object_ref_prefers_exact_pid_match() { + let apps = vec![ + AccessibleAppSummary { + object_ref: ":1.31/org/a11y/atspi/accessible/root".to_string(), + name: Some("electron".to_string()), + pid: Some(2774076), + role: "application".to_string(), + child_count: 1, + bounds: None, + }, + AccessibleAppSummary { + object_ref: ":1.64/org/a11y/atspi/accessible/root".to_string(), + name: Some("cu_atspi_gtk_test.py".to_string()), + pid: Some(2914326), + role: "application".to_string(), + child_count: 1, + bounds: None, + }, + ]; + + let object_ref = select_accessibility_object_ref( + &apps, + 2914326, + &[ + "CU ATSPI GTK Test".to_string(), + "cu_atspi_gtk_test.py".to_string(), + ], + ) + .unwrap(); + + assert_eq!(object_ref, ":1.64/org/a11y/atspi/accessible/root"); + } + + #[test] + fn compact_accessibility_tree_reparents_actionable_descendants() { + let nodes = vec![ + AccessibilityNode { + index: 0, + parent_index: None, + depth: 0, object_ref: ":1.0/root".to_string(), role: "application".to_string(), name: Some("demo-app".to_string()), @@ -8731,7 +9744,7 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" assert_eq!(action_name.as_deref(), Some("Click")); assert_eq!(action_index, 0); } - ClickTarget::Coordinates(_, _) => { + ClickTarget::Coordinates(_, _) | ClickTarget::ElementBounds { .. } => { panic!("expected AT-SPI primary-action fallback") } } @@ -8787,10 +9800,14 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" ..Default::default() }, ] { - assert!(matches!( - backend.resolve_click_target(¶ms).unwrap(), - ClickTarget::Coordinates(_, _) - )); + let explicit = params.x.is_some(); + match backend.resolve_click_target(¶ms).unwrap() { + ClickTarget::Coordinates(470, 408) if explicit => {} + ClickTarget::ElementBounds { center, .. } if !explicit => { + assert_eq!(center, (x + 160, y + 60)); + } + _ => panic!("unexpected click target"), + } } } } @@ -8820,10 +9837,239 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" ..Default::default() }) .unwrap(), - ClickTarget::Coordinates(20, 30) + ClickTarget::ElementBounds { + center: (20, 30), + .. + } + )); + } + + fn kwin_cached_element_nodes() -> Vec { + let mut app = node(0, None); + app.role = "application".to_string(); + let mut frame = node( + 1, + Some(Bounds { + x: 0, + y: 0, + width: 800, + height: 600, + }), + ); + frame.parent_index = Some(0); + frame.depth = 1; + frame.role = "frame".to_string(); + let mut menu = node( + 2, + Some(Bounds { + x: 0, + y: 0, + width: 64, + height: 33, + }), + ); + menu.parent_index = Some(1); + menu.depth = 2; + menu.role = "menu".to_string(); + vec![app, frame, menu] + } + + fn kwin_window(window_id: u64) -> WindowInfo { + let mut window = window_info( + window_id, + Some("Editor"), + Some("editor"), + Some("editor"), + Some(4242), + ); + window.backend = KWIN_BACKEND.to_string(); + window + } + + #[test] + fn kwin_cached_dialog_and_alert_ancestors_keep_window_identity() { + for role in ["frame", "window", "dialog", "alert"] { + let backend = ComputerUseLinux::default(); + let mut nodes = kwin_cached_element_nodes(); + nodes[1].role = role.to_string(); + backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42))); + let target = backend + .cached_kwin_element_target(&nodes[2], (32, 16), None) + .unwrap_or_else(|error| panic!("{role}: {error}")); + assert_eq!(target.window_id, Some(42)); + assert_eq!(target.pid, Some(4242)); + assert_eq!((target.frame.width, target.frame.height), (800, 600)); + } + let mut nodes = kwin_cached_element_nodes(); + nodes[1].role = "panel".to_string(); + assert!(depth_one_kwin_element_frame(&nodes, &nodes[2], (32, 16)).is_err()); + } + + #[test] + fn kwin_cached_element_requires_depth_one_frame_and_exact_window_identity() { + let backend = ComputerUseLinux::default(); + let nodes = kwin_cached_element_nodes(); + backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42))); + + let target = backend + .cached_kwin_element_target(&nodes[2], (32, 16), None) + .unwrap(); + assert_eq!(target.window_id, Some(42)); + assert_eq!(target.pid, Some(4242)); + assert_eq!((target.frame.x, target.frame.y), (0, 0)); + assert_eq!((target.frame.width, target.frame.height), (800, 600)); + + let missing = node(7, nodes[2].bounds.clone()); + let error = depth_one_kwin_element_frame(&nodes, &missing, (32, 16)).unwrap_err(); + assert!(error.contains("depth-1"), "{error}"); + let error = depth_one_kwin_element_frame(&nodes, &nodes[2], (900, 16)).unwrap_err(); + assert!(error.contains("outside"), "{error}"); + + let requested = kwin_window(42); + let focused = kwin_window(43); + let app_only_focus = WindowFocusResult { + requested_window: requested, + focused_window: Some(focused), + exact_window_focused: false, + app_focused: true, + backend: KWIN_BACKEND.to_string(), + note: String::new(), + }; + let error = backend + .cached_kwin_element_target(&nodes[2], (32, 16), Some(&app_only_focus)) + .unwrap_err(); + assert!(error.contains("not app-only focus"), "{error}"); + } + + #[test] + fn kwin_adapter_bypasses_explicit_relative_native_and_non_kwin_paths() { + assert!(!kwin_element_adapter_required( + false, + true, + Some(KWIN_BACKEND), + None + )); + assert!(!kwin_element_adapter_required( + true, + false, + Some(GNOME_SHELL_EXTENSION_BACKEND), + None + )); + assert!(!kwin_element_adapter_required( + true, + true, + Some(KWIN_BACKEND), + Some(GNOME_SHELL_EXTENSION_BACKEND) + )); + assert!(kwin_element_adapter_required( + true, + false, + Some(KWIN_BACKEND), + None + )); + assert!(kwin_element_adapter_required(true, true, None, None)); + + let backend = ComputerUseLinux::default(); + let mut nodes = kwin_cached_element_nodes(); + nodes[2].actions = vec![click_action()]; + backend.cache_snapshot_with_window(&nodes, Some(4242), None, Some(&kwin_window(42))); + assert!(matches!( + backend.resolve_click_target(&ClickParams { + element_index: Some(2), + ..Default::default() + }), + Ok(ClickTarget::PrimaryAction { .. }) + )); + assert!(matches!( + backend.resolve_click_target(&ClickParams { + element_index: Some(2), + x: Some(470), + y: Some(408), + ..Default::default() + }), + Ok(ClickTarget::Coordinates(470, 408)) + )); + + let mut relative = ClickParams { + element_index: Some(2), + x: Some(7), + y: Some(9), + relative: Some(true), + window_id: Some(42), + ..Default::default() + }; + apply_window_relative_click_coordinates(&mut relative, (133, 267, 1067, 800)).unwrap(); + assert!(matches!( + backend.resolve_click_target(&relative), + Ok(ClickTarget::Coordinates(140, 276)) )); } + #[test] + fn kwin_prepared_click_routes_capture_to_uinput_and_logical_to_portal() { + let prepared = PreparedKwinElementClick { + element_index: 2, + point: KwinElementClickPoint { + logical: (278, 352), + capture: (417, 528), + }, + capture_size: (3840, 2160), + }; + assert_eq!(prepared.uinput_point(), (417, 528)); + assert_eq!(prepared.portal_point(), (278, 352)); + + let output = kwin_element_click_dispatch_output( + ActionOutput { + ok: true, + implemented: true, + action: "click".to_string(), + message: String::new(), + received: None, + }, + Some(prepared), + "the test pointer", + ); + assert!(output.ok); + assert!(output.message.contains("capture 417,528")); + assert!(output.message.contains("portal logical 278,352")); + assert!(output.message.contains("Dispatch only")); + assert!(output.message.contains("UI activation was not verified")); + } + + #[test] + fn kwin_adapter_failures_are_not_sent_and_include_recovery_guidance() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_window( + &kwin_cached_element_nodes(), + Some(4242), + None, + Some(&kwin_window(42)), + ); + let output = backend + .kwin_element_click_failure(2, "stale absolute-pointer geometry", None) + .0; + assert!(!output.ok); + assert!(output.message.contains("click was not sent")); + assert!(output.message.contains("No pointer event was emitted")); + assert!(output.message.contains("window_id 42")); + assert!(output.message.contains("screenshot x/y")); + } + + #[tokio::test] + async fn kwin_stale_pointer_recreation_failure_is_fail_closed() { + let backend = ComputerUseLinux::default(); + let error = backend + .create_abs_pointer_for_kwin_capture_with(2560, 1440, true, |_, _| { + anyhow::bail!("uinput unavailable") + }) + .await + .unwrap_err(); + + assert!(error.contains("geometry was stale"), "{error}"); + assert!(error.contains("could not be recreated"), "{error}"); + assert!(backend.abs_pointer.lock().unwrap().is_none()); + } + #[test] fn entry_submit_and_slider_jump_are_not_pointer_clicks() { for (role, action_name) in [ @@ -8856,7 +10102,10 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" ..Default::default() }) .unwrap(), - ClickTarget::Coordinates(60, 40) + ClickTarget::ElementBounds { + center: (60, 40), + .. + } )); let mut node = backend.last_snapshot.lock().unwrap().nodes[0].clone(); for bounds in [ @@ -8922,7 +10171,7 @@ printf '%s %s\n' "${0##*/}" "$*" >> "$CUL_POINTER_TEST_LOG" assert_eq!(action_name.as_deref(), Some("Click")); assert_eq!(action_index, 0); } - ClickTarget::Coordinates(_, _) => { + ClickTarget::Coordinates(_, _) | ClickTarget::ElementBounds { .. } => { panic!("expected AT-SPI primary-action fallback") } } @@ -10612,51 +11861,942 @@ mod focus_probe_feedback_tests { } #[cfg(test)] -mod node_target_scope_tests { +mod action_recovery_tests { use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; - fn cached_node(index: u32, object_ref: &str) -> AccessibilityNode { + const OLD_REF: &str = ":1.10/org/a11y/atspi/accessible/7"; + const NEW_REF: &str = ":1.20/org/a11y/atspi/accessible/7"; + const OTHER_REF: &str = ":1.30/org/a11y/atspi/accessible/7"; + const EXPLICIT_REF: &str = ":1.99/org/a11y/atspi/accessible/9"; + + fn action_node(index: u32, object_ref: &str) -> AccessibilityNode { AccessibilityNode { index, parent_index: None, depth: 1, object_ref: object_ref.to_string(), role: "push button".to_string(), - name: Some("Hit me".to_string()), + name: Some("Save document".to_string()), description: None, child_count: 0, bounds: None, - states: vec!["enabled".to_string()], - actions: Vec::new(), + states: vec!["enabled".to_string(), "showing".to_string()], + actions: vec![AccessibilityAction { + index: 0, + name: "click".to_string(), + description: String::new(), + keybinding: String::new(), + }], value: None, text: None, supports_editable_text: false, } } - #[tokio::test] - async fn index_from_another_apps_snapshot_is_rejected() { - let backend = ComputerUseLinux::default(); - let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42"); - backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_687)); - - let error = backend - .check_node_target(&node, Some(2_690_664)) - .await - .unwrap_err(); - assert!(error.contains("pid 2690687"), "{error}"); - assert!(error.contains("target pid 2690664"), "{error}"); - assert!( - error.contains("Call get_app_state for the target app"), - "{error}" - ); + fn semantic_params() -> ActionParams { + ActionParams { + role: Some("push button".to_string()), + name: Some("save".to_string()), + ..Default::default() + } } - #[tokio::test] - async fn same_target_or_no_target_is_allowed() { - let backend = ComputerUseLinux::default(); - let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42"); - backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_664)); + fn recovery_scope(params: serde_json::Value) -> AccessibilityRecoveryScope { + AccessibilityRecoveryScope { + params: serde_json::from_value(params).unwrap(), + target_pid: None, + } + } + + fn named_scope() -> AccessibilityRecoveryScope { + recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "org.example.Editor" + })) + } + + fn recovery_window(window_id: u64, pid: u32) -> WindowInfo { + WindowInfo { + window_id, + title: Some("Editor".to_string()), + app_id: Some("editor".to_string()), + wm_class: None, + pid: Some(pid), + bounds: None, + workspace: None, + focused: false, + hidden: false, + client_type: None, + backend: KWIN_BACKEND.to_string(), + terminal: None, + } + } + + fn refreshed_snapshot( + nodes: Vec, + scoped: bool, + root_pid: Option, + target_pid: Option, + ) -> RefreshedAccessibilitySnapshot { + RefreshedAccessibilitySnapshot { + snapshot: crate::atspi_tree_impl::AccessibilitySnapshot { + nodes, + scoped, + root_pid, + app_filter_matched: true, + truncated: false, + }, + target_pid, + window: None, + } + } + + fn successful_invocation() -> crate::atspi_tree_impl::ActionInvocation { + crate::atspi_tree_impl::ActionInvocation { + action_index: 0, + action_name: Some("click".to_string()), + ok: true, + } + } + + fn dead_name_error() -> anyhow::Error { + anyhow::Error::new(zbus::fdo::Error::ServiceUnknown( + "the cached unique name has no owner".to_string(), + )) + .context("failed to open the cached AT-SPI object") + } + + #[test] + fn lossy_atspi_dead_name_errors_are_query_phase_recoverable() { + let error = anyhow::Error::new(atspi::AtspiError::from(zbus::fdo::Error::ServiceUnknown( + "dead name".to_string(), + ))); + assert!(is_recoverable_action_query_error(&error)); + assert!(!is_recoverable_action_query_error(&anyhow::anyhow!( + "org.freedesktop.DBus.Error.UnknownObject" + ))); + assert!(!is_recoverable_action_query_error( + &error.context("failed to invoke AT-SPI action 0") + )); + } + + #[test] + fn index_and_semantic_selector_resolve_the_same_object_ref() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope(&[action_node(7, OLD_REF)], None, Some(named_scope())); + let params = semantic_params(); + let selector = params.selector(); + + let by_index = backend + .resolve_object_ref(Some(7), None, &selector, ElementResolvePurpose::Action) + .unwrap(); + let by_selector = backend + .resolve_object_ref(None, None, &selector, ElementResolvePurpose::Action) + .unwrap(); + + assert_eq!(by_index, OLD_REF); + assert_eq!(by_selector, by_index); + } + + #[tokio::test] + async fn app_state_snapshots_retain_scope_only_when_scoped() { + let backend = ComputerUseLinux::default(); + let params: GetAppStateParams = serde_json::from_value(serde_json::json!({ + "app_name_or_bundle_identifier": "org.example.Editor", + "app_id": "org.example.Editor.desktop", + "window_id": 42, + "pid": 1234 + })) + .unwrap(); + + backend + .commit_app_state_snapshot(&[action_node(0, OLD_REF)], Some(1234), None, ¶ms, true) + .await; + let scope = backend + .last_snapshot + .lock() + .unwrap() + .recovery_scope + .clone() + .expect("scoped app-state snapshots need recovery scope"); + assert_eq!( + scope.params.app_name_or_bundle_identifier.as_deref(), + Some("org.example.Editor") + ); + assert_eq!( + scope.params.app_id.as_deref(), + Some("org.example.Editor.desktop") + ); + assert_eq!(scope.params.window_id, Some(42)); + assert_eq!(scope.params.pid, Some(1234)); + assert_eq!(scope.target_pid, Some(1234)); + + backend + .commit_app_state_snapshot(&[action_node(0, OLD_REF)], None, None, ¶ms, false) + .await; + assert!(backend + .last_snapshot + .lock() + .unwrap() + .recovery_scope + .is_none()); + } + + #[tokio::test] + async fn dead_owner_recovers_new_identity_dispatches_once_and_replaces_snapshot() { + let backend = ComputerUseLinux::default(); + let scope = recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "org.example.Editor", + "app_id": "org.example.Editor.desktop", + "window_id": 42, + "pid": 1234, + "max_nodes": 50, + "max_depth": 8 + })); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope)); + let queries = Arc::new(AtomicUsize::new(0)); + let dispatches = Arc::new(AtomicUsize::new(0)); + let refreshes = Arc::new(AtomicUsize::new(0)); + let invoked_refs = Arc::new(Mutex::new(Vec::new())); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let queries = Arc::clone(&queries); + let dispatches = Arc::clone(&dispatches); + let invoked_refs = Arc::clone(&invoked_refs); + move |object_ref, _action| { + let queries = Arc::clone(&queries); + let dispatches = Arc::clone(&dispatches); + let invoked_refs = Arc::clone(&invoked_refs); + async move { + invoked_refs.lock().unwrap().push(object_ref); + if queries.fetch_add(1, Ordering::SeqCst) == 0 { + Err(dead_name_error()) + } else { + dispatches.fetch_add(1, Ordering::SeqCst); + Ok(successful_invocation()) + } + } + } + }, + { + let refreshes = Arc::clone(&refreshes); + move |scope| async move { + refreshes.fetch_add(1, Ordering::SeqCst); + assert_eq!( + scope.params.app_name_or_bundle_identifier.as_deref(), + Some("org.example.Editor") + ); + assert_eq!( + scope.params.app_id.as_deref(), + Some("org.example.Editor.desktop") + ); + assert_eq!(scope.params.window_id, Some(42)); + assert_eq!(scope.params.pid, Some(1234)); + assert_eq!(scope.params.max_nodes, Some(50)); + assert_eq!(scope.params.max_depth, Some(8)); + let mut filler = action_node(0, ":1.20/org/a11y/atspi/accessible/1"); + filler.depth = 2; + filler.role = "filler".to_string(); + filler.name = None; + filler.actions.clear(); + let mut replacement = action_node(1, NEW_REF); + replacement.depth = 2; + replacement.parent_index = Some(0); + let mut refreshed = refreshed_snapshot( + vec![filler, replacement], + true, + Some(1234), + Some(1234), + ); + refreshed.window = Some(recovery_window(42, 1234)); + Ok(refreshed) + } + }, + ) + .await; + + assert!(output.ok, "{}", output.message); + assert_eq!(queries.load(Ordering::SeqCst), 2); + assert_eq!(dispatches.load(Ordering::SeqCst), 1); + assert_eq!(refreshes.load(Ordering::SeqCst), 1); + assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF, NEW_REF]); + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(1234)); + assert_eq!(snapshot.nodes.len(), 1, "recovery must compact the tree"); + assert_eq!(snapshot.nodes[0].index, 0); + assert_eq!(snapshot.nodes[0].object_ref, NEW_REF); + let scope = snapshot.recovery_scope.as_ref().unwrap(); + assert_eq!(scope.params.pid, Some(1234)); + assert_eq!(scope.params.window_id, Some(42)); + } + + #[tokio::test] + async fn recovery_caches_fresh_window_binding_and_clears_unresolved_old_binding() { + let old_window = WindowInfo { + window_id: 42, + title: Some("Editor".to_string()), + app_id: Some("editor".to_string()), + wm_class: None, + pid: Some(1234), + bounds: None, + workspace: None, + focused: false, + hidden: false, + client_type: None, + backend: KWIN_BACKEND.to_string(), + terminal: None, + }; + + for fresh_window_id in [Some(84), None] { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_window( + &[action_node(0, OLD_REF)], + Some(1234), + Some(named_scope()), + Some(&old_window), + ); + let fresh_window = fresh_window_id.map(|window_id| WindowInfo { + window_id, + ..old_window.clone() + }); + let Json(output) = backend + .perform_element_action_with( + &semantic_params(), + Some("click"), + |object_ref, _action| async move { + if object_ref == OLD_REF { + Err(dead_name_error()) + } else { + Ok(successful_invocation()) + } + }, + |_scope| async move { + let mut refreshed = refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + Some(1234), + Some(1234), + ); + refreshed.window = fresh_window; + Ok(refreshed) + }, + ) + .await; + + assert!(output.ok, "{}", output.message); + let cached = backend.last_snapshot.lock().unwrap(); + assert_eq!(cached.nodes[0].object_ref, NEW_REF); + assert_eq!(cached.pid, Some(1234)); + assert_eq!(cached.window_id, fresh_window_id); + assert_eq!( + cached.window_backend.as_deref(), + fresh_window_id.map(|_| KWIN_BACKEND) + ); + } + } + + #[tokio::test] + async fn explicit_window_recovery_rejects_missing_or_different_window_without_replacing_cache() + { + for fresh_window_id in [None, Some(43)] { + let backend = ComputerUseLinux::default(); + let scope = recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "Editor", + "window_id": 42 + })); + backend.cache_snapshot_with_window( + &[action_node(0, OLD_REF)], + Some(1234), + Some(scope), + Some(&recovery_window(42, 1234)), + ); + let queries = Arc::new(AtomicUsize::new(0)); + let Json(output) = backend + .perform_element_action_with( + &semantic_params(), + Some("click"), + { + let queries = Arc::clone(&queries); + move |object_ref, _action| { + let queries = Arc::clone(&queries); + async move { + queries.fetch_add(1, Ordering::SeqCst); + if object_ref == OLD_REF { + Err(dead_name_error()) + } else { + Ok(successful_invocation()) + } + } + } + }, + |_scope| async move { + let mut refreshed = refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + Some(1234), + Some(1234), + ); + refreshed.window = fresh_window_id.map(|id| recovery_window(id, 1234)); + Ok(refreshed) + }, + ) + .await; + assert!(!output.ok, "{fresh_window_id:?}: {}", output.message); + assert_eq!(queries.load(Ordering::SeqCst), 1); + let cached = backend.last_snapshot.lock().unwrap(); + assert_eq!(cached.nodes[0].object_ref, OLD_REF); + assert_eq!(cached.window_id, Some(42)); + assert_eq!(cached.window_backend.as_deref(), Some(KWIN_BACKEND)); + } + } + + #[tokio::test] + async fn window_derived_pid_scope_rejects_name_only_refresh_after_window_loss() { + let backend = ComputerUseLinux::default(); + let mut scope = recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "Editor", + "window_id": 42 + })); + scope.target_pid = Some(1234); + assert_eq!(scope.params.pid, None); + assert_eq!(action_recovery_target_pid(&scope, None), Some(1234)); + assert_eq!(action_recovery_target_pid(&scope, Some(5678)), Some(5678)); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope)); + let queries = Arc::new(AtomicUsize::new(0)); + let dispatches = Arc::new(AtomicUsize::new(0)); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let queries = Arc::clone(&queries); + let dispatches = Arc::clone(&dispatches); + move |_object_ref, _action| { + let queries = Arc::clone(&queries); + let dispatches = Arc::clone(&dispatches); + async move { + if queries.fetch_add(1, Ordering::SeqCst) == 0 { + Err(dead_name_error()) + } else { + dispatches.fetch_add(1, Ordering::SeqCst); + Ok(successful_invocation()) + } + } + } + }, + |scope| async move { + let target_pid = action_recovery_target_pid(&scope, None); + Ok(refreshed_snapshot( + vec![action_node(0, OTHER_REF)], + true, + None, + target_pid, + )) + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("original app scope")); + assert_eq!(queries.load(Ordering::SeqCst), 1); + assert_eq!(dispatches.load(Ordering::SeqCst), 0); + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(1234)); + assert_eq!(snapshot.nodes[0].object_ref, OLD_REF); + } + + #[tokio::test] + async fn pid_scope_rejects_identical_selector_from_another_app() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope( + &[action_node(0, OLD_REF)], + Some(1234), + Some(recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "Editor", + "pid": 1234 + }))), + ); + let invoked_refs = Arc::new(Mutex::new(Vec::new())); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let invoked_refs = Arc::clone(&invoked_refs); + move |object_ref, _action| { + let invoked_refs = Arc::clone(&invoked_refs); + async move { + invoked_refs.lock().unwrap().push(object_ref); + Err(dead_name_error()) + } + } + }, + |_scope| async { + Ok(refreshed_snapshot( + vec![action_node(0, OTHER_REF)], + true, + Some(5678), + Some(1234), + )) + }, + ) + .await; + + assert!(!output.ok); + assert!( + output.message.contains("original app scope"), + "{}", + output.message + ); + assert!(output.message.contains("Call get_app_state")); + assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF]); + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(1234)); + assert_eq!(snapshot.nodes[0].object_ref, OLD_REF); + } + + #[tokio::test] + async fn recovery_rejects_reused_pid_when_original_filter_does_not_match() { + let backend = ComputerUseLinux::default(); + let mut scope = recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "Editor", + "pid": 1234 + })); + scope.target_pid = Some(1234); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope)); + let invoked_refs = Arc::new(Mutex::new(Vec::new())); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let invoked_refs = Arc::clone(&invoked_refs); + move |object_ref, _action| { + let invoked_refs = Arc::clone(&invoked_refs); + async move { + invoked_refs.lock().unwrap().push(object_ref); + Err(dead_name_error()) + } + } + }, + |_scope| async { + let mut refreshed = refreshed_snapshot( + vec![action_node(0, OTHER_REF)], + true, + Some(1234), + Some(1234), + ); + refreshed.snapshot.app_filter_matched = false; + Ok(refreshed) + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("original app scope")); + assert_eq!(*invoked_refs.lock().unwrap(), vec![OLD_REF]); + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(1234)); + assert_eq!(snapshot.nodes[0].object_ref, OLD_REF); + assert_eq!( + snapshot.recovery_scope.as_ref().unwrap().target_pid, + Some(1234) + ); + } + + #[tokio::test] + async fn recovery_uses_refreshed_pid_for_a_second_recovery() { + let backend = ComputerUseLinux::default(); + let mut scope = recovery_scope(serde_json::json!({ + "app_name_or_bundle_identifier": "Editor", + "window_id": 42 + })); + scope.target_pid = Some(1234); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], Some(1234), Some(scope)); + let params = semantic_params(); + let first_refs = Arc::new(Mutex::new(Vec::new())); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let first_refs = Arc::clone(&first_refs); + move |object_ref, _action| { + let first_refs = Arc::clone(&first_refs); + async move { + first_refs.lock().unwrap().push(object_ref.clone()); + if object_ref == OLD_REF { + Err(dead_name_error()) + } else { + Ok(successful_invocation()) + } + } + } + }, + |scope| async move { + assert_eq!(action_recovery_target_pid(&scope, None), Some(1234)); + let mut refreshed = refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + Some(5678), + Some(5678), + ); + refreshed.window = Some(recovery_window(42, 5678)); + Ok(refreshed) + }, + ) + .await; + + assert!(output.ok, "{}", output.message); + assert_eq!(*first_refs.lock().unwrap(), vec![OLD_REF, NEW_REF]); + { + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(5678)); + assert_eq!(snapshot.nodes[0].object_ref, NEW_REF); + let scope = snapshot.recovery_scope.as_ref().unwrap(); + assert_eq!(scope.params.pid, None); + assert_eq!(scope.target_pid, Some(5678)); + } + + let second_refs = Arc::new(Mutex::new(Vec::new())); + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let second_refs = Arc::clone(&second_refs); + move |object_ref, _action| { + let second_refs = Arc::clone(&second_refs); + async move { + second_refs.lock().unwrap().push(object_ref.clone()); + if object_ref == NEW_REF { + Err(dead_name_error()) + } else { + Ok(successful_invocation()) + } + } + } + }, + |scope| async move { + assert_eq!(action_recovery_target_pid(&scope, None), Some(5678)); + let mut refreshed = refreshed_snapshot( + vec![action_node(0, OTHER_REF)], + true, + Some(6789), + Some(6789), + ); + refreshed.window = Some(recovery_window(42, 6789)); + Ok(refreshed) + }, + ) + .await; + + assert!(output.ok, "{}", output.message); + assert_eq!(*second_refs.lock().unwrap(), vec![NEW_REF, OTHER_REF]); + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.pid, Some(6789)); + assert_eq!(snapshot.nodes[0].object_ref, OTHER_REF); + let scope = snapshot.recovery_scope.as_ref().unwrap(); + assert_eq!(scope.params.pid, None); + assert_eq!(scope.target_pid, Some(6789)); + } + + #[tokio::test] + async fn unscoped_refresh_is_not_cached_or_invoked() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], None, Some(named_scope())); + let queries = Arc::new(AtomicUsize::new(0)); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let queries = Arc::clone(&queries); + move |_object_ref, _action| { + let queries = Arc::clone(&queries); + async move { + queries.fetch_add(1, Ordering::SeqCst); + Err(dead_name_error()) + } + } + }, + |_scope| async { + Ok(refreshed_snapshot( + vec![action_node(0, OTHER_REF)], + false, + None, + None, + )) + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("could not prove")); + assert_eq!(queries.load(Ordering::SeqCst), 1); + assert_eq!( + backend.last_snapshot.lock().unwrap().nodes[0].object_ref, + OLD_REF + ); + } + + #[tokio::test] + async fn numeric_and_explicit_identifiers_never_refresh_or_get_rewritten() { + let numeric = ActionParams { + element_index: Some(7), + ..semantic_params() + }; + let explicit = ActionParams { + element_identifier: Some(EXPLICIT_REF.to_string()), + ..semantic_params() + }; + + for (params, expected_ref) in [(numeric, OLD_REF), (explicit, EXPLICIT_REF)] { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope( + &[action_node(7, OLD_REF)], + None, + Some(named_scope()), + ); + let refreshes = Arc::new(AtomicUsize::new(0)); + let invoked_refs = Arc::new(Mutex::new(Vec::new())); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let invoked_refs = Arc::clone(&invoked_refs); + move |object_ref, _action| { + let invoked_refs = Arc::clone(&invoked_refs); + async move { + invoked_refs.lock().unwrap().push(object_ref); + Err(dead_name_error()) + } + } + }, + { + let refreshes = Arc::clone(&refreshes); + move |_scope| async move { + refreshes.fetch_add(1, Ordering::SeqCst); + Ok(refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + None, + None, + )) + } + }, + ) + .await; + + assert!(!output.ok); + assert_eq!(*invoked_refs.lock().unwrap(), vec![expected_ref]); + assert_eq!(refreshes.load(Ordering::SeqCst), 0); + assert_eq!( + backend.last_snapshot.lock().unwrap().nodes[0].object_ref, + OLD_REF + ); + } + } + + #[tokio::test] + async fn invocation_phase_dead_name_error_is_never_retried() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], None, Some(named_scope())); + let queries = Arc::new(AtomicUsize::new(0)); + let refreshes = Arc::new(AtomicUsize::new(0)); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let queries = Arc::clone(&queries); + move |_object_ref, _action| { + let queries = Arc::clone(&queries); + async move { + queries.fetch_add(1, Ordering::SeqCst); + Err(dead_name_error().context("failed to invoke AT-SPI action 0")) + } + } + }, + { + let refreshes = Arc::clone(&refreshes); + move |_scope| async move { + refreshes.fetch_add(1, Ordering::SeqCst); + Ok(refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + None, + None, + )) + } + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("failed to invoke AT-SPI action 0")); + assert_eq!(queries.load(Ordering::SeqCst), 1); + assert_eq!(refreshes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn recovered_dispatch_failure_is_not_retried() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope(&[action_node(0, OLD_REF)], None, Some(named_scope())); + let queries = Arc::new(AtomicUsize::new(0)); + let refreshes = Arc::new(AtomicUsize::new(0)); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + { + let queries = Arc::clone(&queries); + move |_object_ref, _action| { + let queries = Arc::clone(&queries); + async move { + if queries.fetch_add(1, Ordering::SeqCst) == 0 { + Err(dead_name_error()) + } else { + Err(dead_name_error().context("failed to invoke AT-SPI action 0")) + } + } + } + }, + { + let refreshes = Arc::clone(&refreshes); + move |_scope| async move { + refreshes.fetch_add(1, Ordering::SeqCst); + Ok(refreshed_snapshot( + vec![action_node(0, NEW_REF)], + true, + None, + None, + )) + } + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("failed to invoke AT-SPI action 0")); + assert_eq!(queries.load(Ordering::SeqCst), 2); + assert_eq!(refreshes.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn dead_name_without_stable_scope_requires_fresh_app_state() { + let backend = ComputerUseLinux::default(); + backend.cache_nodes(&[action_node(0, OLD_REF)]); + let refreshes = Arc::new(AtomicUsize::new(0)); + let params = semantic_params(); + + let Json(output) = backend + .perform_element_action_with( + ¶ms, + Some("0"), + |_object_ref, _action| async { Err(dead_name_error()) }, + { + let refreshes = Arc::clone(&refreshes); + move |_scope| async move { + refreshes.fetch_add(1, Ordering::SeqCst); + Ok(refreshed_snapshot(Vec::new(), true, None, None)) + } + }, + ) + .await; + + assert!(!output.ok); + assert!(output.message.contains("no stable scoped recovery target")); + assert!(output.message.contains("Call get_app_state")); + assert_eq!(refreshes.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn post_action_snapshot_does_not_retain_stale_recovery_scope() { + let backend = ComputerUseLinux::default(); + backend.cache_snapshot_with_scope( + &[action_node(0, OLD_REF)], + Some(1234), + Some(named_scope()), + ); + + backend + .commit_snapshot(&[action_node(0, NEW_REF)], Some(5678)) + .await; + + let snapshot = backend.last_snapshot.lock().unwrap(); + assert_eq!(snapshot.nodes[0].object_ref, NEW_REF); + assert_eq!(snapshot.pid, Some(5678)); + assert!(snapshot.recovery_scope.is_none()); + } +} + +#[cfg(test)] +mod node_target_scope_tests { + use super::*; + + fn cached_node(index: u32, object_ref: &str) -> AccessibilityNode { + AccessibilityNode { + index, + parent_index: None, + depth: 1, + object_ref: object_ref.to_string(), + role: "push button".to_string(), + name: Some("Hit me".to_string()), + description: None, + child_count: 0, + bounds: None, + states: vec!["enabled".to_string()], + actions: Vec::new(), + value: None, + text: None, + supports_editable_text: false, + } + } + + #[tokio::test] + async fn index_from_another_apps_snapshot_is_rejected() { + let backend = ComputerUseLinux::default(); + let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42"); + backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_687)); + + let error = backend + .check_node_target(&node, Some(2_690_664)) + .await + .unwrap_err(); + assert!(error.contains("pid 2690687"), "{error}"); + assert!(error.contains("target pid 2690664"), "{error}"); + assert!( + error.contains("Call get_app_state for the target app"), + "{error}" + ); + } + + #[tokio::test] + async fn same_target_or_no_target_is_allowed() { + let backend = ComputerUseLinux::default(); + let node = cached_node(2, ":1.9/org/a11y/atspi/accessible/42"); + backend.cache_snapshot(std::slice::from_ref(&node), Some(2_690_664)); assert!(backend .check_node_target(&node, Some(2_690_664)) @@ -10828,6 +12968,71 @@ mod post_action_capture_tests { .is_err()); } + #[test] + fn editable_text_results_map_ok_verified_and_mcp_error_status() { + for (outcome, expected_ok, expected_verified) in [ + (EditableTextSetOutcome::Verified, true, Some(true)), + ( + EditableTextSetOutcome::MismatchAfterDeadline, + false, + Some(false), + ), + (EditableTextSetOutcome::ReadbackFailed, false, Some(false)), + (EditableTextSetOutcome::ReadbackTimedOut, false, Some(false)), + ( + EditableTextSetOutcome::PasswordUnverifiable, + true, + Some(false), + ), + ] { + let (output, verified) = editable_text_action_result(outcome, None); + assert_eq!(output.ok, expected_ok); + assert_eq!(verified, expected_verified); + let result = structured_tool_result( + SetValueToolOutput { + action: output, + verified, + observation: PostActionObservation::default(), + }, + None, + ) + .unwrap(); + assert_eq!(result.is_error, Some(!expected_ok)); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["ok"], serde_json::json!(expected_ok)); + assert_eq!(structured["verified"], serde_json::json!(expected_verified)); + } + } + + #[test] + fn set_value_output_omits_verified_for_numeric_and_early_failures() { + let output = SetValueToolOutput { + action: action_result("set_value", Ok(Vec::new()), None), + verified: None, + observation: PostActionObservation::default(), + }; + + let value = serde_json::to_value(output).unwrap(); + + assert!(!value.as_object().unwrap().contains_key("verified")); + } + + #[test] + fn only_set_value_schema_exposes_verified() { + let tools = ComputerUseLinux::default().mcp_tool_router().list_all(); + for tool in tools { + let value = serde_json::to_value(&tool).unwrap(); + let Some(properties) = value["outputSchema"]["properties"].as_object() else { + continue; + }; + if tool.name == "set_value" { + assert!(properties.contains_key("verified")); + } else { + assert!(!properties.contains_key("verified"), "tool {}", tool.name); + } + } + } + #[test] fn action_tool_output_omits_observation_fields_when_not_requested() { let output = ActionToolOutput { @@ -11160,5 +13365,42 @@ exit 1 .as_str() .unwrap() .contains("WARNING: X11 keyboard layout")); + + std::fs::write(&log_path, "").unwrap(); + let set_value_capture: PostActionCaptureParams = serde_json::from_value( + serde_json::json!({"screenshot_after": "screen", "include_tree": true}), + ) + .unwrap(); + let (output, verified) = + editable_text_action_result(EditableTextSetOutcome::MismatchAfterDeadline, None); + let result = server + .finish_set_value_output(output, verified, &set_value_capture) + .await + .unwrap(); + assert_eq!(result.is_error, Some(true)); + assert_eq!(result.content.len(), 2); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["ok"], false); + assert_eq!(structured["verified"], false); + assert_eq!(structured["screenshot"]["width"], 64); + assert!(structured["accessibility_error"].is_string()); + assert_eq!( + std::fs::read_to_string(&log_path).unwrap(), + "gnome-screenshot\n" + ); + + std::fs::write(&log_path, "").unwrap(); + let output = action_result("set_value", Err("pre-dispatch failure".to_string()), None); + let result = server + .finish_set_value_output(output, None, &set_value_capture) + .await + .unwrap(); + assert_eq!(result.is_error, Some(true)); + let structured = result.structured_content.unwrap(); + assert_eq!(structured["ok"], false); + assert!(structured.get("screenshot").is_none()); + assert!(structured.get("accessibility_tree").is_none()); + assert!(structured.get("accessibility_error").is_none()); + assert!(std::fs::read_to_string(&log_path).unwrap().is_empty()); } } diff --git a/src/windowing/backends/kwin.rs b/src/windowing/backends/kwin.rs index 91384b4..60f50ba 100644 --- a/src/windowing/backends/kwin.rs +++ b/src/windowing/backends/kwin.rs @@ -60,6 +60,11 @@ pub(crate) async fn logical_desktop_rect() -> Result<(i32, i32, i32, i32)> { parse_kwin_logical_desktop_rect(&json) } +pub(crate) async fn element_click_snapshot() -> Result { + let json = call_kwin_window_script().await?; + KwinElementClickSnapshot::from_json(&json) +} + pub async fn activate_window(window_id: u64) -> Result<()> { let uuid = kwin_uuid_for_window_id(window_id).await?.with_context(|| { format!("No KWin window matched window_id {window_id} during activation") @@ -502,23 +507,69 @@ fn kwin_window_script_source( return null; }} - function geometry(window) {{ - var frame = null; + function rect(obj, key) {{ + var value = null; try {{ - frame = window.frameGeometry; + value = obj[key]; }} catch (error) {{}} + if (value === null || value === undefined) {{ + return null; + }} + return {{ + x: read(value, "x"), + y: read(value, "y"), + width: read(value, "width"), + height: read(value, "height") + }}; + }} + + function geometry(window) {{ + var frame = rect(window, "frameGeometry"); var x = read(window, "x"); var y = read(window, "y"); var width = read(window, "width"); var height = read(window, "height"); return {{ - x: x !== null ? x : read(frame, "x"), - y: y !== null ? y : read(frame, "y"), - width: width !== null ? width : read(frame, "width"), - height: height !== null ? height : read(frame, "height") + x: x !== null ? x : (frame === null ? null : frame.x), + y: y !== null ? y : (frame === null ? null : frame.y), + width: width !== null ? width : (frame === null ? null : frame.width), + height: height !== null ? height : (frame === null ? null : frame.height) }}; }} + function outputIdentity(window) {{ + var output = null; + try {{ + output = window.output; + }} catch (error) {{}} + return read(output, "name"); + }} + + function activeOutputs() {{ + var outputs = null; + try {{ + outputs = workspace.screens; + }} catch (error) {{}} + if (outputs === null || outputs === undefined) {{ + try {{ + outputs = workspace.outputs; + }} catch (error) {{}} + }} + if (outputs === null || outputs === undefined) {{ + return []; + }} + var active = Array.prototype.filter.call(outputs, function(output) {{ + return read(output, "enabled") !== false; + }}); + return Array.prototype.map.call(active, function(output) {{ + return {{ + name: read(output, "name"), + geometry: rect(output, "geometry"), + devicePixelRatio: read(output, "devicePixelRatio") + }}; + }}); + }} + function workspaceGeometry() {{ var rect = null; try {{ @@ -545,6 +596,41 @@ fn kwin_window_script_source( return isFinite(parsed) ? parsed : null; }} + function onCurrentDesktop(window) {{ + // KWin::Window exposes desktops/onAllDesktops, not onCurrentDesktop. + // Keep the QObject list intact instead of serializing its members. + if (read(window, "onAllDesktops") === true) {{ + return true; + }} + try {{ + var desktops = window.desktops; + if (desktops === null || desktops === undefined) {{ + return null; + }} + if (desktops.length === 0) {{ + return true; + }} + var current = typeof workspace.currentDesktopForScreen === "function" + ? workspace.currentDesktopForScreen(window.output) + : workspace.currentDesktop; + var currentId = read(current, "id"); + if (currentId === null) {{ + return null; + }} + var unknown = false; + for (var i = 0; i < desktops.length; ++i) {{ + var id = read(desktops[i], "id"); + if (id === currentId) {{ + return true; + }} + unknown = unknown || id === null; + }} + return unknown ? null : false; + }} catch (error) {{ + return null; + }} + }} + function clientType(window) {{ if (read(window, "waylandClient")) {{ return "wayland"; @@ -606,11 +692,23 @@ fn kwin_window_script_source( y: geo.y, width: geo.width, height: geo.height, + frameGeometry: rect(window, "frameGeometry"), + clientGeometry: rect(window, "clientGeometry"), + output: outputIdentity(window), workspace: firstDesktop(window), minimized: read(window, "minimized"), + onCurrentDesktop: onCurrentDesktop(window), active: read(window, "active") || window === activeWindow, clientType: clientType(window), normalWindow: read(window, "normalWindow"), + dialog: read(window, "dialog"), + transient: read(window, "transient"), + managed: read(window, "managed"), + wantsInput: read(window, "wantsInput"), + specialWindow: read(window, "specialWindow"), + popupWindow: read(window, "popupWindow") || read(window, "popupMenu") + || read(window, "dropdownMenu") || read(window, "tooltip") + || read(window, "comboBox") || read(window, "menu"), desktopWindow: read(window, "desktopWindow"), skipTaskbar: read(window, "skipTaskbar"), dock: read(window, "dock") @@ -621,6 +719,7 @@ fn kwin_window_script_source( backend: "kwin", pluginName: pluginName, desktopGeometry: workspaceGeometry(), + outputs: activeOutputs(), windows: windows }})); }})(); @@ -849,10 +948,7 @@ fn parse_kwin_windows(json: &str) -> Result> { let mut windows = snapshot .windows .into_iter() - .filter(|window| !json_value_as_bool(window.desktop_window.as_ref()).unwrap_or(false)) - .filter(|window| !json_value_as_bool(window.dock.as_ref()).unwrap_or(false)) - .filter(|window| !json_value_as_bool(window.skip_taskbar.as_ref()).unwrap_or(false)) - .filter(|window| json_value_as_bool(window.normal_window.as_ref()).unwrap_or(true)) + .filter(element_window) .map(WindowInfo::try_from) .collect::>>()?; windows.sort_by_key(|window| window.window_id); @@ -871,6 +967,8 @@ fn parse_kwin_snapshot(json: &str) -> Result { struct KwinWindowSnapshot { #[serde(default, rename = "desktopGeometry")] desktop_geometry: Option, + #[serde(default)] + outputs: Vec, windows: Vec, } @@ -902,6 +1000,14 @@ struct KwinRawGeometry { height: Option, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct KwinRawOutput { + name: Option, + geometry: Option, + device_pixel_ratio: Option, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] struct KwinRawWindow { @@ -917,11 +1023,21 @@ struct KwinRawWindow { y: Option, width: Option, height: Option, + frame_geometry: Option, + client_geometry: Option, + output: Option, workspace: Option, minimized: Option, + on_current_desktop: Option, active: Option, client_type: Option, normal_window: Option, + dialog: Option, + transient: Option, + managed: Option, + wants_input: Option, + special_window: Option, + popup_window: Option, desktop_window: Option, skip_taskbar: Option, dock: Option, @@ -936,6 +1052,432 @@ impl KwinRawWindow { } } +#[derive(Debug, Clone, Copy)] +pub(crate) struct KwinElementFrame { + pub(crate) x: i32, + pub(crate) y: i32, + pub(crate) width: i32, + pub(crate) height: i32, +} + +#[derive(Debug, Clone, Copy)] +pub(crate) struct KwinElementClickTarget { + pub(crate) window_id: Option, + pub(crate) pid: Option, + pub(crate) frame: KwinElementFrame, + pub(crate) center: (i32, i32), + pub(crate) require_exact_focus: bool, +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(crate) struct KwinElementClickPoint { + pub(crate) logical: (i32, i32), + pub(crate) capture: (i32, i32), +} + +pub(crate) struct KwinElementClickSnapshot { + windows: Vec, + outputs: Vec, +} + +struct KwinElementOutput { + name: String, + geometry: KwinRect, + device_pixel_ratio: f64, +} + +#[derive(Clone, Copy)] +struct KwinRect { + x: f64, + y: f64, + width: f64, + height: f64, +} + +impl KwinRect { + fn new(x: f64, y: f64, width: f64, height: f64) -> Result { + if ![x, y, width, height].into_iter().all(f64::is_finite) || width <= 0.0 || height <= 0.0 { + anyhow::bail!("geometry must be finite with positive width and height"); + } + Ok(Self { + x, + y, + width, + height, + }) + } + + fn from_raw(geometry: Option<&KwinRawGeometry>, label: &str) -> Result { + let geometry = geometry.with_context(|| format!("KWin did not expose {label}"))?; + Self::new( + json_value_as_f64(geometry.x.as_ref()) + .with_context(|| format!("KWin {label} x is unavailable"))?, + json_value_as_f64(geometry.y.as_ref()) + .with_context(|| format!("KWin {label} y is unavailable"))?, + json_value_as_f64(geometry.width.as_ref()) + .with_context(|| format!("KWin {label} width is unavailable"))?, + json_value_as_f64(geometry.height.as_ref()) + .with_context(|| format!("KWin {label} height is unavailable"))?, + ) + .with_context(|| format!("KWin {label} is unusable")) + } + + fn right(self) -> f64 { + self.x + self.width + } + + fn bottom(self) -> f64 { + self.y + self.height + } + + fn contains(self, point: (f64, f64)) -> bool { + point.0 >= self.x && point.0 < self.right() && point.1 >= self.y && point.1 < self.bottom() + } + + fn overlaps(self, other: Self) -> bool { + self.x.max(other.x) < self.right().min(other.right()) + && self.y.max(other.y) < self.bottom().min(other.bottom()) + } +} + +impl KwinElementClickSnapshot { + fn from_json(json: &str) -> Result { + let snapshot = parse_kwin_snapshot(json)?; + let outputs = snapshot + .outputs + .iter() + .map(|output| { + let name = clean_string(output.name.as_deref()) + .context("KWin output name is unavailable")?; + let geometry = KwinRect::from_raw( + output.geometry.as_ref(), + &format!("geometry for output {name}"), + )?; + let device_pixel_ratio = json_value_as_f64(output.device_pixel_ratio.as_ref()) + .with_context(|| { + format!("KWin output {name} devicePixelRatio is unavailable") + })?; + if !device_pixel_ratio.is_finite() || device_pixel_ratio <= 0.0 { + anyhow::bail!("KWin output {name} has an invalid devicePixelRatio"); + } + Ok(KwinElementOutput { + name, + geometry, + device_pixel_ratio, + }) + }) + .collect::>>()?; + if outputs.is_empty() { + anyhow::bail!("KWin did not expose any active outputs"); + } + Ok(Self { + windows: snapshot.windows, + outputs, + }) + } + + pub(crate) fn map_element_click( + &self, + target: &KwinElementClickTarget, + capture_width: u32, + capture_height: u32, + ) -> Result { + let window = self.associated_window(target)?; + if target + .pid + .is_some_and(|pid| json_value_as_u32(window.pid.as_ref()) != Some(pid)) + { + anyhow::bail!("cached KWin window identity and AT-SPI root PID disagree"); + } + if target.require_exact_focus + && !json_value_as_bool(window.active.as_ref()).unwrap_or(false) + { + anyhow::bail!( + "the explicitly targeted KWin window is not exactly focused; app-only focus is insufficient" + ); + } + let window_id = window + .kwin_uuid() + .map(|uuid| kwin_window_id_from_uuid(&uuid)) + .context("associated KWin window has no stable identity")?; + let frame = KwinRect::from_raw( + window.frame_geometry.as_ref(), + &format!("frameGeometry for window_id {window_id}"), + )?; + let client = KwinRect::from_raw( + window.client_geometry.as_ref(), + &format!("clientGeometry for window_id {window_id}"), + )?; + let output_name = clean_string(window.output.as_deref()) + .with_context(|| format!("KWin window_id {window_id} has no output identity"))?; + if self + .outputs + .iter() + .filter(|output| output.name == output_name) + .count() + != 1 + { + anyhow::bail!("KWin window_id {window_id} output identity is missing or ambiguous"); + } + let global = global_element_point(target, frame, client)?; + self.capture_point(global, &output_name, capture_width, capture_height) + } + + fn associated_window(&self, target: &KwinElementClickTarget) -> Result<&KwinRawWindow> { + let pid = target + .pid + .context("cached element has no root PID identity")?; + if let Some(window_id) = target.window_id { + let identified = self + .windows + .iter() + .filter(|window| { + window + .kwin_uuid() + .is_some_and(|uuid| kwin_window_id_from_uuid(&uuid) == window_id) + }) + .collect::>(); + match identified.as_slice() { + [window] if json_value_as_u32(window.pid.as_ref()) == Some(pid) => {} + [_] => anyhow::bail!("cached KWin window identity and AT-SPI root PID disagree"), + [] => anyhow::bail!("no KWin window matched the cached window identity"), + _ => anyhow::bail!("cached window identity matched multiple KWin windows"), + } + } + // PID-scoped AT-SPI trees include every window of an application. A + // requested window or current focus cannot establish which frame owns + // a selected element. Even hidden same-PID windows can be its owner. + let candidates = self + .windows + .iter() + .filter(|window| json_value_as_u32(window.pid.as_ref()) == Some(pid)) + .collect::>(); + let mut matching = Vec::new(); + for window in &candidates { + let frame = + KwinRect::from_raw(window.frame_geometry.as_ref(), "candidate frameGeometry")?; + let client = + KwinRect::from_raw(window.client_geometry.as_ref(), "candidate clientGeometry")?; + if !element_coordinate_matches(target, frame, client)?.is_empty() { + matching.push(*window); + } + } + let window = match matching.as_slice() { + [window] => *window, + [] => anyhow::bail!("AT-SPI frame matches neither client-local, global client, nor global decorated KWin geometry for the root PID"), + _ => anyhow::bail!( + "AT-SPI frame matches multiple same-PID KWin windows; focus or a requested window_id cannot prove element ownership" + ), + }; + if target.window_id.is_some_and(|id| { + window + .kwin_uuid() + .is_none_or(|uuid| kwin_window_id_from_uuid(&uuid) != id) + }) { + anyhow::bail!("selected AT-SPI frame belongs to a different KWin window than the cached or requested window identity"); + } + if !element_window(window) { + anyhow::bail!( + "unsupported KWin window type: element mapping requires a normal window, dialog, or verified interactive top-level transient; desktop, dock, and popup windows cannot be mapped by refreshing the snapshot" + ); + } + if !visible_element_window(window) { + anyhow::bail!( + "associated KWin window is hidden or its current-desktop visibility is unknown" + ); + } + Ok(window) + } + + fn capture_point( + &self, + global: (f64, f64), + window_output: &str, + capture_width: u32, + capture_height: u32, + ) -> Result { + if capture_width == 0 || capture_height == 0 { + anyhow::bail!("raw desktop capture is empty"); + } + let mut names = std::collections::BTreeSet::new(); + if !self + .outputs + .iter() + .all(|output| names.insert(output.name.as_str())) + { + anyhow::bail!("KWin active-output identities are ambiguous"); + } + for (index, output) in self.outputs.iter().enumerate() { + if self.outputs[index + 1..] + .iter() + .any(|other| output.geometry.overlaps(other.geometry)) + { + anyhow::bail!("overlapping or mirrored KWin outputs are unsupported"); + } + } + let union = logical_output_union(&self.outputs)?; + let scale_x = f64::from(capture_width) / union.width; + let scale_y = f64::from(capture_height) / union.height; + if !scale_x.is_finite() + || !scale_y.is_finite() + || scale_x <= 0.0 + || scale_y <= 0.0 + || (scale_x - scale_y).abs() > 0.01 + { + anyhow::bail!("raw capture does not have a finite, positive, uniform desktop scale"); + } + if let Some(output) = self.outputs.iter().find(|output| { + (output.device_pixel_ratio - scale_x).abs() > 0.01 + || (output.device_pixel_ratio - scale_y).abs() > 0.01 + }) { + anyhow::bail!( + "mixed or unknown KWin output scales are unsupported (output {} DPR {}, capture scale {:.4}x{:.4})", + output.name, + output.device_pixel_ratio, + scale_x, + scale_y + ); + } + let containing = self + .outputs + .iter() + .filter(|output| output.geometry.contains(global)) + .collect::>(); + let output = match containing.as_slice() { + [output] => *output, + [] => anyhow::bail!("mapped element point is outside every active KWin output"), + _ => anyhow::bail!("mapped element point belongs to multiple KWin outputs"), + }; + if output.name != window_output { + anyhow::bail!("mapped element point is not on the associated KWin window's output"); + } + let logical = (round_coordinate(global.0)?, round_coordinate(global.1)?); + let capture = ( + round_coordinate((global.0 - union.x) * scale_x)?, + round_coordinate((global.1 - union.y) * scale_y)?, + ); + if capture.0 < 0 + || capture.1 < 0 + || capture.0 as u32 >= capture_width + || capture.1 as u32 >= capture_height + { + anyhow::bail!("mapped element point is outside the raw desktop capture"); + } + Ok(KwinElementClickPoint { logical, capture }) + } +} + +fn visible_element_window(window: &KwinRawWindow) -> bool { + !json_value_as_bool(window.minimized.as_ref()).unwrap_or(false) + && json_value_as_bool(window.on_current_desktop.as_ref()) == Some(true) +} + +fn element_window(window: &KwinRawWindow) -> bool { + if [ + &window.desktop_window, + &window.dock, + &window.popup_window, + &window.special_window, + ] + .into_iter() + .any(|flag| json_value_as_bool(flag.as_ref()) == Some(true)) + { + return false; + } + // skipTaskbar is a presentation hint. Dialogs and proven interactive + // transients remain targetable even when they do not have a taskbar entry. + let dialog = json_value_as_bool(window.dialog.as_ref()) == Some(true); + let interactive_transient = json_value_as_bool(window.transient.as_ref()) == Some(true) + && json_value_as_bool(window.managed.as_ref()) == Some(true) + && json_value_as_bool(window.wants_input.as_ref()) == Some(true) + && json_value_as_bool(window.special_window.as_ref()) == Some(false) + && json_value_as_bool(window.popup_window.as_ref()) == Some(false); + let normal = json_value_as_bool(window.normal_window.as_ref()).unwrap_or(true) + && !json_value_as_bool(window.skip_taskbar.as_ref()).unwrap_or(false); + normal || dialog || interactive_transient +} + +fn global_element_point( + target: &KwinElementClickTarget, + frame: KwinRect, + client: KwinRect, +) -> Result<(f64, f64)> { + let mut matches = element_coordinate_matches(target, frame, client)?; + // Undecorated global client/frame signatures describe the same mapping. + // Collapse only identical points; tolerance must not hide different ones. + matches.dedup(); + match matches.as_slice() { + [point] => Ok(*point), + [] => anyhow::bail!( + "AT-SPI frame matches neither client-local, global client, nor global decorated KWin geometry" + ), + _ => anyhow::bail!("AT-SPI frame has an ambiguous KWin coordinate signature"), + } +} + +fn element_coordinate_matches( + target: &KwinElementClickTarget, + frame: KwinRect, + client: KwinRect, +) -> Result> { + let atspi = KwinRect::new( + f64::from(target.frame.x), + f64::from(target.frame.y), + f64::from(target.frame.width), + f64::from(target.frame.height), + ) + .context("AT-SPI frame bounds are unusable")?; + let center = (f64::from(target.center.0), f64::from(target.center.1)); + if !atspi.contains(center) { + anyhow::bail!("element center is outside its depth-1 AT-SPI frame"); + } + let local_origin = near(atspi.x, 0.0) && near(atspi.y, 0.0); + let client_size = near(atspi.width, client.width) && near(atspi.height, client.height); + let decorated_size = near(atspi.width, frame.width) && near(atspi.height, frame.height); + let mut matches = Vec::new(); + if local_origin && client_size { + matches.push((client.x + center.0 - atspi.x, client.y + center.1 - atspi.y)); + } + if near(atspi.x, client.x) && near(atspi.y, client.y) && client_size { + matches.push(center); + } + if near(atspi.x, frame.x) && near(atspi.y, frame.y) && decorated_size { + matches.push(center); + } + Ok(matches) +} + +fn near(left: f64, right: f64) -> bool { + (left - right).abs() <= 1.0 +} + +fn logical_output_union(outputs: &[KwinElementOutput]) -> Result { + let first = outputs + .first() + .context("KWin active-output list is empty")? + .geometry; + let left = outputs + .iter() + .fold(first.x, |value, output| value.min(output.geometry.x)); + let top = outputs + .iter() + .fold(first.y, |value, output| value.min(output.geometry.y)); + let right = outputs.iter().fold(first.right(), |value, output| { + value.max(output.geometry.right()) + }); + let bottom = outputs.iter().fold(first.bottom(), |value, output| { + value.max(output.geometry.bottom()) + }); + KwinRect::new(left, top, right - left, bottom - top).context("KWin output union is unusable") +} + +fn round_coordinate(value: f64) -> Result { + if !value.is_finite() || value < i32::MIN as f64 || value > i32::MAX as f64 { + anyhow::bail!("mapped coordinate is not a finite i32"); + } + Ok(value.round() as i32) +} + impl TryFrom for WindowInfo { type Error = anyhow::Error; @@ -1204,6 +1746,568 @@ mod adapter_tests { } } +#[cfg(test)] +mod element_click_tests { + use super::*; + use serde_json::{json, Value}; + + const UUID: &str = "b4dfacf8-a559-43c9-8b1f-ecd5cfd78359"; + const OTHER_UUID: &str = "11111111-2222-3333-4444-555555555555"; + const PID: u32 = 4242; + + fn rect(x: i32, y: i32, width: i32, height: i32) -> Value { + json!({"x": x, "y": y, "width": width, "height": height}) + } + + fn output(name: &str, geometry: Value, dpr: Value) -> Value { + json!({"name": name, "geometry": geometry, "devicePixelRatio": dpr}) + } + + fn window(uuid: &str, frame: Value, client: Value, output: &str) -> Value { + json!({ + "uuid": uuid, + "pid": PID, + "frameGeometry": frame, + "clientGeometry": client, + "output": output, + "active": true, + "normalWindow": true, + "minimized": false, + "onCurrentDesktop": true + }) + } + + fn snapshot(windows: Vec, outputs: Vec) -> KwinElementClickSnapshot { + KwinElementClickSnapshot::from_json( + &json!({"backend": "kwin", "windows": windows, "outputs": outputs}).to_string(), + ) + .unwrap() + } + + fn target(atspi: [i32; 4], center: (i32, i32)) -> KwinElementClickTarget { + KwinElementClickTarget { + window_id: Some(kwin_window_id_from_uuid(UUID)), + pid: Some(PID), + frame: KwinElementFrame { + x: atspi[0], + y: atspi[1], + width: atspi[2], + height: atspi[3], + }, + center, + require_exact_focus: false, + } + } + + fn reporter_window() -> Value { + window( + UUID, + rect(246, 308, 800, 628), + rect(246, 336, 800, 600), + "DP-1", + ) + } + + #[test] + fn reporter_menu_center_maps_to_global_logical_and_raw_capture_points() { + let snapshot = snapshot( + vec![reporter_window()], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + + let point = snapshot + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .unwrap(); + + assert_eq!(point.logical, (278, 352)); + assert_eq!(point.capture, (417, 528)); + } + + fn dialog_window(caption: &str, skip_taskbar: bool) -> Value { + let mut dialog = reporter_window(); + dialog["caption"] = json!(caption); + dialog["normalWindow"] = json!(false); + dialog["dialog"] = json!(true); + dialog["transient"] = json!(true); + dialog["skipTaskbar"] = json!(skip_taskbar); + dialog + } + + #[test] + fn dialog_windows_are_listed_even_when_they_skip_the_taskbar() { + for caption in ["Save File", "Open File", "Confirm overwrite"] { + for skip_taskbar in [false, true] { + let json = json!({"windows": [dialog_window(caption, skip_taskbar)]}).to_string(); + let listed = parse_kwin_windows(&json).unwrap(); + assert_eq!(listed.len(), 1, "{caption}: skipTaskbar={skip_taskbar}"); + assert_eq!(listed[0].title.as_deref(), Some(caption)); + } + } + } + + #[test] + fn dialog_element_clicks_map_even_when_they_skip_the_taskbar() { + for caption in ["Save File", "Open File", "Confirm overwrite"] { + for skip_taskbar in [false, true] { + let snapshot = snapshot( + vec![dialog_window(caption, skip_taskbar)], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + let point = snapshot + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .unwrap(); + assert_eq!(point.logical, (278, 352)); + assert_eq!(point.capture, (417, 528)); + } + } + } + + #[test] + fn interactive_transients_require_proven_top_level_input_capabilities() { + let mut transient = dialog_window("Tool window", true); + transient["dialog"] = json!(false); + transient["managed"] = json!(true); + transient["wantsInput"] = json!(true); + transient["specialWindow"] = json!(false); + transient["popupWindow"] = json!(false); + let json = json!({"windows": [transient.clone()]}).to_string(); + assert_eq!(parse_kwin_windows(&json).unwrap().len(), 1); + let outputs = vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))]; + assert!(snapshot(vec![transient.clone()], outputs.clone()) + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .is_ok()); + for flag in ["managed", "wantsInput", "specialWindow", "popupWindow"] { + let mut unproven = transient.clone(); + unproven[flag] = Value::Null; + assert!( + parse_kwin_windows(&json!({"windows": [unproven.clone()]}).to_string()) + .unwrap() + .is_empty(), + "{flag}" + ); + let error = snapshot(vec![unproven], outputs.clone()) + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsupported KWin window type"), + "{flag}: {error}" + ); + } + } + + #[test] + fn desktop_dock_special_and_popup_windows_are_neither_listed_nor_mapped() { + for flag in ["desktopWindow", "dock", "specialWindow", "popupWindow"] { + let mut excluded = dialog_window("Excluded surface", true); + excluded[flag] = json!(true); + assert!( + parse_kwin_windows(&json!({"windows": [excluded.clone()]}).to_string()) + .unwrap() + .is_empty(), + "{flag}" + ); + let snapshot = snapshot( + vec![excluded], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + let error = snapshot + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .unwrap_err() + .to_string(); + assert!( + error.contains("unsupported KWin window type"), + "{flag}: {error}" + ); + } + } + + #[test] + fn same_pid_local_frames_cannot_borrow_focused_or_explicit_window_identity() { + let first = reporter_window(); + let mut second = window( + OTHER_UUID, + rect(1100, 308, 800, 628), + rect(1100, 336, 800, 600), + "DP-1", + ); + second["active"] = json!(false); + let snapshot = snapshot( + vec![first, second], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + for window_id in [None, Some(kwin_window_id_from_uuid(UUID))] { + let mut target = target([0, 0, 800, 600], (32, 16)); + target.window_id = window_id; + assert!(snapshot.map_element_click(&target, 3840, 2160).is_err()); + } + } + + #[test] + fn unknown_or_other_desktop_visibility_never_dispatches() { + for visibility in [Value::Null, json!(false)] { + let mut window = reporter_window(); + window["onCurrentDesktop"] = visibility; + let snapshot = snapshot( + vec![window], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + assert!(snapshot + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .is_err()); + } + } + + #[test] + fn equivalent_global_undecorated_signatures_share_one_point() { + let snapshot = snapshot( + vec![window( + UUID, + rect(100, 100, 200, 200), + rect(100, 100, 200, 200), + "DP-1", + )], + vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.0))], + ); + assert_eq!( + snapshot + .map_element_click(&target([100, 100, 200, 200], (110, 110)), 1000, 800) + .unwrap() + .logical, + (110, 110) + ); + } + + #[test] + fn asymmetric_and_undecorated_frames_use_actual_client_origin() { + let outputs = vec![output("DP-1", rect(0, 0, 1280, 720), json!(1.0))]; + let asymmetric = snapshot( + vec![window( + UUID, + rect(100, 100, 800, 630), + rect(110, 130, 780, 600), + "DP-1", + )], + outputs.clone(), + ); + assert_eq!( + asymmetric + .map_element_click(&target([0, 0, 780, 600], (40, 50)), 1280, 720) + .unwrap(), + KwinElementClickPoint { + logical: (150, 180), + capture: (150, 180) + } + ); + + let undecorated = snapshot( + vec![window( + UUID, + rect(300, 200, 640, 480), + rect(300, 200, 640, 480), + "DP-1", + )], + outputs, + ); + assert_eq!( + undecorated + .map_element_click(&target([0, 0, 640, 480], (20, 30)), 1280, 720) + .unwrap() + .logical, + (320, 230) + ); + } + + #[test] + fn global_client_and_decorated_frames_do_not_apply_another_offset() { + let outputs = vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))]; + for atspi in [[246, 336, 800, 600], [246, 308, 800, 628]] { + let snapshot = snapshot(vec![reporter_window()], outputs.clone()); + let point = snapshot + .map_element_click(&target(atspi, (300, 400)), 3840, 2160) + .unwrap(); + assert_eq!(point.logical, (300, 400)); + assert_eq!(point.capture, (450, 600)); + } + } + + #[test] + fn signature_tolerance_is_one_logical_pixel_and_fractional_capture_rounds() { + let tolerant = snapshot( + vec![window( + UUID, + rect(100, 100, 200, 200), + rect(100, 100, 200, 200), + "DP-1", + )], + vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.25))], + ); + assert_eq!( + tolerant + .map_element_click(&target([1, -1, 201, 199], (10, 10)), 1250, 1000) + .unwrap() + .logical, + (109, 111) + ); + + let rounded = snapshot( + vec![window( + UUID, + rect(100, 100, 200, 200), + rect(100, 100, 200, 200), + "DP-1", + )], + vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.25))], + ); + let point = rounded + .map_element_click(&target([0, 0, 200, 200], (2, 3)), 1250, 1000) + .unwrap(); + assert_eq!(point.logical, (102, 103)); + assert_eq!(point.capture, (128, 129)); + } + + #[test] + fn negative_and_second_monitor_origins_are_preserved() { + let left = output("HDMI-1", rect(-1280, 0, 1280, 1024), json!(1.25)); + let right = output("DP-1", rect(0, 0, 1600, 900), json!(1.25)); + let snapshot = snapshot( + vec![window( + UUID, + rect(-1000, 100, 200, 200), + rect(-1000, 100, 200, 200), + "HDMI-1", + )], + vec![left, right], + ); + + let point = snapshot + .map_element_click(&target([0, 0, 200, 200], (10, 20)), 3600, 1280) + .unwrap(); + + assert_eq!(point.logical, (-990, 120)); + assert_eq!(point.capture, (363, 150)); + } + + #[test] + fn mixed_unknown_and_mismatched_output_scales_fail() { + let window = reporter_window(); + let mixed = snapshot( + vec![window.clone()], + vec![ + output("DP-1", rect(0, 0, 1000, 800), json!(1.25)), + output("HDMI-1", rect(1000, 0, 1000, 800), json!(1.0)), + ], + ); + let error = mixed + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 2500, 1000) + .unwrap_err() + .to_string(); + assert!(error.contains("mixed or unknown"), "{error}"); + + let mismatch = snapshot( + vec![window], + vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.0))], + ); + assert!(mismatch + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 2000, 1600) + .is_err()); + + let error = KwinElementClickSnapshot::from_json( + &json!({ + "backend": "kwin", + "windows": [], + "outputs": [output("DP-1", rect(0, 0, 1000, 800), Value::Null)] + }) + .to_string(), + ) + .err() + .unwrap() + .to_string(); + assert!(error.contains("devicePixelRatio"), "{error}"); + } + + #[test] + fn overlap_ambiguous_signature_and_out_of_frame_geometry_fail() { + let overlap = snapshot( + vec![reporter_window()], + vec![ + output("DP-1", rect(0, 0, 2560, 1440), json!(1.5)), + output("HDMI-1", rect(0, 0, 2560, 1440), json!(1.5)), + ], + ); + let error = overlap + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 3840, 2160) + .unwrap_err() + .to_string(); + assert!(error.contains("overlapping or mirrored"), "{error}"); + + let ambiguous = snapshot( + vec![window( + UUID, + rect(1, 0, 200, 200), + rect(1, 0, 200, 200), + "DP-1", + )], + vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.0))], + ); + let error = ambiguous + .map_element_click(&target([0, 0, 200, 200], (110, 110)), 1000, 800) + .unwrap_err() + .to_string(); + assert!( + error.contains("ambiguous KWin coordinate signature"), + "{error}" + ); + + let error = ambiguous + .map_element_click(&target([0, 0, 200, 200], (250, 10)), 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("outside"), "{error}"); + let error = ambiguous + .map_element_click(&target([50, 50, 20, 20], (55, 55)), 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("matches neither"), "{error}"); + + let outside_output = snapshot( + vec![reporter_window()], + vec![output("DP-1", rect(1000, 1000, 100, 100), json!(1.0))], + ); + let error = outside_output + .map_element_click(&target([0, 0, 800, 600], (32, 16)), 100, 100) + .unwrap_err() + .to_string(); + assert!( + error.contains("outside every active KWin output"), + "{error}" + ); + } + + #[test] + fn same_pid_requires_unique_frame_geometry_and_explicit_focus_is_exact() { + let mut first = window( + UUID, + rect(100, 100, 200, 200), + rect(100, 100, 200, 200), + "DP-1", + ); + let mut second = window( + OTHER_UUID, + rect(400, 100, 200, 200), + rect(400, 100, 200, 200), + "DP-1", + ); + first["active"] = json!(false); + second["active"] = json!(false); + let outputs = vec![output("DP-1", rect(0, 0, 1000, 800), json!(1.0))]; + let ambiguous = snapshot(vec![first.clone(), second.clone()], outputs.clone()); + let mut pid_target = target([0, 0, 200, 200], (10, 10)); + pid_target.window_id = None; + let error = ambiguous + .map_element_click(&pid_target, 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("multiple same-PID KWin windows"), "{error}"); + + second["active"] = json!(true); + let focused = snapshot(vec![first.clone(), second], outputs.clone()); + let error = focused + .map_element_click(&pid_target, 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("multiple same-PID KWin windows"), "{error}"); + + // Equal-size global frames still associate by their distinct origins. + pid_target.frame.x = 400; + pid_target.frame.y = 100; + pid_target.center = (410, 110); + assert_eq!( + focused + .map_element_click(&pid_target, 1000, 800) + .unwrap() + .logical, + (410, 110) + ); + pid_target.window_id = Some(kwin_window_id_from_uuid(UUID)); + let error = focused + .map_element_click(&pid_target, 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("different KWin window"), "{error}"); + let mut mismatched_identity = target([0, 0, 200, 200], (10, 10)); + mismatched_identity.pid = Some(PID + 1); + let error = focused + .map_element_click(&mismatched_identity, 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("root PID disagree"), "{error}"); + + let not_focused = snapshot(vec![first], outputs); + let mut exact_target = target([0, 0, 200, 200], (10, 10)); + exact_target.require_exact_focus = true; + let error = not_focused + .map_element_click(&exact_target, 1000, 800) + .unwrap_err() + .to_string(); + assert!(error.contains("app-only focus is insufficient"), "{error}"); + } + + #[test] + fn distinct_same_pid_client_sizes_allow_local_frame_association() { + let first = reporter_window(); + let second = window( + OTHER_UUID, + rect(1100, 308, 600, 428), + rect(1100, 336, 600, 400), + "DP-1", + ); + let snapshot = snapshot( + vec![first, second], + vec![output("DP-1", rect(0, 0, 2560, 1440), json!(1.5))], + ); + let mut target = target([0, 0, 600, 400], (32, 16)); + target.window_id = None; + assert_eq!( + snapshot + .map_element_click(&target, 3840, 2160) + .unwrap() + .logical, + (1132, 352) + ); + } + + #[test] + fn snapshot_script_reads_only_coherent_frame_client_and_output_metadata() { + let script = kwin_window_script_source(":1.234", "/callback", "plugin").unwrap(); + for required in [ + "frameGeometry", + "clientGeometry", + "outputIdentity", + "activeOutputs", + "workspace.screens", + "read(output, \"enabled\")", + "devicePixelRatio", + "outputs: activeOutputs()", + "dialog: read(window, \"dialog\")", + "transient: read(window, \"transient\")", + "read(window, \"popupMenu\")", + "read(window, \"dropdownMenu\")", + "read(window, \"tooltip\")", + "read(window, \"comboBox\")", + "read(window, \"menu\")", + "window.desktops", + "workspace.currentDesktopForScreen(window.output)", + "read(window, \"onAllDesktops\")", + ] { + assert!(script.contains(required), "missing {required}"); + } + assert!(!script.contains("bufferGeometry")); + assert!(!script.contains("shadow")); + assert!(!script.contains("read(window, \"onCurrentDesktop\")")); + } +} + #[cfg(test)] mod callback_tests { use super::*; diff --git a/src/windowing/mod.rs b/src/windowing/mod.rs index 7e4a9b7..3b2261e 100644 --- a/src/windowing/mod.rs +++ b/src/windowing/mod.rs @@ -14,6 +14,8 @@ pub use target::{ window_permission_hint, }; #[allow(unused_imports)] +pub(crate) use types::app_id_leaf; +#[allow(unused_imports)] pub use types::{WindowBounds, WindowFocusResult, WindowInfo, WindowTarget}; #[cfg(test)] @@ -256,6 +258,104 @@ mod tests { assert_eq!(matched.window_id, 2); } + #[test] + fn app_id_alias_matches_reverse_dns_leaf_case_insensitively() { + let windows = vec![window(1, "KWrite", "org.kde.kwrite", "kwrite")]; + for app_id in ["kwrite", "KWRITE", "org.kde.kwrite"] { + let matched = resolve_window_target( + &windows, + &WindowTarget { + app_id: Some(app_id.to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(matched.window_id, 1); + } + + let ghostty = vec![window( + 2, + "Ghostty", + "com.mitchellh.ghostty.desktop", + "Ghostty", + )]; + assert!(resolve_window_target( + &ghostty, + &WindowTarget { + app_id: Some("ghostty".to_string()), + ..Default::default() + }, + ) + .is_ok()); + assert!(resolve_window_target( + &ghostty, + &WindowTarget { + app_id: Some("desktop".to_string()), + ..Default::default() + }, + ) + .is_err()); + } + + #[test] + fn exact_app_id_match_precedes_leaf_aliases() { + let windows = vec![ + window(1, "Other KWrite", "org.kde.kwrite", "kwrite"), + window(2, "Exact KWrite", "kwrite", "kwrite"), + ]; + + let matched = resolve_window_target( + &windows, + &WindowTarget { + app_id: Some("kwrite".to_string()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(matched.window_id, 2); + } + + #[test] + fn app_id_alias_rejects_ambiguous_applications() { + let windows = vec![ + window(1, "KDE KWrite", "org.kde.kwrite", "kwrite"), + window(2, "Other KWrite", "com.example.kwrite", "kwrite"), + ]; + + let error = resolve_window_target( + &windows, + &WindowTarget { + app_id: Some("kwrite".to_string()), + ..Default::default() + }, + ) + .unwrap_err() + .to_string(); + + assert!(error.contains("matched multiple applications")); + assert!(error.contains("full app_id")); + } + + #[test] + fn app_id_alias_allows_multiple_windows_of_one_application() { + let windows = vec![ + window(1, "First", "org.kde.kwrite", "kwrite"), + window(2, "Second", "org.kde.kwrite", "kwrite"), + ]; + + let matched = resolve_window_target( + &windows, + &WindowTarget { + app_id: Some("kwrite".to_string()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(matched.window_id, 1); + } + #[test] fn resolves_large_window_id_after_json_number_rounding() { let exact_window_id = 7_511_476_032_840_641_491; diff --git a/src/windowing/target.rs b/src/windowing/target.rs index 406e761..df600cb 100644 --- a/src/windowing/target.rs +++ b/src/windowing/target.rs @@ -142,15 +142,7 @@ pub fn resolve_window_target<'a>( } if let Some(app_id) = normalized_target(target.app_id.as_deref()) { - if let Some(window) = windows.iter().find(|window| { - window - .app_id - .as_deref() - .is_some_and(|value| value.eq_ignore_ascii_case(&app_id)) - }) { - return Ok(window); - } - bail!("No window matched app_id {app_id}."); + return resolve_app_id_target(windows, &app_id); } if let Some(wm_class) = normalized_target(target.wm_class.as_deref()) { @@ -181,6 +173,34 @@ pub fn resolve_window_target<'a>( bail!("Pass window_id, pid, app_id, wm_class, title, tty, terminal_pid, terminal_command, or terminal_cwd to target a window."); } +fn resolve_app_id_target<'a>(windows: &'a [WindowInfo], app_id: &str) -> Result<&'a WindowInfo> { + if let Some(window) = windows.iter().find(|window| { + window + .app_id + .as_deref() + .is_some_and(|value| value.eq_ignore_ascii_case(app_id)) + }) { + return Ok(window); + } + + if app_id.contains('.') { + bail!("No window matched app_id {app_id}."); + } + + let mut matches = windows.iter().filter(|window| { + window.app_id.as_deref().is_some_and(|value| { + crate::windowing::types::app_id_leaf(value).eq_ignore_ascii_case(app_id) + }) + }); + let Some(first) = matches.next() else { + bail!("No window matched app_id {app_id}."); + }; + if matches.any(|window| !same_optional_string(&first.app_id, &window.app_id)) { + bail!("app_id {app_id} matched multiple applications; use the full app_id."); + } + Ok(first) +} + fn resolve_window_id_target<'a>( windows: &'a [WindowInfo], target: &WindowTarget, diff --git a/src/windowing/types.rs b/src/windowing/types.rs index 8282d94..3f09cfc 100644 --- a/src/windowing/types.rs +++ b/src/windowing/types.rs @@ -27,6 +27,12 @@ pub struct WindowBounds { pub height: u32, } +pub(crate) fn app_id_leaf(app_id: &str) -> &str { + let app_id = app_id.trim(); + let app_id = app_id.strip_suffix(".desktop").unwrap_or(app_id); + app_id.rsplit('.').next().unwrap_or(app_id) +} + #[derive(Debug, Clone, Default, Deserialize, Serialize, JsonSchema)] pub struct WindowTarget { #[serde(default)] diff --git a/src/x11_display.rs b/src/x11_display.rs index 4e0c536..322d5a9 100644 --- a/src/x11_display.rs +++ b/src/x11_display.rs @@ -239,6 +239,11 @@ impl X11TypingGuard { let display = X11Display::connect()?; let conn = &display.conn; let focus = conn.get_input_focus()?.reply()?.focus; + // PointerRoot (1) is not a window: focus can follow pointer motion + // without an observable focus event on the current target. + if focus == 1 { + bail!("X11 keyboard monitoring does not support PointerRoot; input follows the window under the pointer"); + } // GetInputFocus stays unchanged during an active keyboard grab. Core // focus events still report NotifyGrab/NotifyUngrab, including when // the application itself reads keyboard input through XInput2.