diff --git a/mac/Sources/Wind/CaretTracker.swift b/mac/Sources/Wind/CaretTracker.swift index 86529280..4e9c9417 100644 --- a/mac/Sources/Wind/CaretTracker.swift +++ b/mac/Sources/Wind/CaretTracker.swift @@ -93,6 +93,7 @@ final class CaretTracker { guard on, let app = front, app.processIdentifier != getpid() else { return } pid = app.processIdentifier reEnhanced = false + isShell = false isTerminal = CaretTracker.terminalBundles.contains(app.bundleIdentifier ?? "") let el = AXUIElementCreateApplication(pid) AXUIElementSetMessagingTimeout(el, 0.25) @@ -128,6 +129,41 @@ final class CaretTracker { } CFRunLoopAddSource(loop, AXObserverGetRunLoopSource(obs), .defaultMode) Log.verbose("caret: attached to \(app.localizedName ?? "?") (\(pid))") + isShell = !isChromium && CaretTracker.isChromiumShell(app) + if isShell { wakeTree(el); Log.verbose("caret: Chromium shell, its tree woken") } + } + + /// Some Chromium shells refuse both switches (ChatGPT, com.openai.codex: AXManualAccessibility + /// unsupported, AXEnhancedUserInterface not implemented) and keep their web content's tree off: the + /// window is six empty groups, the focused element a native one in the title bar, so typing there + /// never moved the view (owner 2026-11-07). Reading the application element's role, subrole and + /// description switches it on, the way a screen reader's first look does (bisected 2026-11-07: no + /// read of the window's elements, no switch, no waiting, typing or clicking did). Repeated on every + /// key press, so the tree cannot go quiet while the owner types. + private func wakeTree(_ app: AXUIElement) { + for name in [kAXRoleAttribute, kAXSubroleAttribute, kAXDescriptionAttribute] { + var v: CFTypeRef? + AXUIElementCopyAttributeValue(app, name as CFString, &v) + } + } + + /// The app carries Chromium renderer helpers (Electron, ChatGPT's own shell) and is not a browser + /// Wind already switches on. Looked up once per bundle. + nonisolated(unsafe) private static var shellCache: [String: Bool] = [:] + private static func isChromiumShell(_ app: NSRunningApplication) -> Bool { + guard let url = app.bundleURL else { return false } + if let hit = shellCache[url.path] { return hit } + // Electron: Frameworks/ Helper (Renderer).app. Other shells keep the helpers in the + // framework: ChatGPT's Codex Framework.framework/Helpers/Codex (Aperitif Renderer).app. + let fm = FileManager.default, fw = url.appendingPathComponent("Contents/Frameworks") + let top = (try? fm.contentsOfDirectory(atPath: fw.path)) ?? [] + let isRenderer = { (name: String) in name.hasSuffix("Renderer).app") } + let found = top.contains(where: isRenderer) || top.contains { name in + name.hasSuffix(".framework") && + ((try? fm.contentsOfDirectory(atPath: fw.appendingPathComponent(name).appendingPathComponent("Helpers").path)) ?? []).contains(where: isRenderer) + } + shellCache[url.path] = found + return found } /// Chromium's full accessibility mode slows window managers and costs CPU, so it is on only @@ -175,7 +211,7 @@ final class CaretTracker { if notification == kAXFocusedUIElementChangedNotification as String { if let c = caretInfo(of: el) { if !parkedInTerminal(c, el) { publish(.caret, c.rect) } - } else if let r = frame(of: el) { + } else if let r = frame(of: el), !keepsCaret(el, r) { publish(.focus, r) } } else if let c = caretInfo(of: el), !parkedInTerminal(c, el) { @@ -218,6 +254,7 @@ final class CaretTracker { private func poll() { guard live, pollTimer?.isValid != true, Date() >= pollPausedUntil else { return } if observer == nil { DispatchQueue.main.async { [weak self] in self?.attachFrontmost() } } + if isShell, let app = appElement { wakeTree(app) } pollTimer = Timer.scheduledTimer(withTimeInterval: 0.04, repeats: false) { [weak self] _ in guard let self else { return } guard let el = self.focusedElement() else { @@ -238,7 +275,7 @@ final class CaretTracker { let started = Date() if let c = self.caretInfo(of: el) { if !self.parkedInTerminal(c, el) { self.publish(.caret, c.rect) } - } else if let r = self.frame(of: el) { + } else if let r = self.frame(of: el), !self.keepsCaret(el, r) { // A focused control without a caret (button, link, list): keyboard focus. Chromium // freshly launched sent no focus notifications, so Tab moved the focus and the view // stayed put (e2e_focus, 2026-11-01); the poll after the key finds it. An unchanged @@ -249,9 +286,20 @@ final class CaretTracker { } } private var pollPausedUntil = Date.distantPast + private var isShell = false // a Chromium shell outside chromiumBundles (wakeTree) private var reEnhanced = false // the Chromium switch was re-applied once for this app private var pollTimer: Timer? + /// A text field whose caret could not be measured this time, with Wind's last caret inside it: keep + /// that caret rather than jump to the whole field (ChatGPT, a read now and then while typing). + private func keepsCaret(_ el: AXUIElement, _ field: Rc) -> Bool { + guard published.kind == .caret || pending?.0 == .caret else { return false } + let c = pending?.0 == .caret ? pending!.1 : published.rect + var v: CFTypeRef? + guard AXUIElementCopyAttributeValue(el, kAXSelectedTextRangeAttribute as CFString, &v) == .success else { return false } + return c.l >= field.l - 2 && c.r <= field.r + 2 && c.t >= field.t - 2 && c.b <= field.b + 2 + } + private func publish(_ kind: TrackKind, _ r: Rc) { if let p = pending, p.0 == kind, p.1 == r { return } // a repeat does not restart the wait Log.verbose("caret raw: \(kind) \(Int(r.l)),\(Int(r.t)) \(Int(r.width))x\(Int(r.height))") @@ -302,9 +350,66 @@ final class CaretTracker { let r = bounds(el, CFRange(location: range.location - 1, length: 1)), r.height > 0 { return (Rc(l: r.r, t: r.t, r: r.r + 1, b: r.b), true) } + if range.length == 0, let r = textRunCaret(el, range.location) { return (r, true) } return nil } + /// The insertion point from the frames of the text runs inside the field, for apps whose text has + /// no character bounds: ChatGPT (a Chromium shell without full accessibility) answers every + /// AXBoundsForRange with an empty rect, but its static text runs carry real frames (2026-11-07). + /// Exact at a run's end, where typing happens; inside a run the place is estimated from the share + /// of characters before it, line by line for a run that wraps. + private func textRunCaret(_ el: AXUIElement, _ index: Int) -> Rc? { + guard isShell else { return nil } // a walk per key: only where it is needed + var val: CFTypeRef? + guard AXUIElementCopyAttributeValue(el, kAXValueAttribute as CFString, &val) == .success, + let full = val as? String else { return nil } + let all = full as NSString + var runs: [(start: Int, len: Int, rect: CGRect)] = [] + var from = 0, seen = 0 + func walk(_ e: AXUIElement, _ depth: Int) { + guard seen < 300, depth < 12 else { return } + seen += 1 + var role: CFTypeRef? + AXUIElementCopyAttributeValue(e, kAXRoleAttribute as CFString, &role) + if (role as? String) == (kAXStaticTextRole as String) { + var tv: CFTypeRef?, fv: CFTypeRef? + guard AXUIElementCopyAttributeValue(e, kAXValueAttribute as CFString, &tv) == .success, + let text = tv as? String, !text.isEmpty, + AXUIElementCopyAttributeValue(e, "AXFrame" as CFString, &fv) == .success, let fv, + CFGetTypeID(fv) == AXValueGetTypeID() else { return } + var rect = CGRect.zero + guard AXValueGetValue(fv as! AXValue, .cgRect, &rect), rect.height > 0 else { return } + let at = all.range(of: text, range: NSRange(location: from, length: all.length - from)) + guard at.location != NSNotFound else { return } + runs.append((at.location, at.length, rect)); from = at.location + at.length + return + } + var kids: CFTypeRef? + guard AXUIElementCopyAttributeValue(e, kAXChildrenAttribute as CFString, &kids) == .success, + let list = kids as? [AXUIElement] else { return } + for k in list { walk(k, depth + 1) } + } + walk(el, 0) + // A run leaves out the spaces that end its line: a caret after them lies past the run's end, + // by the run's average character width per space (the view jumped to the whole field at every + // space before, owner 2026-11-07). + var extra = 0 + var hit = runs.first(where: { index >= $0.start && index <= $0.start + $0.len }) + if hit == nil, let last = runs.last(where: { $0.start + $0.len < index }) { + let gap = all.substring(with: NSRange(location: last.start + last.len, length: index - last.start - last.len)) + if gap.allSatisfy({ $0 == " " || $0 == "\u{a0}" }) { hit = last; extra = gap.count } + } + guard let run = hit else { return nil } + let lineH = runs.map(\.rect.height).min()! + let lines = max(1, Int((run.rect.height / lineH).rounded())) + let pos = Double(min(index, run.start + run.len) - run.start) / Double(max(1, run.len)) * Double(lines) + let line = min(lines - 1, Int(pos)) + let x = run.rect.minX + run.rect.width * (pos - Double(line)) + Double(extra) * run.rect.width * Double(lines) / Double(max(1, run.len)) + let top = run.rect.minY + Double(line) * run.rect.height / Double(lines) + return Rc(l: x, t: top, r: x + 1, b: top + run.rect.height / Double(lines)) + } + private func bounds(_ el: AXUIElement, _ range: CFRange) -> Rc? { var range = range guard let rv = AXValueCreate(.cfRange, &range) else { return nil } diff --git a/mac/VERSION b/mac/VERSION index 7d856835..d1d899fa 100644 --- a/mac/VERSION +++ b/mac/VERSION @@ -1 +1 @@ -0.5.4 +0.5.5 diff --git a/mac/docs/architecture.md b/mac/docs/architecture.md index 3080d2b8..10551b1b 100644 --- a/mac/docs/architecture.md +++ b/mac/docs/architecture.md @@ -558,6 +558,20 @@ rest it trailed by 9 pt and carried it in one frame). Explorer, zooming scenario - **Chromium.** Chromium browsers get `AXEnhancedUserInterface` while they are frontmost (switched off when they lose focus and when Wind quits). Without it Chrome sent no caret after a zoom-in; turning it on only at zoom-in was too late for the first keys. +- **Chromium shells (ChatGPT, 2026-11-07).** An app with Chromium renderer helpers in its bundle + (Electron's `Frameworks/ Helper (Renderer).app`, or a framework's `Helpers/`, like ChatGPT's + `Codex Framework.framework`) that is not in the browser list. ChatGPT (`com.openai.codex`) refuses + both switches (`AXManualAccessibility` -25205, `AXEnhancedUserInterface` -25208 on the app) and keeps + its web content out of the tree: the window is six empty groups and the focused element a native + one in the title bar. Reading the application element's role, subrole and description turns the tree + on (bisected: reads of the window's elements, setting either switch on the content view, waiting, + typing or clicking did not). Wind does that on attach and again on every key press while zoomed. + Even then its text has no character bounds (`AXBoundsForRange` and the text-marker bounds give an + empty rect), so the caret comes from the frames of the field's `AXStaticText` runs: exact at a + run's end, estimated by character share inside a run (line by line when it wraps), and past the + run's end by one average character per trailing space (a run leaves them out). A text field whose + caret cannot be measured on one read keeps Wind's last caret inside it instead of publishing the + whole field as focus (the view jumped to the field and back while typing). - **Key-press poll.** Every key press while zoomed also polls the focused element 40 ms later, so tracking never depends on an app's notifications alone. - **Tests.** `scripts/e2e_caret.py` types real keys (the Karabiner virtual keyboard in