diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h index 33493bec41..6584cc2849 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.h @@ -11,6 +11,7 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionCallsKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionPendingKey; FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionMissedKey; +FOUNDATION_EXPORT NSString *const RunnerAXSnapshotDeepExtensionBlockedKey; /// Keys of the `customActions` dictionary in the snapshot response: how many /// merged elements were eligible for an action read, and how many the bounded @@ -112,6 +113,12 @@ FOUNDATION_EXPORT NSString *const RunnerAXSnapshotCustomActionsBlockedKey; /// Total reads refused by single-flight admission. + (NSInteger)customActionReadBlockedCount; +/// Contained snapshot-request counters, exposed so the unit bundle can pin the +/// no-queued-work invariant for a wedged private AX snapshot request. ++ (NSInteger)snapshotReadsInFlight; ++ (NSInteger)snapshotReadDispatchCount; ++ (NSInteger)snapshotReadBlockedCount; + /// The shared AX client (`XCUIDevice.accessibilityInterface`), or nil when the /// private interface is unavailable. + (nullable id)accessibilityClient; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m index 81d1692e13..f8ef7cc35e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerAXSnapshotBridge.m @@ -14,6 +14,7 @@ NSString *const RunnerAXSnapshotDeepExtensionNodesAddedKey = @"nodesAdded"; NSString *const RunnerAXSnapshotDeepExtensionPendingKey = @"pendingFrontiers"; NSString *const RunnerAXSnapshotDeepExtensionMissedKey = @"missedFrontiers"; +NSString *const RunnerAXSnapshotDeepExtensionBlockedKey = @"blockedFrontiers"; NSString *const RunnerAXSnapshotCustomActionsKey = @"customActions"; NSString *const RunnerAXSnapshotCustomActionsReadKey = @"read"; @@ -30,6 +31,25 @@ /// capture's budget and starve every remaining candidate, turning a bounded /// enrichment into a capture-length stall. static const NSTimeInterval RunnerAXCustomActionReadTimeout = 1.0; +static const NSTimeInterval RunnerAXSnapshotReadTimeout = 2.0; + +/// Snapshot requests are synchronous private-XPC calls too. Keep them off the +/// XCTest main thread and refuse a second request while a timed-out call is +/// still outstanding; the client call itself cannot be cancelled. +static dispatch_queue_t RunnerAXSnapshotReadQueue(void) +{ + static dispatch_queue_t queue; + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + queue = dispatch_queue_create("com.callstack.agentdevice.runner.ax-snapshot", + DISPATCH_QUEUE_SERIAL); + }); + return queue; +} + +static atomic_int RunnerAXSnapshotReadsInFlight = 0; +static atomic_long RunnerAXSnapshotReadDispatches = 0; +static atomic_long RunnerAXSnapshotReadBlocked = 0; /// The AX call is a synchronous XPC round trip that cannot be cancelled once /// issued, so the deadline above only frees the CALLER — the call itself keeps @@ -85,6 +105,16 @@ static dispatch_queue_t RunnerAXCustomActionReadQueue(void) @implementation RunnerAXSnapshotFrontier @end +@interface RunnerAXSnapshotBridge () ++ (nullable id)requestSnapshotContainedFromClient:(id)axClient + target:(id)target + attributes:(NSArray *)attributes + maxDepth:(NSInteger)maxDepth + maxNodes:(NSInteger)maxNodes + deadline:(nullable NSDate *)deadline + error:(NSError **)error; +@end + @implementation RunnerAXSnapshotBridge + (NSDictionary *)snapshotTreeForApplication:(XCUIApplication *)application @@ -127,12 +157,13 @@ @implementation RunnerAXSnapshotBridge @try { NSArray *attributes = [self snapshotAttributes]; NSError *error = nil; - id root = [self requestSnapshotFromClient:axClient - target:target - attributes:attributes - maxDepth:maxDepth - maxNodes:maxNodes - error:&error]; + id root = [self requestSnapshotContainedFromClient:axClient + target:target + attributes:attributes + maxDepth:maxDepth + maxNodes:maxNodes + deadline:deadline + error:&error]; if (nil == root) { return [self failure:error.localizedDescription ?: @"AX snapshot request returned nil"]; } @@ -222,10 +253,11 @@ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray 0) { if (callsUsed >= callsAllowed || *nodeCount >= maxNodes || (nil != deadline && deadline.timeIntervalSinceNow <= 0)) { @@ -242,13 +274,22 @@ + (nullable NSDictionary *)extendSnapshotFrontiers:(NSMutableArray *keyPaths = @[ @@ -649,6 +761,21 @@ + (NSInteger)customActionReadBlockedCount return (NSInteger)atomic_load(&RunnerAXCustomActionReadBlocked); } ++ (NSInteger)snapshotReadsInFlight +{ + return atomic_load(&RunnerAXSnapshotReadsInFlight); +} + ++ (NSInteger)snapshotReadDispatchCount +{ + return (NSInteger)atomic_load(&RunnerAXSnapshotReadDispatches); +} + ++ (NSInteger)snapshotReadBlockedCount +{ + return (NSInteger)atomic_load(&RunnerAXSnapshotReadBlocked); +} + + (nullable id)accessibilityClient { return [self objectFrom:XCUIDevice.sharedDevice selectorName:@"accessibilityInterface"]; diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift index df933a24ab..2106132d7e 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+AXSnapshotFallback.swift @@ -31,6 +31,15 @@ extension RunnerTests { guard let pendingFrontiers, let missedFrontiers else { return true } return pendingFrontiers > 0 || missedFrontiers > 0 } + + static func privateAXContainmentFailure(blockedFrontiers: Int?) -> SnapshotCaptureFailure? { + guard let blockedFrontiers, blockedFrontiers > 0 else { return nil } + return SnapshotCaptureFailure( + code: "IOS_SNAPSHOT_AX_CONTAINMENT_FAILED", + message: "The private AX snapshot could not complete a contained accessibility read.", + hint: "Retry after the accessibility server responds; no partial tree was treated as complete." + ) + } /// Deep React Native trees make the AX server reject bulk snapshot requests outright with /// kAXErrorIllegalArgument once the requested depth crosses a tree-size-dependent limit /// (observed between depth 56 and 64 on the Bluesky Home feed; the limit moves with live @@ -165,7 +174,7 @@ extension RunnerTests { target: SnapshotCaptureTarget, hint: CaptureHint, deadline: Date = .distantFuture - ) -> SnapshotAcquisition? { + ) throws -> SnapshotAcquisition? { #if os(iOS) && targetEnvironment(simulator) let app = target.app let requestedDepth = hint.rawTraversalDepth ?? 64 @@ -238,6 +247,11 @@ extension RunnerTests { // depth-limited would send agents chasing deeper content that is not // there. Pending or missed frontiers keep the depth-limited verdict. let deepExtension = response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any] + if let failure = Self.privateAXContainmentFailure( + blockedFrontiers: deepExtension?[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int + ) { + throw failure + } let depthLimited = Self.privateAXDepthLimited( effectiveDepth: effectiveDepth, requestedDepth: requestedDepth, diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift index a96560e417..0e695931a8 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+CommandExecution.swift @@ -422,6 +422,19 @@ extension RunnerTests { guard let x = command.x, let y = command.y else { return Response(ok: false, error: ErrorPayload(message: "readText requires x and y")) } + if command.inspectPoint == true { + let inspection = readPointAt(app: activeApp, x: x, y: y) + guard inspection.complete else { + return Response( + ok: false, + error: ErrorPayload( + code: "COMMAND_FAILED", + message: "Point inspection could not prove a miss because its accessibility capture was truncated" + ) + ) + } + return Response(ok: true, data: DataPayload(text: inspection.text, elements: inspection.elements)) + } guard let text = readTextAt(app: activeApp, x: x, y: y) else { return Response(ok: false, error: ErrorPayload(message: "readText did not resolve text")) } diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift index 002459b19c..fb9456fbc4 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Interaction.swift @@ -320,46 +320,212 @@ extension RunnerTests { func readTextAt(app: XCUIApplication, x: Double, y: Double) -> String? { let point = CGPoint(x: x, y: y) let textInputCandidates = textInputCandidatesAt(app: app, point: point) - for element in textInputCandidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { - return text - } + let candidates = app.descendants(matching: .any).allElementsBoundByIndex + .filter { $0.exists && !$0.frame.isEmpty && $0.frame.contains(point) } + .sorted(by: smallestElementFirst) + return firstReadableText(in: textInputCandidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: false) + } + + /// Keep live point text resolution identical for legacy `get text` and point inspection. + func firstReadableText(in elements: [XCUIElement], preferredOnly: Bool) -> String? { + for element in elements { + if preferredOnly && !prefersExpandedTextRead(element) { continue } + if let text = readableText(for: element) { return text } } + return nil + } + func readPointAt( + app: XCUIApplication, + x: Double, + y: Double + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool) { +#if os(iOS) && targetEnvironment(simulator) + // System-owned sheets can make XCTest's unbounded descendants query hold + // the main thread past the command watchdog. The Simulator-only private AX + // bridge supplies the bounded accessibility tree used by snapshot recovery; + // filter that tree at the requested point before touching the XCTest query. + if let inspection = privateAXPointInspection(app: app, x: x, y: y) { + return inspection + } +#endif + let point = CGPoint(x: x, y: y) + let textInputCandidates = textInputCandidatesAt(app: app, point: point) let candidates = app.descendants(matching: .any).allElementsBoundByIndex .filter { element in element.exists && !element.frame.isEmpty && element.frame.contains(point) } .sorted(by: smallestElementFirst) - for element in candidates where prefersExpandedTextRead(element) { - if let text = readableText(for: element) { - return text - } + // Resolve the legacy text semantics before materializing inspection payloads. + let text = firstReadableText(in: textInputCandidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: true) + ?? firstReadableText(in: candidates, preferredOnly: false) + + let elements = Array(candidates.prefix(24)).map { element in + let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) + let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) + let value = String(describing: element.value ?? "") + .trimmingCharacters(in: .whitespacesAndNewlines) + return PointInspectionElementPayload( + text: readableText(for: element), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + type: elementTypeName(element.elementType), + role: nil, + value: value.isEmpty ? nil : value, + frame: SnapshotRect(element.frame), + hittable: nil + ) } - for element in candidates { - if let text = readableText(for: element) { - return text + return (text, elements, true) + } + +#if os(iOS) && targetEnvironment(simulator) + func privateAXPointInspection( + app: XCUIApplication, + x: Double, + y: Double + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool)? { + let response = RunnerAXSnapshotBridge.snapshotTree( + for: app, + maxDepth: 56, + maxNodes: 5_000, + deepExtensionCallLimit: 4, + customActionLimit: 0, + deadline: Date().addingTimeInterval(8) + ) + guard (response["ok"] as? NSNumber)?.boolValue == true, + let root = response["root"] as? [String: Any] + else { + return nil + } + let deepExtension = response[RunnerAXSnapshotDeepExtensionKey] as? [String: Any] + let completeDeepExtension: Bool + if let deepExtension { + if let pending = deepExtension[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, + let missed = deepExtension[RunnerAXSnapshotDeepExtensionMissedKey] as? Int + { + let blocked = deepExtension[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int ?? 0 + completeDeepExtension = pending == 0 && missed == 0 && blocked == 0 + } else { + completeDeepExtension = false + } + } else { + // The bridge omits this field only when the initial tree had no capped + // frontiers to extend. + completeDeepExtension = true + } + return privateAXPointInspection( + root: root, + point: CGPoint(x: x, y: y), + truncated: (response["truncated"] as? NSNumber)?.boolValue == true, + completeDeepExtension: completeDeepExtension + ) + } + + func privateAXPointInspection( + root: [String: Any], + point: CGPoint, + truncated: Bool = false, + completeDeepExtension: Bool = true + ) -> (text: String?, elements: [PointInspectionElementPayload], complete: Bool) { + guard !truncated && completeDeepExtension else { return (nil, [], false) } + var candidates: [(payload: PointInspectionElementPayload, area: CGFloat)] = [] + + func visit(_ raw: [String: Any]) { + let frame = privateAXRect(raw["frame"]) + if !frame.isEmpty && frame.contains(point) { + let fields = privateAXFields(raw) + let rawType = fields.rawType + // The private tree includes the owning application at the full screen + // frame. Its label is the app name, not a control under the point, so + // returning it can make point-based system-UI probes tap a false match. + if rawType == XCUIElement.ElementType.application.rawValue { + for child in raw["children"] as? [[String: Any]] ?? [] { + visit(child) + } + return + } + let type = fields.elementType.map(elementTypeName) ?? "Element(\(rawType))" + let label = fields.label.isEmpty ? nil : fields.label + let identifier = fields.identifier.isEmpty ? nil : fields.identifier + let value = fields.value.isEmpty ? nil : fields.value + let text = pointReadableText( + type: type, + label: label, + identifier: identifier, + value: value + ) + candidates.append(( + PointInspectionElementPayload( + text: text, + label: label, + identifier: identifier, + type: type, + role: nil, + value: value, + frame: SnapshotRect(frame), + hittable: nil + ), + max(1, frame.width * frame.height) + )) + } + for child in raw["children"] as? [[String: Any]] ?? [] { + visit(child) } } - return nil + + visit(root) + let orderedCandidates = candidates + .sorted { left, right in + if left.area != right.area { return left.area < right.area } + if left.payload.frame.y != right.payload.frame.y { + return left.payload.frame.y < right.payload.frame.y + } + if left.payload.frame.x != right.payload.frame.x { + return left.payload.frame.x < right.payload.frame.x + } + return (left.payload.type ?? "") < (right.payload.type ?? "") + } + let textInputTypes: Set = ["TextField", "SecureTextField", "SearchField", "TextView"] + // Keep text resolution independent of the bounded descriptor response. + // A large accessibility surface can have more than 24 smaller controls at + // one point while the underlying text input still owns the correct value. + let text = orderedCandidates.first(where: { + textInputTypes.contains($0.payload.type ?? "") && $0.payload.text != nil + })?.payload.text + ?? orderedCandidates.compactMap { $0.payload.text }.first + let elements = orderedCandidates.prefix(24).map(\.payload) + return (text, elements, true) + } + + func pointReadableText( + type: String, + label: String?, + identifier: String?, + value: String? + ) -> String? { + if ["TextField", "SecureTextField", "SearchField", "TextView"].contains(type) { + return value ?? label ?? identifier + } + return label ?? value ?? identifier } +#endif private func readableText(for element: XCUIElement) -> String? { let label = element.label.trimmingCharacters(in: .whitespacesAndNewlines) let identifier = element.identifier.trimmingCharacters(in: .whitespacesAndNewlines) let valueText = String(describing: element.value ?? "") .trimmingCharacters(in: .whitespacesAndNewlines) - switch element.elementType { - case .textField, .secureTextField, .searchField, .textView: - if !valueText.isEmpty { return valueText } - if !label.isEmpty { return label } - return identifier.isEmpty ? nil : identifier - default: - if !label.isEmpty { return label } - if !valueText.isEmpty { return valueText } - return identifier.isEmpty ? nil : identifier - } + return pointReadableText( + type: elementTypeName(element.elementType), + label: label.isEmpty ? nil : label, + identifier: identifier.isEmpty ? nil : identifier, + value: valueText.isEmpty ? nil : valueText + ) } private func prefersExpandedTextRead(_ element: XCUIElement) -> Bool { diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift index 73c7c52761..a7b6f633ca 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+Models.swift @@ -184,6 +184,7 @@ extension Command { struct Command: Codable { let command: CommandType + let inspectPoint: Bool? let commandId: String? let statusCommandId: String? let appBundleId: String? @@ -391,6 +392,7 @@ struct DataPayload: Codable { var message: String? var imageBase64: String? var text: String? + var elements: [PointInspectionElementPayload]? var found: Bool? var items: [String]? var nodes: [PresentedNode]? @@ -451,6 +453,17 @@ struct DataPayload: Codable { var screenshotMetadata: ScreenshotMetadataPayload? } +struct PointInspectionElementPayload: Codable { + let text: String? + let label: String? + let identifier: String? + let type: String? + let role: String? + let value: String? + let frame: SnapshotRect + let hittable: Bool? +} + /// `kind` mirrors the TS `SnapshotKeyboardBandFact`: "visible" carries `frame`, "unmeasurable" /// carries `reason`, and "absent" carries nothing because there is nothing to say. `frame` is in the /// app's own orientation space — the same space `SnapshotGeometrySpace` publishes every node rect in diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift index f6365d592c..e08b229b2d 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+PrivateAXPresentation.swift @@ -46,7 +46,7 @@ extension RunnerTests { } } - private func privateAXFields(_ raw: [String: Any]) -> PrivateAXFields { + func privateAXFields(_ raw: [String: Any]) -> PrivateAXFields { let rawType = privateAXPresentationInt(raw["type"]) ?? 0 return PrivateAXFields( rect: privateAXRect(raw["frame"]), @@ -80,7 +80,7 @@ extension RunnerTests { actions: fields.actions) } - private func privateAXPresentationString(_ value: Any?) -> String { + func privateAXPresentationString(_ value: Any?) -> String { guard let value else { return "" } return (value as? String ?? String(describing: value)) .trimmingCharacters(in: .whitespacesAndNewlines) diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift index 8ef6bc8ca7..085975dbab 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/RunnerTests+SnapshotCapturePlan.swift @@ -492,7 +492,7 @@ extension RunnerTests { return (sweep.acquisition, sweep.outcome) case .privateAX: return ( - self.privateAXSnapshotAcquisition( + try self.privateAXSnapshotAcquisition( target: target, hint: hint, deadline: deadline diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift index 4e4fd9ff44..56115355c9 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+AXSnapshotFallbackTests.swift @@ -5,6 +5,15 @@ import AgentDeviceSnapshotPresentation // MARK: - In-bundle unit tests extension RunnerTests { + func testPrivateAXContainmentFailureIsTypedAndNotADepthLimit() { + XCTAssertNil(Self.privateAXContainmentFailure(blockedFrontiers: nil)) + XCTAssertNil(Self.privateAXContainmentFailure(blockedFrontiers: 0)) + XCTAssertEqual( + Self.privateAXContainmentFailure(blockedFrontiers: 1)?.code, + "IOS_SNAPSHOT_AX_CONTAINMENT_FAILED" + ) + } + func testPrivateAXAttemptDepthsAppliesRememberedDepth() { XCTAssertEqual( Self.privateAXAttemptDepths(requestedDepth: 64, rememberedDepth: nil), @@ -24,12 +33,10 @@ extension RunnerTests { XCTAssertEqual(Self.privateAXAttemptDepths(requestedDepth: 24, rememberedDepth: 56), [24, 12]) } - /// Executed producer contract for the #1627 review blocker: a frontier whose - /// live element vanished, and one whose re-rooted request fails, must BOTH - /// count as missed — an all-miss extension reporting itself drained would - /// present a capped capture as complete. Goes red if either miss-path - /// increment in extendSnapshotFrontiers is removed. - func testDeepExtensionCountsMissedFrontiers() { + /// A vanished element is a missed frontier; a contained request failure is + /// a blocked frontier and must fail the backend instead of degrading to a + /// benign depth-limit verdict. + func testDeepExtensionSeparatesMissedAndBlockedFrontiers() { // Element vanished (list churn between serialization and extension): the // fabricated snapshot answers nil for accessibilityElement — missed, and // no request call is consumed. (An explicit nil property: bare NSObject @@ -38,7 +45,7 @@ extension RunnerTests { orphan.snapshot = FrontierSnapshotWithoutElementForTesting() orphan.node = NSMutableDictionary() // Re-rooted request fails: the element resolves but the client cannot - // serve requestSnapshotForElement — one consumed call AND a miss. + // serve requestSnapshotForElement — one consumed call AND a blocked read. let unreachable = RunnerAXSnapshotFrontier() unreachable.snapshot = FrontierSnapshotWithElementForTesting() unreachable.node = NSMutableDictionary() @@ -58,15 +65,13 @@ extension RunnerTests { deadline: nil ) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 2) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionMissedKey] as? Int, 1) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionBlockedKey] as? Int, 1) XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionCallsKey] as? Int, 1) - XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 0) + XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionPendingKey] as? Int, 1) XCTAssertEqual(outcome?[RunnerAXSnapshotDeepExtensionNodesAddedKey] as? Int, 0) - XCTAssertFalse(truncated.boolValue) - // And the consumer verdict over exactly this outcome: still depth-limited. - XCTAssertTrue( - Self.privateAXDepthLimited( - effectiveDepth: 56, requestedDepth: 64, pendingFrontiers: 0, missedFrontiers: 2)) + XCTAssertTrue(truncated.boolValue) + XCTAssertNotNil(Self.privateAXContainmentFailure(blockedFrontiers: 1)) } func testPrivateAXDepthLimitedRequiresEveryFrontierResolved() { @@ -291,6 +296,65 @@ extension RunnerTests { RunnerAXSnapshotBridge.customActionReadDispatchCount(), dispatchesBefore + 2) } + func testHungInitialSnapshotRequestIsContainedAndRecovers() { + let client = HungSnapshotAXClientForTesting() + let dispatchesBefore = RunnerAXSnapshotBridge.snapshotReadDispatchCount() + let blockedBefore = RunnerAXSnapshotBridge.snapshotReadBlockedCount() + defer { client.release() } + + let firstStarted = Date() + let first = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(0.25) + ) + XCTAssertEqual(first["ok"] as? Bool, false) + XCTAssertGreaterThanOrEqual(-firstStarted.timeIntervalSinceNow, 0.2) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadsInFlight(), 1) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 1) + + // Repeated captures fail closed without adding work behind the wedged XPC call. + for _ in 0..<4 { + let repeated = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(1) + ) + XCTAssertEqual(repeated["ok"] as? Bool, false) + } + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 1) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadBlockedCount(), blockedBefore + 4) + + client.release() + let drained = expectation(description: "wedged initial request drains") + DispatchQueue.global().async { + while RunnerAXSnapshotBridge.snapshotReadsInFlight() > 0 { + usleep(20_000) + } + drained.fulfill() + } + wait(for: [drained], timeout: 5) + + _ = RunnerAXSnapshotBridge.snapshotTree( + withClient: client, + target: NSObject(), + maxDepth: 4, + maxNodes: 16, + deepExtensionCallLimit: 0, + customActionLimit: 0, + deadline: Date().addingTimeInterval(1) + ) + XCTAssertEqual(RunnerAXSnapshotBridge.snapshotReadDispatchCount(), dispatchesBefore + 2) + } + /// The element budget bounds how many elements we read; these caps bound what /// any ONE element can put in the response. Clipping must be reported, since /// a clipped list looks exactly like a complete one. @@ -484,6 +548,38 @@ extension RunnerTests { } } +/// Stands in for a snapshot AX client whose request never returns. `release()` +/// lets the contained request finish so timeout recovery is observable. +private final class HungSnapshotAXClientForTesting: NSObject { + private let gate = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var released = false + + @objc(requestSnapshotForElement:attributes:parameters:error:) + func requestSnapshot( + forElement element: Any, + attributes: Any, + parameters: Any, + error: NSErrorPointer + ) -> Any? { + lock.lock() + let alreadyReleased = released + lock.unlock() + if !alreadyReleased { + gate.wait() + } + return nil + } + + func release() { + lock.lock() + defer { lock.unlock() } + guard !released else { return } + released = true + gate.signal() + } +} + /// Stands in for an AX client whose `attributesForElement:` never returns — /// the wedged-server case the containment exists for. `release()` lets the /// hung call finish so recovery is observable. diff --git a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift index 659dbd9c8b..2d082dbd88 100644 --- a/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift +++ b/apple/runner/AgentDeviceRunner/AgentDeviceRunnerUITests/UnitTests/RunnerTests+InteractionTests.swift @@ -23,5 +23,207 @@ extension RunnerTests { XCTAssertEqual(events.count, 1) XCTAssertEqual(events.first?.vertical, -200) } + +#if os(iOS) && targetEnvironment(simulator) + func testPrivateAXPointInspectionReturnsContainingElementsSmallestFirst() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Native Action", + "identifier": "native-action", + "value": "", + "frame": ["x": 100, "y": 500, "width": 200, "height": 48], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 520)) + + XCTAssertEqual(inspection.text, "Native Action") + XCTAssertEqual(inspection.elements.count, 2) + XCTAssertEqual(inspection.elements.first?.identifier, "native-action") + XCTAssertEqual(inspection.elements.last?.identifier, "root") + XCTAssertEqual(inspection.elements.first?.frame, SnapshotRect(x: 100, y: 500, width: 200, height: 48)) + XCTAssertNil(inspection.elements.first?.hittable) + } + + func testPrivateAXPointInspectionPrefersTextInputValueAndDescribesNumericValues() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.textField.rawValue), + "label": "Search", + "identifier": "search-field", + "value": "long live field value", + "frame": ["x": 100, "y": 500, "width": 200, "height": 48], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Clear", + "identifier": "clear-button", + "value": "", + "frame": ["x": 250, "y": 510, "width": 24, "height": 24], + "children": [], + ]], + ], [ + "type": NSNumber(value: XCUIElement.ElementType.switch.rawValue), + "label": "Enabled", + "identifier": "enabled-switch", + "value": NSNumber(value: 1), + "frame": ["x": 255, "y": 515, "width": 10, "height": 10], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 260, y: 520)) + + XCTAssertEqual(inspection.text, "long live field value") + XCTAssertEqual(inspection.elements.first?.identifier, "enabled-switch") + XCTAssertTrue(inspection.elements.contains { $0.identifier == "clear-button" }) + XCTAssertEqual(inspection.elements.last(where: { $0.identifier == "enabled-switch" })?.value, "1") + } + + func testPrivateAXPointTextResolutionIsNotLimitedByDescriptorCap() { + let smallerControls: [[String: Any]] = (0..<30).map { index in + [ + "type": NSNumber(value: XCUIElement.ElementType.button.rawValue), + "label": "Control \(index)", + "identifier": "control-\(index)", + "value": "", + "frame": ["x": 200, "y": 200, "width": 10, "height": 10], + "children": [], + ] + } + let field: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.textField.rawValue), + "label": "Full name", + "identifier": "full-name", + "value": "Alexandria Alexandra", + "frame": ["x": 100, "y": 180, "width": 200, "height": 40], + "children": [], + ] + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": smallerControls + [field], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 205, y: 205)) + + XCTAssertTrue(inspection.complete) + XCTAssertEqual(inspection.elements.count, 24) + XCTAssertFalse(inspection.elements.contains { $0.identifier == "full-name" }) + XCTAssertEqual(inspection.text, "Alexandria Alexandra") + } + + func testPointTextPolicyIsSharedAcrossXCTestAndPrivateAXDescriptors() { + let cases: [(String, String?, String?, String?, String?)] = [ + ("TextField", "Name", "name-field", "Alexandra", "Alexandra"), + ("SecureTextField", "Password", "password-field", "secret", "secret"), + ("Button", "Continue", "continue-button", "enabled", "Continue"), + ("Switch", nil, "enabled-switch", "1", "1"), + ] + + for (type, label, identifier, value, expected) in cases { + XCTAssertEqual( + pointReadableText(type: type, label: label, identifier: identifier, value: value), + expected, + "Expected shared text policy for \(type)" + ) + } + } + + func testPrivateAXPointInspectionReturnsNoElementForHonestMiss() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 200)) + + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } + + func testPrivateAXPointInspectionDoesNotReportMissForTruncatedCapture() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection( + root: root, + point: CGPoint(x: 200, y: 200), + truncated: true + ) + + XCTAssertFalse(inspection.complete) + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } + + func testPrivateAXPointInspectionDoesNotReportMissForIncompleteDeepExtension() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Root", + "identifier": "root", + "value": "", + "frame": ["x": 0, "y": 0, "width": 100, "height": 100], + "children": [], + ] + + let inspection = privateAXPointInspection( + root: root, + point: CGPoint(x: 200, y: 200), + completeDeepExtension: false + ) + + XCTAssertFalse(inspection.complete) + XCTAssertNil(inspection.text) + XCTAssertTrue(inspection.elements.isEmpty) + } + + func testPrivateAXPointInspectionOmitsOwningApplicationLabel() { + let root: [String: Any] = [ + "type": NSNumber(value: XCUIElement.ElementType.application.rawValue), + "label": "ET N Action", + "identifier": "com.expotargets.example.native.action", + "value": "", + "frame": ["x": 0, "y": 0, "width": 400, "height": 800], + "children": [[ + "type": NSNumber(value: XCUIElement.ElementType.window.rawValue), + "label": "Share Sheet", + "identifier": "share-sheet", + "value": "", + "frame": ["x": 0, "y": 400, "width": 400, "height": 400], + "children": [], + ]], + ] + + let inspection = privateAXPointInspection(root: root, point: CGPoint(x: 200, y: 520)) + + XCTAssertEqual(inspection.text, "Share Sheet") + XCTAssertFalse(inspection.elements.contains { $0.label == "ET N Action" }) + } +#endif } #endif diff --git a/contracts/fixtures/runner-requests.json b/contracts/fixtures/runner-requests.json index 730c8cb359..ea8b1a6af2 100644 --- a/contracts/fixtures/runner-requests.json +++ b/contracts/fixtures/runner-requests.json @@ -38,6 +38,7 @@ {"name": "ios-simulator.interactor-back.system", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "backSystem", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-find-text.text", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "findText", "text": "Ready", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-home.press", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "home", "appBundleId": "com.example.app", "commandId": ""}}, + {"name": "ios-simulator.interactor-inspect-point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "inspectPoint": true, "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-dismiss.dismiss", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardDismiss", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-keyboard-enter.return", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "keyboardReturn", "appBundleId": "com.example.app", "commandId": ""}}, {"name": "ios-simulator.interactor-read-text.point", "producer": "packages/platform-apple/src/__tests__/runner-requests.test.ts", "request": {"command": "readText", "x": 10, "y": 20, "appBundleId": "com.example.app", "commandId": ""}}, diff --git a/packages/command-registry/src/__tests__/command-result.test.ts b/packages/command-registry/src/__tests__/command-result.test.ts index 4cc3f6a7db..117990e346 100644 --- a/packages/command-registry/src/__tests__/command-result.test.ts +++ b/packages/command-registry/src/__tests__/command-result.test.ts @@ -143,6 +143,7 @@ test('CommandResultMap is seeded only from already-existing contract result type | 'test' | 'record' | 'trace' + | 'inspect-point' > = true; void keys; }); diff --git a/packages/command-registry/src/command-result.ts b/packages/command-registry/src/command-result.ts index 11f9d271e9..f6a4ea85b8 100644 --- a/packages/command-registry/src/command-result.ts +++ b/packages/command-registry/src/command-result.ts @@ -2,6 +2,7 @@ import type { DiffSnapshotCommandResult, ViewportCommandResult, } from '@agent-device/contracts/capture'; +import type { PointInspectionResult } from '@agent-device/contracts/client'; import type { PrepareCommandResult, PushCommandResult } from '@agent-device/contracts/command'; import type { AppStateCommandResult, @@ -72,6 +73,7 @@ export interface CommandResultMap { fold: FoldCommandResult; home: HomeCommandResult; hover: HoverCommandResponseData; + 'inspect-point': PointInspectionResult; keyboard: KeyboardCommandResult; longpress: LongPressCommandResponseData; orientation: OrientationCommandResult; diff --git a/packages/command-registry/src/flag-definitions-action.ts b/packages/command-registry/src/flag-definitions-action.ts index cd697ca8c3..57f3d02a8c 100644 --- a/packages/command-registry/src/flag-definitions-action.ts +++ b/packages/command-registry/src/flag-definitions-action.ts @@ -3,6 +3,24 @@ import { RECORDING_SCOPE_VALUES } from '@agent-device/contracts/recording'; import type { FlagDefinition } from './flag-types.ts'; export const ACTION_FLAG_DEFINITIONS: readonly FlagDefinition[] = [ + { + key: 'pointX', + names: ['--x'], + type: 'number', + usageLabel: '--x ', + usageDescription: 'Inspect-point: horizontal screen coordinate', + projectConfig: false, + recorded: false, + }, + { + key: 'pointY', + names: ['--y'], + type: 'number', + usageLabel: '--y ', + usageDescription: 'Inspect-point: vertical screen coordinate', + projectConfig: false, + recorded: false, + }, { key: 'keyframes', names: ['--keyframes'], diff --git a/packages/command-registry/src/registry.ts b/packages/command-registry/src/registry.ts index 43ebe1a20c..0096841628 100644 --- a/packages/command-registry/src/registry.ts +++ b/packages/command-registry/src/registry.ts @@ -57,6 +57,7 @@ import { gestureRuntimePlanUses, gestureViewportRuntimeUse, homeRuntimeUse, + inspectPointRuntimeUse, hoverRuntimeUses, keyboardRuntimePlanUses, longPressRuntimeUses, @@ -1078,6 +1079,19 @@ export const RAW_COMMAND_DESCRIPTORS = [ platformExecution: { kind: 'device-runtime', uses: closeApplicationRuntimePlanUses }, }, + { + name: 'inspect-point', + deviceClaimPolicy: 'require-owner', + ...(ownerFilesEnabled ? { ownerFiles: ['src/commands/capture/index.ts'] as const } : {}), + catalog: { group: 'public' }, + frameworkTier: 'extended', + recordsSessionAction: false, + daemon: { route: 'generic', refFrameEffect: 'preserve' }, + timeoutPolicy: PRESERVE_DAEMON_TIMEOUT_POLICY, + batchable: true, + platformExecution: { kind: 'device-runtime', uses: [inspectPointRuntimeUse] }, + }, + // -- snapshot (route: snapshot) -- { name: 'snapshot', diff --git a/packages/contracts/package.json b/packages/contracts/package.json index 9582f1ada8..598264821f 100644 --- a/packages/contracts/package.json +++ b/packages/contracts/package.json @@ -188,6 +188,10 @@ "types": "./src/element-text-runtime.ts", "default": "./src/element-text-runtime.ts" }, + "./point-inspection-runtime": { + "types": "./src/point-inspection-runtime.ts", + "default": "./src/point-inspection-runtime.ts" + }, "./fill-evidence": { "types": "./src/fill-evidence.ts", "default": "./src/fill-evidence.ts" diff --git a/packages/contracts/src/cli-flags.ts b/packages/contracts/src/cli-flags.ts index 8cd4412922..a00950478c 100644 --- a/packages/contracts/src/cli-flags.ts +++ b/packages/contracts/src/cli-flags.ts @@ -109,6 +109,8 @@ export type CliFlags = CloudProviderProfileFields & holdMs?: number; jitterPx?: number; pixels?: number; + pointX?: number; + pointY?: number; /** Scroll: repeat passes until this selector is visible on screen. */ until?: string; doubleTap?: boolean; diff --git a/packages/contracts/src/client-capture.ts b/packages/contracts/src/client-capture.ts index 844897f725..03ece82945 100644 --- a/packages/contracts/src/client-capture.ts +++ b/packages/contracts/src/client-capture.ts @@ -11,6 +11,7 @@ import type { SnapshotVisibility, } from '@agent-device/kernel/snapshot'; import type { ScreenshotResultData } from './snapshot-types.ts'; +import type { PointInspectionElement } from './interactor-types.ts'; import type { AgentDeviceIdentifiers, AgentDeviceRequestOverrides, @@ -102,6 +103,22 @@ export type CaptureScreenshotResult = ScreenshotResultData & { identifiers: AgentDeviceIdentifiers; }; +export type InspectPointOptions = DeviceCommandBaseOptions & { + point: Readonly<{ x: number; y: number }>; +}; + +export type PointInspectionResult = + | Readonly<{ + status: 'inspected'; + point: Readonly<{ x: number; y: number }>; + elements: readonly [PointInspectionElement, ...PointInspectionElement[]]; + }> + | Readonly<{ + status: 'no-element-at-point'; + point: Readonly<{ x: number; y: number }>; + elements: readonly []; + }>; + export type CaptureDiffOptions = DeviceCommandBaseOptions & Pick & { kind: 'snapshot'; diff --git a/packages/contracts/src/facades/client.ts b/packages/contracts/src/facades/client.ts index 8cd5d9dad2..5ecb33cd2e 100644 --- a/packages/contracts/src/facades/client.ts +++ b/packages/contracts/src/facades/client.ts @@ -23,6 +23,8 @@ export type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + InspectPointOptions, + PointInspectionResult, FindSnapshotCommandOptions, SelectorSnapshotCommandOptions, } from '../client-capture.ts'; diff --git a/packages/contracts/src/interactor-types.ts b/packages/contracts/src/interactor-types.ts index dd5cb82363..6f5e0d8cd7 100644 --- a/packages/contracts/src/interactor-types.ts +++ b/packages/contracts/src/interactor-types.ts @@ -39,6 +39,24 @@ export type RunnerContext = { runnerLeaseContext?: RunnerLogicalLeaseContext; }; +/** One bounded accessibility descriptor returned by a native point inspection. */ +export type PointInspectionElement = Readonly<{ + text?: string; + label?: string; + identifier?: string; + type?: string; + role?: string; + value?: string; + frame: Rect; + hittable?: boolean; +}>; + +/** Native point-read payload. Elements are ordered from smallest to largest containing frame. */ +export type PointInspectionRead = Readonly<{ + text?: string; + elements: readonly PointInspectionElement[]; +}>; + /** Subset of {@link RunnerContext} forwarded to runner command invocations. */ export type RunnerCallOptions = Pick< RunnerContext, @@ -355,6 +373,10 @@ export type Interactor = { point: Point, options?: { appBundleId?: string; surface?: SessionSurface; signal?: AbortSignal }, ): Promise; + inspectPoint?( + point: Point, + options?: { appBundleId?: string; signal?: AbortSignal }, + ): Promise; /** * Native text-presence reading, when the backend has one that does not require a tree capture. * A `true` answer is authoritative; anything else means "not proven here" and the caller diff --git a/packages/contracts/src/platform-runtime-operations.ts b/packages/contracts/src/platform-runtime-operations.ts index 3d6981294d..9d364cd6b3 100644 --- a/packages/contracts/src/platform-runtime-operations.ts +++ b/packages/contracts/src/platform-runtime-operations.ts @@ -11,7 +11,10 @@ import type { ScreenRecordingRuntimeHost } from './screen-recording-runtime-host import type { ScreenRecordingRuntimeOperations } from './screen-recording-runtime.ts'; import type { ScreenshotRuntimeOperations } from './screenshot-runtime.ts'; import type { SnapshotRuntimeHost, SnapshotRuntimeOperations } from './snapshot-runtime.ts'; -import type { SelectorObservationRuntimeOperations } from './selector-observation-runtime.ts'; +import type { + PointInspectionRuntimeOperations, + SelectorObservationRuntimeOperations, +} from './selector-observation-runtime.ts'; import type { ViewportRuntimeOperations } from './viewport-runtime.ts'; import type { FocusRuntimeOperations } from './focus-runtime.ts'; import type { GestureCommandInput, GestureSemanticInput } from './gesture-plan-types.ts'; @@ -74,6 +77,7 @@ export type PlatformRuntimeOperations = AppLogRuntimeOperations & ScrollRuntimeOperations & TypeTextRuntimeOperations & ElementTextRuntimeOperations & + PointInspectionRuntimeOperations & BackRuntimeOperations & OrientationRuntimeOperations & FoldRuntimeOperations & @@ -108,6 +112,7 @@ export const captureSnapshotUse = defineUse({ required: ['captureSnapshot'] }); export const viewportRuntimeUse = defineUse({ required: ['setViewport'] }); export const focusRuntimeUse = defineUse({ required: ['focusPoint'] }); export const typeTextRuntimeUse = defineUse({ required: ['typeText'] }); +export const inspectPointRuntimeUse = defineUse({ required: ['inspectPoint'] }); export const backRuntimeUse = defineUse({ required: ['back'] }); export const homeRuntimeUse = defineUse({ required: ['home'] }); export const orientationRuntimeUse = defineUse({ required: ['setOrientation'] }); diff --git a/packages/contracts/src/platform-runtime-unavailable.ts b/packages/contracts/src/platform-runtime-unavailable.ts index 7e84509948..6064c769a6 100644 --- a/packages/contracts/src/platform-runtime-unavailable.ts +++ b/packages/contracts/src/platform-runtime-unavailable.ts @@ -13,7 +13,10 @@ import type { } from './platform-runtime.ts'; import { screenshotRuntimeOperationFacts } from './screenshot-runtime.ts'; import { snapshotRuntimeOperationFacts } from './snapshot-runtime.ts'; -import { selectorObservationRuntimeOperationFacts } from './selector-observation-runtime.ts'; +import { + pointInspectionRuntimeOperationFacts, + selectorObservationRuntimeOperationFacts, +} from './selector-observation-runtime.ts'; import { viewportRuntimeOperationFacts } from './viewport-runtime.ts'; import { focusRuntimeOperationFacts } from './focus-runtime.ts'; import { gestureRuntimeOperationFacts } from './gesture-runtime.ts'; @@ -237,6 +240,7 @@ export function createUnavailablePlatformRuntimeFacts( fill: frozen.touch, }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: frozen.elementText }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: frozen.elementText }), ...backRuntimeOperationFacts({ back: frozen.back }), ...orientationRuntimeOperationFacts({ orientation: frozen.orientation }), // Stated directly rather than through `foldRuntimeOperationFacts`, so this hub does not diff --git a/packages/contracts/src/point-inspection-runtime.ts b/packages/contracts/src/point-inspection-runtime.ts new file mode 100644 index 0000000000..5d02f6e53d --- /dev/null +++ b/packages/contracts/src/point-inspection-runtime.ts @@ -0,0 +1,5 @@ +export { + bindPointInspectionRuntime, + pointInspectionRuntimeOperationFacts, + type PointInspectionRuntimeOperations, +} from './selector-observation-runtime.ts'; diff --git a/packages/contracts/src/runtime-operation-names.ts b/packages/contracts/src/runtime-operation-names.ts index 25bf9a1c7b..e12d228429 100644 --- a/packages/contracts/src/runtime-operation-names.ts +++ b/packages/contracts/src/runtime-operation-names.ts @@ -69,6 +69,7 @@ export const RUNTIME_OPERATION_NAMES = [ 'readClipboard', 'readSetting', 'readTextAtPoint', + 'inspectPoint', 'resolveOpenTarget', 'screenRecordingCleanup', 'screenRecordingReattach', diff --git a/packages/contracts/src/selector-observation-runtime.ts b/packages/contracts/src/selector-observation-runtime.ts index 318d092bda..45ddcb3152 100644 --- a/packages/contracts/src/selector-observation-runtime.ts +++ b/packages/contracts/src/selector-observation-runtime.ts @@ -1,3 +1,6 @@ +import type { DeviceInfo } from '@agent-device/kernel/device'; +import type { Point } from '@agent-device/kernel/snapshot'; +import type { Interactor, PointInspectionRead, RunnerContext } from './interactor-types.ts'; import type { RuntimeOperationFact } from './platform-runtime.ts'; import type { SessionSurface } from './session-surface.ts'; import type { SnapshotRuntimeExecution } from './snapshot-runtime.ts'; @@ -33,3 +36,52 @@ export function selectorObservationRuntimeOperationFacts( ): SelectorObservationRuntimeOperationFacts { return Object.freeze({ findText: input.findText }); } + +export type InspectPointRuntimeInput = Readonly<{ + point: Point; + options?: Readonly<{ appBundleId?: string }>; + execution?: SnapshotRuntimeExecution; +}>; + +export type PointInspectionRuntimeOperations = Readonly<{ + inspectPoint(input: InspectPointRuntimeInput): Promise; +}>; + +export type PointInspectionRuntimeOperationFacts = Readonly<{ + inspectPoint: RuntimeOperationFact; +}>; + +export function pointInspectionRuntimeOperationFacts( + input: PointInspectionRuntimeOperationFacts, +): PointInspectionRuntimeOperationFacts { + return Object.freeze({ inspectPoint: input.inspectPoint }); +} + +export function bindPointInspectionRuntime( + params: Readonly<{ + device: DeviceInfo; + signal: AbortSignal; + resolveInteractor: (device: DeviceInfo, runner: RunnerContext) => Promise; + }>, +): PointInspectionRuntimeOperations { + return Object.freeze({ + inspectPoint: async (input) => { + params.signal.throwIfAborted(); + const interactor = await params.resolveInteractor(params.device, { + ...input.execution, + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + if (typeof interactor.inspectPoint !== 'function') { + const { invalidRuntimeContract } = await import('./runtime-contract-error.ts'); + throw invalidRuntimeContract( + 'Runtime owner advertised inspectPoint without an interactor implementation', + ); + } + return await interactor.inspectPoint(input.point, { + appBundleId: input.options?.appBundleId, + signal: params.signal, + }); + }, + }); +} diff --git a/packages/platform-android/src/runtime.test.ts b/packages/platform-android/src/runtime.test.ts index 011afb24f9..7fc39cd969 100644 --- a/packages/platform-android/src/runtime.test.ts +++ b/packages/platform-android/src/runtime.test.ts @@ -78,6 +78,11 @@ test.each([ expect(facts.operations.captureSnapshot).toEqual({ available: true }); // uiautomator reads text at a point over the same adb transport the capture uses. expect(facts.operations.readTextAtPoint).toEqual({ available: true }); + expect(facts.operations.inspectPoint).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'inspect-point is currently supported on iOS Simulator targets only.', + }); expect(facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(facts.operations.captureSnapshotWithoutActiveApp).toEqual({ available: true }); expect(facts.operations.setViewport).toMatchObject({ available: false }); diff --git a/packages/platform-android/src/runtime.ts b/packages/platform-android/src/runtime.ts index 2a0e4c9f3b..27bba280fd 100644 --- a/packages/platform-android/src/runtime.ts +++ b/packages/platform-android/src/runtime.ts @@ -15,6 +15,7 @@ import { availableApplicationLifecycleOperations, } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { focusRuntimeOperationFacts } from '@agent-device/contracts/focus-runtime'; import { ANDROID_TV_MULTI_TOUCH_UNSUPPORTED_HINT, @@ -365,6 +366,13 @@ export function createAndroidPlatformRuntime(host: PlatformRuntimeHost): Platfor ...elementTextRuntimeOperationFacts({ readTextAtPoint: device.kind === 'simulator' ? elementTextKindUnavailable : available, }), + ...pointInspectionRuntimeOperationFacts({ + inspectPoint: Object.freeze({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'inspect-point is currently supported on iOS Simulator targets only.', + } as const), + }), ...backRuntimeOperationFacts({ back: androidTouchFact(device) }), // `home` and `app-switcher` are one `input keyevent` each, admitted wherever the retired // `ANDROID_ALL` bucket admitted them. diff --git a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts index eed7d1d34e..56f319c93c 100644 --- a/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts +++ b/packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts @@ -69,6 +69,10 @@ const RUNNER_TRANSPORT_METHODS: Record< invoke: (i) => i.readTextAtPoint!({ x: 10, y: 20 }), runnerCommand: 'readText', }, + inspectPoint: { + invoke: (i) => i.inspectPoint!({ x: 10, y: 20 }), + runnerCommand: 'readText', + }, findText: { invoke: (i) => i.findText!('Ready'), runnerCommand: 'findText' }, back: { invoke: (i) => i.back(), runnerCommand: 'backInApp' }, home: { invoke: (i) => i.home!(), runnerCommand: 'home' }, @@ -124,6 +128,76 @@ test('provider-backed interactor routes runner-command methods through the injec } }); +test('inspectPoint uses its explicit runner mode without changing readTextAtPoint wire requests', async () => { + const calls: RecordedRunnerCall[] = []; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + { appBundleId: 'com.example.app' }, + recordingRunnerProvider(calls), + ); + + await interactor.readTextAtPoint!({ x: 10, y: 20 }); + assert.equal(calls[0]!.command.inspectPoint, undefined); + + calls.length = 0; + await interactor.inspectPoint!({ x: 10, y: 20 }); + assert.equal(calls[0]!.command.inspectPoint, true); +}); + +test('inspectPoint rejects a malformed runner payload instead of reporting an honest miss', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { text: 'ignored' } }), + ); + await assert.rejects( + interactor.inspectPoint!({ x: 10, y: 20 }), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('inspectPoint preserves an empty elements payload as an honest miss', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { elements: [] } }), + ); + + const result = await interactor.inspectPoint!({ x: 10, y: 20 }); + assert.deepEqual(result, { elements: [] }); +}); + +test('inspectPoint rejects non-finite accessibility frames', async () => { + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { + readText: { + elements: [{ label: 'Bad frame', frame: { x: Number.NaN, y: 0, width: 10, height: 10 } }], + }, + }), + ); + await assert.rejects( + interactor.inspectPoint!({ x: 10, y: 20 }), + (error: unknown) => error instanceof AppError && error.code === 'COMMAND_FAILED', + ); +}); + +test('inspectPoint returns valid runner descriptors in their reported order', async () => { + const elements = [ + { label: 'Smallest', frame: { x: 10, y: 10, width: 10, height: 10 } }, + { label: 'Containing', frame: { x: 0, y: 0, width: 40, height: 40 } }, + ]; + const interactor = createAppleInteractor( + IOS_SIMULATOR, + {}, + recordingRunnerProvider([], { readText: { elements } }), + ); + + const result = await interactor.inspectPoint!({ x: 12, y: 12 }); + assert.deepEqual(result.elements, elements); +}); + test('provider-backed interactor rejects local Apple tooling methods with a clear error', async () => { const interactor = createAppleInteractor(IOS_SIMULATOR, {}, recordingRunnerProvider([])); for (const [method, invoke] of Object.entries(LOCAL_TOOL_METHODS)) { diff --git a/packages/platform-apple/src/__tests__/recording-runner-provider.ts b/packages/platform-apple/src/__tests__/recording-runner-provider.ts index 5c97b0745b..a814838636 100644 --- a/packages/platform-apple/src/__tests__/recording-runner-provider.ts +++ b/packages/platform-apple/src/__tests__/recording-runner-provider.ts @@ -24,7 +24,9 @@ export function recordingRunnerProvider( }; } -export function runnerResultFor(sent: Pick) { +export function runnerResultFor( + sent: Pick, +) { switch (sent.command) { case 'snapshot': return { @@ -44,6 +46,8 @@ export function runnerResultFor(sent: Pick = { IOS_SIMULATOR, (i) => i.readTextAtPoint!({ x: 10, y: 20 }, { appBundleId: APP }), ], + 'ios-simulator.interactor-inspect-point': [ + IOS_SIMULATOR, + (i) => i.inspectPoint!({ x: 10, y: 20 }, { appBundleId: APP }), + ], 'ios-simulator.alert.get': [IOS_SIMULATOR, (i) => i.readAlert!({ appBundleId: APP })], 'ios-simulator.alert.accept': [IOS_SIMULATOR, (i) => i.acceptAlert!({ appBundleId: APP })], 'ios-simulator.alert.dismiss': [IOS_SIMULATOR, (i) => i.dismissAlert!({ appBundleId: APP })], diff --git a/packages/platform-apple/src/interactor.ts b/packages/platform-apple/src/interactor.ts index 0d960fea39..93d04c5dcf 100644 --- a/packages/platform-apple/src/interactor.ts +++ b/packages/platform-apple/src/interactor.ts @@ -24,6 +24,8 @@ import { withMethodScope } from '@agent-device/kernel/scoped-provider'; import type { Point, SnapshotQualityVerdict } from '@agent-device/kernel/snapshot'; import type { Interactor, + PointInspectionElement, + PointInspectionRead, RunnerCallOptions, RunnerContext, ScreenshotOptions, @@ -76,6 +78,8 @@ export function createAppleInteractor( ? await readMacOsSurfaceTextAtPoint(point, helper, options?.appBundleId) : await readRunnerTextAtPoint(device, point, options, runnerOpts); }, + inspectPoint: async (point, options) => + await inspectRunnerPoint(device, point, options, runnerOpts), // The XCTest runner's own text reading: it observes the live accessibility hierarchy // directly, so it answers without the cost — and without the pruning — of a tree capture. // Only a positive answer is authoritative; see `FindTextResult`. @@ -454,3 +458,75 @@ async function readRunnerTextAtPoint( // The runner answers `message` when it reached the element but rendered no readable text. return typeof result.message === 'string' ? result.message : undefined; } + +async function inspectRunnerPoint( + device: DeviceInfo, + point: Point, + options: { appBundleId?: string; signal?: AbortSignal } | undefined, + runnerOpts: RunnerCallOptions, +): Promise { + const result = await runAppleRunnerCommand( + device, + { + command: 'readText', + inspectPoint: true, + x: point.x, + y: point.y, + appBundleId: options?.appBundleId, + }, + options?.signal ? { ...runnerOpts, signal: options.signal } : runnerOpts, + ); + if (!Array.isArray(result.elements)) { + throw new AppError( + 'COMMAND_FAILED', + 'iOS runner returned an invalid point-inspection payload: missing elements', + ); + } + const elements = result.elements.map((value) => { + const element = readPointInspectionElement(value); + if (!element) { + throw new AppError( + 'COMMAND_FAILED', + 'iOS runner returned an invalid point-inspection element', + ); + } + return element; + }); + return { + ...(typeof result.text === 'string' ? { text: result.text } : {}), + elements, + }; +} + +function readPointInspectionElement(value: unknown): PointInspectionElement | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const record = value as Record; + const frame = readPointInspectionFrame(record.frame); + if (!frame) return undefined; + return { + ...readPointInspectionStrings(record), + frame, + ...(typeof record.hittable === 'boolean' ? { hittable: record.hittable } : {}), + }; +} + +function readPointInspectionFrame(value: unknown): PointInspectionElement['frame'] | undefined { + if (typeof value !== 'object' || value === null) return undefined; + const frame = value as Record; + const dimensions = [frame.x, frame.y, frame.width, frame.height]; + if (!dimensions.every((dimension) => typeof dimension === 'number' && Number.isFinite(dimension))) + return undefined; + const [x, y, width, height] = dimensions as [number, number, number, number]; + return { x, y, width, height }; +} + +function readPointInspectionStrings( + record: Record, +): Partial> { + const strings: Record = {}; + for (const key of ['text', 'label', 'identifier', 'type', 'role', 'value'] as const) { + const value = record[key]; + if (typeof value === 'string' && value !== '') strings[key] = value; + } + return strings; +} diff --git a/packages/platform-apple/src/runner-demand.ts b/packages/platform-apple/src/runner-demand.ts index e37e380376..b69c1bceb3 100644 --- a/packages/platform-apple/src/runner-demand.ts +++ b/packages/platform-apple/src/runner-demand.ts @@ -58,6 +58,7 @@ const APPLE_SIMULATOR_OPERATION_HOSTS: Readonly< captureScreenshot: 'simulator', findText: 'runner', readTextAtPoint: 'runner', + inspectPoint: 'runner', // Every interaction and runner-driven capture. tapPoint: 'runner', tapRef: 'runner', diff --git a/packages/platform-apple/src/runner/runner-contract.ts b/packages/platform-apple/src/runner/runner-contract.ts index 1a87558441..36ec67bc66 100644 --- a/packages/platform-apple/src/runner/runner-contract.ts +++ b/packages/platform-apple/src/runner/runner-contract.ts @@ -40,6 +40,7 @@ export const MAIN_THREAD_TIMEOUT_RUNNER_CODE = 'MAIN_THREAD_TIMEOUT'; const APP_NOT_RUNNING_RUNNER_CODE = 'APP_NOT_RUNNING'; export type RunnerCommand = { + inspectPoint?: boolean; command: | 'tap' | 'mouseClick' diff --git a/packages/platform-apple/src/runtime.test.ts b/packages/platform-apple/src/runtime.test.ts index fdee94259d..9647469ba9 100644 --- a/packages/platform-apple/src/runtime.test.ts +++ b/packages/platform-apple/src/runtime.test.ts @@ -25,6 +25,18 @@ const presentIosAcquisition: SnapshotRuntimeHost['presentIosAcquisition'] = asyn nodes: [], }); +function expectPointInspectionBinding( + binding: DeviceBinding, + available: boolean, +): void { + expect(binding.facts.operations.inspectPoint.available).toBe(available); + if (available) { + expect(binding.operations.inspectPoint).toBeTypeOf('function'); + } else { + expect(binding.operations.inspectPoint).toBeUndefined(); + } +} + function appleDevice(overrides: Partial = {}): DeviceInfo { return { platform: 'apple', @@ -91,85 +103,91 @@ function expectAppStateFact( } test.each([ - ['iOS simulator', leaves.ios, true, undefined], + ['iOS simulator', leaves.ios, true, undefined, true], [ 'iOS physical CoreDevice', appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'coredevice' }), true, undefined, + false, ], [ 'iOS physical XCTest', appleDevice({ kind: 'device', iosPhysicalDeviceBackend: 'xctest' }), false, 'CoreDevice-backed physical iOS device', + false, ], - ['iPadOS simulator', leaves.ipados, true, undefined], - ['tvOS simulator', leaves.tvos, true, undefined], - ['macOS host', leaves.macos, true, undefined], - ['visionOS simulator', leaves.visionos, true, undefined], - ['watchOS sentinel', leaves.watchos, false, 'watchOS app logs are not supported'], -])('classifies the %s leaf explicitly', async (_name, device, available, hint) => { - const binding = await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ - device, - intent: { kind: 'ordinary' }, - scope: { - signal: new AbortController().signal, - diagnostics: { emit: () => {} }, - progress: { report: () => {} }, - }, - }); - const { facts } = binding; - expect(facts.device.providerMode).toBe('local'); - expectAppStateFact(device, binding); - expect(facts.operations.networkDump).toEqual({ available: true }); - expect(facts.operations.listApps.available).toBe( - device.appleOs !== 'watchos' && device.iosPhysicalDeviceBackend !== 'xctest', - ); - // R40/R41: touch and text ride the Apple interactor, which exists for the simulator and - // physical device kinds — every leaf in this table is one of those two, so both cells are - // available across it (parity with the retired buckets). - expect(facts.operations.focusPoint).toEqual({ available: true }); - expect(facts.operations.typeText).toEqual({ available: true }); - expect(binding.operations.focusPoint).toBeTypeOf('function'); - expect(binding.operations.typeText).toBeTypeOf('function'); - for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) { - const fact = facts.operations[operation]; - expect(fact.available).toBe(available); - if (!available && hint) expect(fact).toHaveProperty('hint', expect.stringContaining(hint)); - } - for (const operation of [ - 'screenRecordingStart', - 'screenRecordingReattach', - 'screenRecordingCleanup', - ] as const) { - expect(facts.operations[operation].available).toBe(available); - } - expectApplePerfAvailability(binding, available); - if (device.iosPhysicalDeviceBackend === 'xctest') { - expect(facts.operations.screenRecordingStart).toMatchObject({ - hint: expect.stringContaining('CoreDevice-backed physical iOS device'), + ['iPadOS simulator', leaves.ipados, true, undefined, false], + ['tvOS simulator', leaves.tvos, true, undefined, false], + ['macOS host', leaves.macos, true, undefined, false], + ['visionOS simulator', leaves.visionos, true, undefined, false], + ['watchOS sentinel', leaves.watchos, false, 'watchOS app logs are not supported', false], +])( + 'classifies the %s leaf explicitly', + async (_name, device, available, hint, inspectPointAvailable) => { + const binding = await createApplePlatformRuntime(platformRuntimeHostFixture()).bind({ + device, + intent: { kind: 'ordinary' }, + scope: { + signal: new AbortController().signal, + diagnostics: { emit: () => {} }, + progress: { report: () => {} }, + }, }); - } - if (device.appleOs === 'watchos') { - expect(facts.operations.screenRecordingStart).toMatchObject({ - hint: 'watchOS recording is not supported.', + const { facts } = binding; + expect(facts.device.providerMode).toBe('local'); + expectAppStateFact(device, binding); + expect(facts.operations.networkDump).toEqual({ available: true }); + expect(facts.operations.listApps.available).toBe( + device.appleOs !== 'watchos' && device.iosPhysicalDeviceBackend !== 'xctest', + ); + // R40/R41: touch and text ride the Apple interactor, which exists for the simulator and + // physical device kinds — every leaf in this table is one of those two, so both cells are + // available across it (parity with the retired buckets). + expect(facts.operations.focusPoint).toEqual({ available: true }); + expect(facts.operations.typeText).toEqual({ available: true }); + expectPointInspectionBinding(binding, inspectPointAvailable); + expect(binding.operations.focusPoint).toBeTypeOf('function'); + expect(binding.operations.typeText).toBeTypeOf('function'); + for (const operation of ['appLogInspect', 'appLogDoctor', 'appLogStart'] as const) { + const fact = facts.operations[operation]; + expect(fact.available).toBe(available); + if (!available && hint) expect(fact).toHaveProperty('hint', expect.stringContaining(hint)); + } + for (const operation of [ + 'screenRecordingStart', + 'screenRecordingReattach', + 'screenRecordingCleanup', + ] as const) { + expect(facts.operations[operation].available).toBe(available); + } + expectApplePerfAvailability(binding, available); + if (device.iosPhysicalDeviceBackend === 'xctest') { + expect(facts.operations.screenRecordingStart).toMatchObject({ + hint: expect.stringContaining('CoreDevice-backed physical iOS device'), + }); + } + if (device.appleOs === 'watchos') { + expect(facts.operations.screenRecordingStart).toMatchObject({ + hint: 'watchOS recording is not supported.', + }); + } + expect(facts.operations.ensureReady.available).toBe(device.appleOs !== 'watchos'); + expect(facts.operations.bootTarget.available).toBe( + device.appleOs !== 'macos' && device.appleOs !== 'watchos', + ); + expect(facts.operations.bootTargetHeadless.available).toBe(false); + expect(facts.operations.setViewport).toEqual({ + available: false, + reason: 'unsupported-platform-leaf', + hint: 'viewport resizes web targets only (--platform web). Apple screen geometry is fixed by the selected simulator or device type — open a different simulator to test another screen size.', }); - } - expect(facts.operations.ensureReady.available).toBe(device.appleOs !== 'watchos'); - expect(facts.operations.bootTarget.available).toBe( - device.appleOs !== 'macos' && device.appleOs !== 'watchos', - ); - expect(facts.operations.bootTargetHeadless.available).toBe(false); - expect(facts.operations.setViewport).toEqual({ - available: false, - reason: 'unsupported-platform-leaf', - hint: 'viewport resizes web targets only (--platform web). Apple screen geometry is fixed by the selected simulator or device type — open a different simulator to test another screen size.', - }); - expect(binding.operations.setViewport).toBeUndefined(); - expectAppleCaptureAvailability(binding, device); - expectAppleSnapshotAvailability(binding, device); -}); + expect(binding.operations.setViewport).toBeUndefined(); + expectAppleCaptureAvailability(binding, device); + expectAppleSnapshotAvailability(binding, device); + }, +); function expectApplePerfAvailability( binding: DeviceBinding, diff --git a/packages/platform-apple/src/runtime.ts b/packages/platform-apple/src/runtime.ts index 3907f76689..02ff5cf778 100644 --- a/packages/platform-apple/src/runtime.ts +++ b/packages/platform-apple/src/runtime.ts @@ -21,6 +21,11 @@ import { bindElementTextRuntime, elementTextRuntimeOperationFacts, } from '@agent-device/contracts/element-text-runtime'; +import { + bindPointInspectionRuntime, + pointInspectionRuntimeOperationFacts, + type PointInspectionRuntimeOperations, +} from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -322,6 +327,7 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR ...(isIosFamily(device) ? { tapElementSelector: appleFocusFact(device) } : {}), }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: appleElementTextFact(device) }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: applePointInspectionFact(device) }), ...appleNavigationFacts(device), ...appleFoldableFacts(device), ...appleSystemFacts(device), @@ -357,8 +363,18 @@ export function createApplePlatformRuntime(host: PlatformRuntimeHost): PlatformR resolveInteractor: host.localInteractors.resolve, }), ); + const pointInspectionOperations: Partial = whenAdmitted( + facts.operations.inspectPoint, + () => + bindPointInspectionRuntime({ + device: request.device, + signal: request.scope.signal, + resolveInteractor: host.localInteractors.resolve, + }), + ); const operations: DeviceBinding['operations'] = { ...appStateOperations, + ...pointInspectionOperations, ...logs.operations, ...createAppleAppDeploymentOperations({ host, @@ -547,6 +563,12 @@ function appleElementTextFact(device: DeviceInfo) { : elementTextKindUnavailable; } +function applePointInspectionFact(device: DeviceInfo): RuntimeOperationFact { + return resolveDeviceAppleOs(device) === 'ios' && device.kind === 'simulator' + ? available + : unavailable; +} + function appleSnapshotFact(device: DeviceInfo) { if (resolveDeviceAppleOs(device) === 'watchos') return snapshotKindUnavailable; return device.kind === 'simulator' || device.kind === 'device' diff --git a/packages/platform-harmonyos/src/runtime.ts b/packages/platform-harmonyos/src/runtime.ts index 68036c3e61..ad52844513 100644 --- a/packages/platform-harmonyos/src/runtime.ts +++ b/packages/platform-harmonyos/src/runtime.ts @@ -10,6 +10,7 @@ import { } from '@agent-device/contracts/application-lifecycle-runtime'; import { backRuntimeOperationFacts } from '@agent-device/contracts/back-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -270,6 +271,7 @@ export function createHarmonyPlatformRuntime(host: PlatformRuntimeHost): Platfor // HarmonyOS has no point-read tool: `get` answers from the captured tree, which is what // the legacy dispatch already did after its Apple-runner attempt failed. ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...backRuntimeOperationFacts({ back: harmonyFocusFact(device) }), // Home and the app switcher ride the same HDC-driven key input; no other system button // has a HarmonyOS control for HDC to press. diff --git a/packages/platform-linux/src/runtime.test.ts b/packages/platform-linux/src/runtime.test.ts index 9da74e6353..cc1e3ac3b0 100644 --- a/packages/platform-linux/src/runtime.test.ts +++ b/packages/platform-linux/src/runtime.test.ts @@ -108,6 +108,7 @@ test.each([ // The Linux read is value-first where the captured tree is label-first, so the desktop row // genuinely reads differently from its snapshot text and advertises the live read. expect(binding.facts.operations.readTextAtPoint.available).toBe(device.kind === 'device'); + expect(binding.facts.operations.inspectPoint.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithCustomActions.available).toBe(false); expect(binding.facts.operations.captureSnapshotWithoutActiveApp.available).toBe( device.kind === 'device', diff --git a/packages/platform-linux/src/runtime.ts b/packages/platform-linux/src/runtime.ts index 2d17a9e940..6523ce1417 100644 --- a/packages/platform-linux/src/runtime.ts +++ b/packages/platform-linux/src/runtime.ts @@ -15,6 +15,7 @@ import { } from '@agent-device/contracts/application-lifecycle-runtime'; import { backRuntimeOperationFacts } from '@agent-device/contracts/back-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { focusRuntimeOperationFacts } from '@agent-device/contracts/focus-runtime'; import { TARGET_AUTHORED_DRAG_UNSUPPORTED_HINT } from '@agent-device/contracts/gesture-admission'; import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; @@ -261,6 +262,7 @@ function linuxFacts(device: DeviceInfo): RuntimeFacts ...elementTextRuntimeOperationFacts({ readTextAtPoint: linuxDesktopFact(device, elementTextKindUnavailable), }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextKindUnavailable }), // Parity with the retired `back`/`home` capability bucket (`{ device: true }`): the desktop // is the only Linux cell with a target to drive. ...backRuntimeOperationFacts({ back: linuxDesktopFact(device, backKindUnavailable) }), diff --git a/packages/platform-web/src/runtime.test.ts b/packages/platform-web/src/runtime.test.ts index 5ba868a300..565b4b8021 100644 --- a/packages/platform-web/src/runtime.test.ts +++ b/packages/platform-web/src/runtime.test.ts @@ -60,6 +60,7 @@ test('preserves a narrow web provider dump including empty successful entries', // No point-addressed read on the web backend: `get` answers from the captured DOM tree. The // legacy `read` dispatch had no web arm at all and threw on every call before falling back. expect(binding.facts.operations.readTextAtPoint.available).toBe(false); + expect(binding.facts.operations.inspectPoint.available).toBe(false); expect(binding.operations.readTextAtPoint).toBeUndefined(); // R40/R41: the browser device drives touch and text through the one web interactor. expect(binding.facts.operations.focusPoint).toEqual({ available: true }); diff --git a/packages/platform-web/src/runtime.ts b/packages/platform-web/src/runtime.ts index 883ac73069..6c0a2c2d97 100644 --- a/packages/platform-web/src/runtime.ts +++ b/packages/platform-web/src/runtime.ts @@ -16,6 +16,7 @@ import { availableApplicationLifecycleOperations, } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { bindLocalFocusInteractor, focusRuntimeOperationFacts, @@ -417,6 +418,7 @@ function webRuntimeFacts( // The web backend has no point-addressed read: `get` answers from the captured DOM tree, // which is what the legacy dispatch already did once its Apple-runner attempt failed. ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...backRuntimeOperationFacts({ back: navigationUnavailable }), ...orientationRuntimeOperationFacts({ orientation: navigationUnavailable }), ...tvRemoteRuntimeOperationFacts({ tvRemote: navigationUnavailable }), diff --git a/packages/provider-limrun/src/facts-runtime.ts b/packages/provider-limrun/src/facts-runtime.ts index a68d22c991..6700736098 100644 --- a/packages/provider-limrun/src/facts-runtime.ts +++ b/packages/provider-limrun/src/facts-runtime.ts @@ -1,6 +1,7 @@ import { isIosFamily, type DeviceInfo } from '@agent-device/kernel/device'; import { applicationLifecycleOperationFacts } from '@agent-device/contracts/application-lifecycle-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import type { PlatformRuntimeOperations } from '@agent-device/contracts/platform-runtime-operations'; import type { RuntimeFacts } from '@agent-device/contracts/platform-runtime'; import { screenshotRuntimeOperationFacts } from '@agent-device/contracts/screenshot-runtime'; @@ -191,6 +192,7 @@ export function limrunAppLogFacts( // device always has one, so it is available wherever a capture is. ...limrunInteractionOperationFacts(device), ...elementTextRuntimeOperationFacts({ readTextAtPoint: elementTextUnavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: elementTextUnavailable }), ...limrunNavigationOperationFacts(device), ...limrunKeyboardOperationFacts(device), ...limrunClipboardOperationFacts(device), diff --git a/scripts/integration-progress-model.test.ts b/scripts/integration-progress-model.test.ts index dc90be093a..8a19c05acc 100644 --- a/scripts/integration-progress-model.test.ts +++ b/scripts/integration-progress-model.test.ts @@ -3,7 +3,42 @@ import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; import { test } from 'node:test'; import os from 'node:os'; import path from 'node:path'; -import { buildIntegrationProgressModel } from './integration-progress-model.ts'; +import { + buildIntegrationProgressModel, + countInspectPointCoordinateReferences, +} from './integration-progress-model.ts'; + +test('integration progress scopes point coordinates to each inspect-point step object', () => { + const source = ` + const steps = [ + { + flags: { label: 'unrelated object with braces', nested: { positionals: ['wrong'] } }, + command: 'inspect-point', + assert: (response) => response.json, + positionals: ['196', '122'], + }, + { command: 'tap', positionals: ['900', '901'] }, + { command: 'inspect-point', positionals: ['196'] }, + { command: 'inspect-point', positionals: [] }, + ]; + // { command: 'inspect-point', positionals: ['unrelated', 'literal'] } + `; + + assert.equal(countInspectPointCoordinateReferences(source, 0), 2); + assert.equal(countInspectPointCoordinateReferences(source, 1), 1); +}); + +test('integration progress ignores regex literals while finding inspect-point steps', () => { + const source = String.raw` + const steps = [ + { command: 'inspect-point', positionals: ['1', '2'], assert: (response) => /[{]/.test(response.json) }, + { command: 'inspect-point', positionals: ['3', '4'] }, + ]; + `; + + assert.equal(countInspectPointCoordinateReferences(source, 0), 2); + assert.equal(countInspectPointCoordinateReferences(source, 1), 2); +}); test('integration progress counts explicit generic Apple host-tool usage only', async () => { const root = await mkdtemp(path.join(os.tmpdir(), 'agent-device-progress-')); diff --git a/scripts/integration-progress-model.ts b/scripts/integration-progress-model.ts index 3a1be8a0b8..c63513951f 100644 --- a/scripts/integration-progress-model.ts +++ b/scripts/integration-progress-model.ts @@ -148,6 +148,8 @@ function summarizeProviderScenarioFlagCoverage(files) { ['retainPaths', 'retained install-source materialization'], ['retentionMs', 'install-source materialization TTL'], ['count', 'repeated press/click/swipe input'], + ['pointX', 'point inspection horizontal coordinate'], + ['pointY', 'point inspection vertical coordinate'], ['pointerCount', 'one- vs two-pointer pan gesture topology'], ['fps', 'recording frame-rate request'], ['quality', 'recording quality scaling'], @@ -200,14 +202,218 @@ function summarizeProviderScenarioFlagCoverage(files) { ]; const sources = files.map((file) => fs.readFileSync(file, 'utf8')).join('\n'); return flagTargets.map(([key, reason, aliases = []]) => { - const references = [key, ...aliases].reduce( - (count, candidate) => count + countFlagReferences(sources, candidate), - 0, - ); + // inspect-point takes x/y as positional arguments, unlike the named + // workflow flags counted below. Attribute those coordinates only to an + // inspect-point provider scenario so an unrelated geometry literal cannot + // satisfy this coverage row. + const positionalCoordinate = key === 'pointX' ? 0 : key === 'pointY' ? 1 : null; + const references = + positionalCoordinate === null + ? [key, ...aliases].reduce( + (count, candidate) => count + countFlagReferences(sources, candidate), + 0, + ) + : countInspectPointCoordinateReferences(sources, positionalCoordinate); return { key, reason, references }; }); } +export function countInspectPointCoordinateReferences(text, coordinateIndex) { + const code = maskScenarioStringsAndComments(text); + let count = 0; + const commandPattern = /\bcommand\s*:\s*(['"])inspect-point\1/g; + for (const match of text.matchAll(commandPattern)) { + const commandCodePrefix = match[0].slice(0, match[0].indexOf(match[1])); + if (code.slice(match.index, match.index + commandCodePrefix.length) !== commandCodePrefix) { + continue; + } + + const openBrace = findEnclosingBrace(code, match.index); + if (openBrace < 0) continue; + const closeBrace = findMatchingDelimiter(code, openBrace, '{', '}'); + if (closeBrace < 0) continue; + const positionals = findDirectArrayProperty(text, code, openBrace, closeBrace, 'positionals'); + if (positionals !== undefined && countArrayTokens(text, code, positionals) > coordinateIndex) { + count += 1; + } + } + return count; +} + +function maskScenarioStringsAndComments(text) { + const chars = text.split(''); + let index = 0; + while (index < chars.length) { + const end = + maskComment(chars, index) ?? maskRegex(chars, index) ?? maskQuotedText(chars, index); + if (end === null) { + index += 1; + } else { + index = end; + } + } + return chars.join(''); +} + +function maskComment(chars, index) { + if (chars[index] !== '/') return null; + if (chars[index + 1] === '/') return maskLineComment(chars, index); + if (chars[index + 1] === '*') return maskBlockComment(chars, index); + return null; +} + +function maskLineComment(chars, index) { + while (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + return index; +} + +function maskBlockComment(chars, index) { + chars[index++] = ' '; + chars[index++] = ' '; + while (index < chars.length && !(chars[index] === '*' && chars[index + 1] === '/')) { + if (chars[index] !== '\n') chars[index] = ' '; + index += 1; + } + return index < chars.length ? maskCommentEnd(chars, index) : index; +} + +function maskCommentEnd(chars, index) { + chars[index++] = ' '; + chars[index++] = ' '; + return index; +} + +function maskRegex(chars, index) { + if (chars[index] !== '/' || !beginsRegexLiteral(chars, index)) return null; + chars[index++] = ' '; + return scanRegexBody(chars, index); +} + +function scanRegexBody(chars, index) { + let inCharacterClass = false; + while (index < chars.length && chars[index] !== '\n') { + const current = chars[index]; + if (current === '\\') { + index = maskEscapedCharacter(chars, index); + continue; + } + inCharacterClass = updateCharacterClass(current, inCharacterClass); + if (current === '/' && !inCharacterClass) return maskRegexEnd(chars, index); + chars[index++] = ' '; + } + return index; +} + +function updateCharacterClass(character, inCharacterClass) { + if (character === '[') return true; + if (character === ']') return false; + return inCharacterClass; +} + +function maskEscapedCharacter(chars, index) { + chars[index++] = ' '; + if (index < chars.length && chars[index] !== '\n') chars[index++] = ' '; + return index; +} + +function maskRegexEnd(chars, index) { + chars[index++] = ' '; + while (index < chars.length && /[a-z]/i.test(chars[index])) chars[index++] = ' '; + return index; +} + +function maskQuotedText(chars, index) { + const quote = chars[index]; + if (quote !== '"' && quote !== "'" && quote !== '`') return null; + chars[index++] = ' '; + while (index < chars.length && chars[index] !== quote) { + if (chars[index] === '\\') { + index = maskEscapedCharacter(chars, index); + } else { + if (chars[index] !== '\n') chars[index] = ' '; + index += 1; + } + } + return index < chars.length ? index + maskQuoteEnd(chars, index) : index; +} + +function maskQuoteEnd(chars, index) { + chars[index] = ' '; + return 1; +} + +function beginsRegexLiteral(chars, index) { + let previous = index - 1; + while (previous >= 0 && /\s/.test(chars[previous])) previous -= 1; + return previous < 0 || /[(:,=!?&|;{[>]/.test(chars[previous]); +} + +function findEnclosingBrace(code, position) { + const stack = []; + for (let index = 0; index < position; index += 1) { + if (code[index] === '{') stack.push(index); + else if (code[index] === '}') stack.pop(); + } + return stack.at(-1) ?? -1; +} + +function findMatchingDelimiter(code, openIndex, open, close) { + let depth = 0; + for (let index = openIndex; index < code.length; index += 1) { + if (code[index] === open) depth += 1; + else if (code[index] === close && --depth === 0) return index; + } + return -1; +} + +function findDirectArrayProperty(text, code, openBrace, closeBrace, name) { + const propertyPattern = new RegExp(`\\b${name}\\s*:\\s*\\[`, 'g'); + const body = code.slice(openBrace + 1, closeBrace); + for (const match of body.matchAll(propertyPattern)) { + const position = openBrace + 1 + match.index; + if (!isTopLevelObjectPosition(code, openBrace, position)) continue; + const openBracket = position + match[0].lastIndexOf('['); + const closeBracket = findMatchingDelimiter(code, openBracket, '[', ']'); + if (closeBracket >= 0 && closeBracket < closeBrace) return [openBracket, closeBracket]; + } + return undefined; +} + +function isTopLevelObjectPosition(code, openBrace, position) { + let depth = 0; + for (let index = openBrace + 1; index < position; index += 1) { + if (code[index] === '{' || code[index] === '[' || code[index] === '(') depth += 1; + else if (code[index] === '}' || code[index] === ']' || code[index] === ')') depth -= 1; + } + return depth === 0; +} + +function countArrayTokens(text, code, [openBracket, closeBracket]) { + const tokens = []; + let start = openBracket + 1; + let depth = 0; + for (let index = start; index < closeBracket; index += 1) { + depth = updateDelimiterDepth(code[index], depth); + if (code[index] === ',' && depth === 0) { + tokens.push(text.slice(start, index)); + start = index + 1; + } + } + tokens.push(text.slice(start, closeBracket)); + return tokens.filter(isPresentArrayToken).length; +} + +function updateDelimiterDepth(character, depth) { + if (character === '{' || character === '[' || character === '(') return depth + 1; + if (character === '}' || character === ']' || character === ')') return depth - 1; + return depth; +} + +function isPresentArrayToken(token) { + const trimmed = token.trim(); + return trimmed.length > 0 && trimmed !== "''" && trimmed !== '""'; +} + function countFlagReferences(text, key) { const escaped = key.replaceAll(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); return text.match(new RegExp(`\\b${escaped}\\s*:`, 'g'))?.length ?? 0; diff --git a/scripts/layering/contracts-exports.snapshot.json b/scripts/layering/contracts-exports.snapshot.json index 3095a7699e..f1e08380a5 100644 --- a/scripts/layering/contracts-exports.snapshot.json +++ b/scripts/layering/contracts-exports.snapshot.json @@ -85,6 +85,7 @@ "@agent-device/contracts/platform-runtime-host", "@agent-device/contracts/platform-runtime-operations", "@agent-device/contracts/platform-runtime-unavailable", + "@agent-device/contracts/point-inspection-runtime", "@agent-device/contracts/progress", "@agent-device/contracts/react-native-overlay", "@agent-device/contracts/record-runtime-execution", diff --git a/src/__tests__/cli-client-commands.test.ts b/src/__tests__/cli-client-commands.test.ts index b3382c6ba9..4250d3c166 100644 --- a/src/__tests__/cli-client-commands.test.ts +++ b/src/__tests__/cli-client-commands.test.ts @@ -1167,12 +1167,7 @@ function createStubClient(params: { const command = createThrowingMethodGroup(); return { command, - devices: { - list: async () => [], - capabilities: unexpectedCommandCall, - boot: unexpectedCommandCall, - shutdown: unexpectedCommandCall, - }, + devices: createThrowingMethodGroup({ list: async () => [] }), sessions: { list: async () => [], stateDir: async () => '/tmp/agent-device-state', @@ -1250,7 +1245,7 @@ function createStubClient(params: { transport: 'http' as const, })), }, - capture: { + capture: createThrowingMethodGroup({ snapshot: async () => ({ nodes: [], truncated: false, @@ -1262,15 +1257,13 @@ function createStubClient(params: { path: '/tmp/screenshot.png', identifiers: { session: 'default' }, })), - diff: unexpectedCommandCall, - }, + }), interactions: createThrowingMethodGroup(), replay: createThrowingMethodGroup(), batch: createThrowingMethodGroup(), - observability: { - ...createThrowingMethodGroup(), + observability: createThrowingMethodGroup({ events: params.events ?? unexpectedCommandCall, - }, + }), debug: createThrowingMethodGroup(), recording: createThrowingMethodGroup(), settings: { @@ -1279,11 +1272,11 @@ function createStubClient(params: { }; } -function createThrowingMethodGroup(): T { +function createThrowingMethodGroup(methods: Partial = {}): T { const unexpectedCommandCall = async (): Promise => { throw new Error('unexpected command call'); }; - return new Proxy({} as Partial, { + return new Proxy(methods, { get: (target, property) => target[property as keyof T] ?? unexpectedCommandCall, }) as T; } diff --git a/src/__tests__/command-descriptor-timeout-policy.test.ts b/src/__tests__/command-descriptor-timeout-policy.test.ts index d8d444d278..cf2e33e579 100644 --- a/src/__tests__/command-descriptor-timeout-policy.test.ts +++ b/src/__tests__/command-descriptor-timeout-policy.test.ts @@ -61,6 +61,8 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { // Interaction commands joined in #1105: their target resolution runs the // same capture as snapshot, and resetting the daemon on a wedged capture // destroyed healthy app sessions. + // inspect-point joins because its XCTest accessibility read has the same + // blocked-bridge failure mode and must not reset unrelated sessions. // scroll/back joined in #1638: `--settle` gives them the same post-action // capture loop, so a wedged bridge is now their dominant hang mode too. // The lease route joined in #1774: those commands act on BILLED provider @@ -77,6 +79,7 @@ test('daemon-preserving timeout commands are a bounded, reviewed set', () => { 'find', 'get', 'hover', + 'inspect-point', 'is', 'lease_allocate', 'lease_heartbeat', diff --git a/src/__tests__/test-utils/property-arbitraries.ts b/src/__tests__/test-utils/property-arbitraries.ts index 4767f11f81..3cdfcf64f1 100644 --- a/src/__tests__/test-utils/property-arbitraries.ts +++ b/src/__tests__/test-utils/property-arbitraries.ts @@ -301,6 +301,7 @@ const REPLAY_SCRIPT_LINE_PLANS = { focus: GENERIC_REPLAY_LINE, home: GENERIC_REPLAY_LINE, install: GENERIC_REPLAY_LINE, + 'inspect-point': GENERIC_REPLAY_LINE, 'install-from-source': GENERIC_REPLAY_LINE, is: GENERIC_REPLAY_LINE, keyboard: GENERIC_REPLAY_LINE, diff --git a/src/__tests__/test-utils/runtime-operation-facts.ts b/src/__tests__/test-utils/runtime-operation-facts.ts index 350630632d..e070ed2aaf 100644 --- a/src/__tests__/test-utils/runtime-operation-facts.ts +++ b/src/__tests__/test-utils/runtime-operation-facts.ts @@ -2,6 +2,7 @@ import { applicationLifecycleOperationFacts } from '@agent-device/contracts/appl import { audioProbeRuntimeOperationFacts } from '@agent-device/contracts/audio-probe-runtime'; import { clipboardRuntimeOperationFacts } from '@agent-device/contracts/clipboard-runtime'; import { elementTextRuntimeOperationFacts } from '@agent-device/contracts/element-text-runtime'; +import { pointInspectionRuntimeOperationFacts } from '@agent-device/contracts/point-inspection-runtime'; import { gestureRuntimeOperationFacts } from '@agent-device/contracts/gesture-runtime'; import { keyboardRuntimeOperationFacts } from '@agent-device/contracts/keyboard-runtime'; import type { @@ -60,6 +61,7 @@ export const unavailableDeploymentSnapshotAndShutdownOperationFacts = Object.fre ...gestureRuntimeOperationFacts({ unsupported: unavailable }), ...scrollRuntimeOperationFacts({ scroll: unavailable }), ...elementTextRuntimeOperationFacts({ readTextAtPoint: unavailable }), + ...pointInspectionRuntimeOperationFacts({ inspectPoint: unavailable }), back: unavailable, setOrientation: unavailable, tvRemote: unavailable, diff --git a/src/agent-device-client.ts b/src/agent-device-client.ts index e616dabbc2..1e2fef821f 100644 --- a/src/agent-device-client.ts +++ b/src/agent-device-client.ts @@ -16,6 +16,7 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + PointInspectionResult, DragOptions, FlingOptions, MaterializationReleaseOptions, @@ -356,6 +357,8 @@ export function createAgentDeviceClient( }), }, capture: { + inspectPoint: async (options) => + await executeCommand('inspect-point', options), snapshot: async (options: CaptureSnapshotOptions = {}) => { const session = resolveRequestSession(options); const data = await executeCommand>('snapshot', options); diff --git a/src/cli/parser/__tests__/args-parse-session.test.ts b/src/cli/parser/__tests__/args-parse-session.test.ts index 14c87a98f7..50caefe688 100644 --- a/src/cli/parser/__tests__/args-parse-session.test.ts +++ b/src/cli/parser/__tests__/args-parse-session.test.ts @@ -252,6 +252,17 @@ test('parseArgs recognizes command-specific flag combinations', async () => { } }); +test('parseArgs rejects empty inline numeric values', () => { + assert.throws( + () => parseArgs(['inspect-point', '--x=', '--y=12'], { strictFlags: true }), + (error: unknown) => error instanceof Error && /Invalid x/.test(error.message), + ); + assert.throws( + () => parseArgs(['inspect-point', '--x=12', '--y='], { strictFlags: true }), + (error: unknown) => error instanceof Error && /Invalid y/.test(error.message), + ); +}); + test('parseArgs recognizes device isolation flags', () => { const parsed = parseArgs( [ diff --git a/src/cli/parser/args.ts b/src/cli/parser/args.ts index cf7cc19230..80fc1ccaf7 100644 --- a/src/cli/parser/args.ts +++ b/src/cli/parser/args.ts @@ -51,77 +51,92 @@ export function parseArgs(argv: string[], options?: FinalizeArgsOptions): Parsed } export function parseRawArgs(argv: string[]): RawParsedArgs { - const flags: CliFlags = { json: false, help: false, version: false }; - let command: string | null = null; - let rawCommand: string | null = null; - const positionals: string[] = []; - const warnings: string[] = []; - const providedFlags: ParsedFlagRecord[] = []; - let parseFlags = true; - - for (let i = 0; i < argv.length; i += 1) { - const arg = argv[i]!; - if (parseFlags && arg === '--') { - parseFlags = false; - continue; - } - if (!parseFlags) { - if (!command) { - rawCommand = arg; - command = normalizeCommandAlias(arg); - } else positionals.push(arg); - continue; - } - if (shouldPreservePostCommandArgs(command)) { - positionals.push(arg); - continue; - } - const isLongFlag = arg.startsWith('--'); - const isShortFlag = arg.startsWith('-') && arg.length > 1; - if (!isLongFlag && !isShortFlag) { - if (!command) { - rawCommand = arg; - command = normalizeCommandAlias(arg); - } else positionals.push(arg); - continue; - } + const state: RawParseState = { + command: null, + flags: { json: false, help: false, version: false }, + parseFlags: true, + positionals: [], + providedFlags: [], + rawCommand: null, + warnings: [], + }; + for (let index = 0; index < argv.length; index += parseRawArgument(state, argv, index) + 1) { + // parseRawArgument returns one extra consumed argument when a value follows its flag. + } + applyAliasImpliedFlags(state.rawCommand, state.flags); + return state; +} - const [token, inlineValue] = isLongFlag ? splitLongFlag(arg) : [arg, undefined]; - if (isLegacyIgnoredSnapshotShortFlag(command, token)) { - continue; - } - const definition = resolveFlagDefinition(token, command); - if (shouldPassThroughLocalToolFlag(command, definition)) { - positionals.push(arg); - continue; - } - if (!definition) { - if (shouldTreatUnknownDashTokenAsPositional(command, positionals, arg)) { - if (!command) command = arg; - else positionals.push(arg); - continue; - } - throw new AppError('INVALID_ARGS', formatUnknownFlagMessage(token, command)); - } +type RawParseState = RawParsedArgs & { rawCommand: string | null; parseFlags: boolean }; + +function parseRawArgument(state: RawParseState, argv: string[], index: number): number { + const arg = argv[index]!; + if (arg === '--' && state.parseFlags) { + state.parseFlags = false; + return 0; + } + if (appendRawPositional(state, arg)) return 0; + return parseRawFlag(state, arg, argv[index + 1]); +} - const parsed = parseFlagValue(definition, token, inlineValue, argv[i + 1]); - if (parsed.consumeNext) i += 1; - const existingValue = (flags as Record)[definition.key]; - if (definition.multiple) { - const values = Array.isArray(existingValue) - ? [...existingValue, parsed.value] - : existingValue === undefined - ? [parsed.value] - : [existingValue, parsed.value]; - (flags as Record)[definition.key] = values; +function appendRawPositional(state: RawParseState, arg: string): boolean { + if (!state.parseFlags || shouldPreservePostCommandArgs(state.command) || !isFlagToken(arg)) { + if (!state.command) { + state.rawCommand = arg; + state.command = normalizeCommandAlias(arg); } else { - (flags as Record)[definition.key] = parsed.value; + state.positionals.push(arg); } - providedFlags.push({ key: definition.key, token }); + return true; + } + return false; +} + +function isFlagToken(arg: string): boolean { + return arg.startsWith('--') || (arg.startsWith('-') && arg.length > 1); +} + +function parseRawFlag(state: RawParseState, arg: string, nextArg: string | undefined): number { + const [token, inlineValue] = arg.startsWith('--') ? splitLongFlag(arg) : [arg, undefined]; + if (isLegacyIgnoredSnapshotShortFlag(state.command, token)) return 0; + const definition = resolveFlagDefinition(token, state.command); + if (shouldPassThroughLocalToolFlag(state.command, definition)) { + state.positionals.push(arg); + return 0; + } + if (!definition) return parseUnknownRawFlag(state, token, arg); + const parsed = parseFlagValue(definition, token, inlineValue, nextArg); + appendParsedFlag(state, definition, token, parsed.value); + return Number(parsed.consumeNext); +} + +function parseUnknownRawFlag(state: RawParseState, token: string, arg: string): number { + if (shouldTreatUnknownDashTokenAsPositional(state.command, state.positionals, arg)) { + if (!state.command) state.command = arg; + else state.positionals.push(arg); + return 0; } + throw new AppError('INVALID_ARGS', formatUnknownFlagMessage(token, state.command)); +} + +function appendParsedFlag( + state: RawParseState, + definition: FlagDefinition, + token: string, + value: unknown, +): void { + const flags = state.flags as Record; + const existingValue = flags[definition.key]; + flags[definition.key] = definition.multiple + ? appendMultipleFlagValue(existingValue, value) + : value; + state.providedFlags.push({ key: definition.key, token }); +} - applyAliasImpliedFlags(rawCommand, flags); - return { command, positionals, flags, warnings, providedFlags }; +function appendMultipleFlagValue(existingValue: unknown, value: unknown): unknown[] { + if (Array.isArray(existingValue)) return [...existingValue, value]; + if (existingValue === undefined) return [value]; + return [existingValue, value]; } function applyAliasImpliedFlags(rawCommand: string | null, flags: CliFlags): void { @@ -259,68 +274,131 @@ function parseFlagValue( inlineValue: string | undefined, nextArg: string | undefined, ): { value: unknown; consumeNext: boolean } { - if (definition.setValue !== undefined) { - if (inlineValue !== undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); - } - return { value: definition.setValue, consumeNext: false }; + if (definition.setValue !== undefined) return parseSetValue(definition, token, inlineValue); + if (definition.type === 'boolean') return parseBooleanValue(token, inlineValue); + if (definition.type === 'booleanOrString') { + return parseBooleanOrStringValue(token, inlineValue, nextArg); } - if (definition.type === 'boolean') { - if (inlineValue !== undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); - } - return { value: true, consumeNext: false }; + return parseRequiredFlagValue(definition, token, inlineValue, nextArg); +} + +function parseSetValue( + definition: FlagDefinition, + token: string, + inlineValue: string | undefined, +): { value: unknown; consumeNext: boolean } { + assertNoInlineValue(token, inlineValue); + return { value: definition.setValue, consumeNext: false }; +} + +function parseBooleanValue( + token: string, + inlineValue: string | undefined, +): { value: unknown; consumeNext: boolean } { + assertNoInlineValue(token, inlineValue); + return { value: true, consumeNext: false }; +} + +function assertNoInlineValue(token: string, inlineValue: string | undefined): void { + if (inlineValue !== undefined) { + throw new AppError('INVALID_ARGS', `Flag ${token} does not take a value.`); } - if (definition.type === 'booleanOrString') { - if (inlineValue !== undefined) { - if (inlineValue.trim().length === 0) { - throw new AppError( - 'INVALID_ARGS', - `Flag ${token} requires a non-empty value when provided.`, - ); - } - return { value: inlineValue, consumeNext: false }; - } - if (nextArg === undefined || looksLikeFlagToken(nextArg)) { - return { value: true, consumeNext: false }; - } - if (shouldConsumeOptionalPathValue(nextArg)) { - return { value: nextArg, consumeNext: true }; - } +} + +function parseBooleanOrStringValue( + token: string, + inlineValue: string | undefined, + nextArg: string | undefined, +): { value: unknown; consumeNext: boolean } { + if (inlineValue !== undefined) return parseInlineBooleanOrString(token, inlineValue); + if (nextArg === undefined || looksLikeFlagToken(nextArg)) { return { value: true, consumeNext: false }; } + return shouldConsumeOptionalPathValue(nextArg) + ? { value: nextArg, consumeNext: true } + : { value: true, consumeNext: false }; +} +function parseInlineBooleanOrString( + token: string, + value: string, +): { value: unknown; consumeNext: boolean } { + if (value.trim().length === 0) { + throw new AppError('INVALID_ARGS', `Flag ${token} requires a non-empty value when provided.`); + } + return { value, consumeNext: false }; +} + +function parseRequiredFlagValue( + definition: FlagDefinition, + token: string, + inlineValue: string | undefined, + nextArg: string | undefined, +): { value: unknown; consumeNext: boolean } { const value = inlineValue ?? nextArg; - if (value === undefined) { - throw new AppError('INVALID_ARGS', `Flag ${token} requires a value.`); + assertRequiredFlagValue(token, value, inlineValue); + if (definition.type === 'string') return { value, consumeNext: inlineValue === undefined }; + if (definition.type === 'enum') { + return parseEnumFlagValue(definition, token, value, inlineValue === undefined); } - if (inlineValue === undefined && looksLikeFlagToken(value)) { + return parseNumericFlagValue(definition, token, value, inlineValue === undefined); +} + +function assertRequiredFlagValue( + token: string, + value: string | undefined, + inlineValue: string | undefined, +): asserts value is string { + if (value === undefined || (inlineValue === undefined && looksLikeFlagToken(value))) { throw new AppError('INVALID_ARGS', `Flag ${token} requires a value.`); } +} - if (definition.type === 'string') { - return { value, consumeNext: inlineValue === undefined }; - } - if (definition.type === 'enum') { - if (!definition.enumValues?.includes(value)) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); - } - return { value, consumeNext: inlineValue === undefined }; +function parseEnumFlagValue( + definition: FlagDefinition, + token: string, + value: string, + consumeNext: boolean, +): { value: unknown; consumeNext: boolean } { + if (!definition.enumValues?.includes(value)) { + throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); } + return { value, consumeNext }; +} + +function parseNumericFlagValue( + definition: FlagDefinition, + token: string, + value: string, + consumeNext: boolean, +): { value: unknown; consumeNext: boolean } { const parsed = Number(value); - if (!Number.isFinite(parsed)) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + if (value.trim().length === 0 || !Number.isFinite(parsed)) { + return invalidNumericFlagValue(token, value); } + assertNumericBounds(definition, token, value, parsed); + return { + value: definition.type === 'int' ? Math.floor(parsed) : parsed, + consumeNext, + }; +} + +function invalidNumericFlagValue(token: string, value: string): never { + throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); +} + +function assertNumericBounds( + definition: FlagDefinition, + token: string, + value: string, + parsed: number, +): void { if (typeof definition.min === 'number' && parsed < definition.min) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + invalidNumericFlagValue(token, value); } if (typeof definition.max === 'number' && parsed > definition.max) { - throw new AppError('INVALID_ARGS', `Invalid ${labelForFlag(token)}: ${value}`); + invalidNumericFlagValue(token, value); } - return { - value: definition.type === 'int' ? Math.floor(parsed) : parsed, - consumeNext: inlineValue === undefined, - }; } function labelForFlag(token: string): string { diff --git a/src/client/client-types.ts b/src/client/client-types.ts index 25b493df43..78cb854ba8 100644 --- a/src/client/client-types.ts +++ b/src/client/client-types.ts @@ -31,20 +31,13 @@ export type { // Contracts/kernel types re-exported into the PUBLISHED surface: `agent-device-client.ts` picks // these up via `export type *`, and that is their only job — every internal consumer imports -// them from the declaring module instead. Fallow therefore sees no consumer, which is exactly -// right and exactly not actionable: deleting them would remove names from the package's public -// types. Suppressed per name rather than baselined so the reason travels with the code. -// fallow-ignore-next-line unused-type +// them from the declaring module instead. They remain part of the package's public type surface +// for downstream consumers even though repository-local code does not import them here. export type { TargetShutdownResult } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { MetroBridgeScope } from '@agent-device/contracts/remote'; -// fallow-ignore-next-line unused-type export type { AppsFilter } from '@agent-device/contracts/device'; -// fallow-ignore-next-line unused-type export type { AlertAction } from '@agent-device/contracts/alert-contract'; -// fallow-ignore-next-line unused-type export type { AppleOS } from '@agent-device/kernel/device'; -// fallow-ignore-next-line unused-type export type { JsonObject } from '@agent-device/contracts/client'; export type { BatchRunResult } from '@agent-device/command-registry/batch'; @@ -80,6 +73,8 @@ import type { CaptureScreenshotResult, CaptureSnapshotOptions, CaptureSnapshotResult, + InspectPointOptions, + PointInspectionResult, ClickOptions, ClipboardCommandOptions, CloudArtifactsOptions, @@ -250,6 +245,7 @@ export type AgentDeviceClient = { }; capture: { snapshot: (options?: CaptureSnapshotOptions) => Promise; + inspectPoint: (options: InspectPointOptions) => Promise; screenshot: (options?: CaptureScreenshotOptions) => Promise; diff: (options: CaptureDiffOptions) => Promise>; }; diff --git a/src/commands/capture/index.test.ts b/src/commands/capture/index.test.ts index 03e381e5d1..e899e4b18e 100644 --- a/src/commands/capture/index.test.ts +++ b/src/commands/capture/index.test.ts @@ -2,6 +2,7 @@ import { describe, expect, test } from 'vitest'; import type { CliFlags } from '@agent-device/contracts/command'; import { alertCliReader, alertDaemonWriter } from './alert.ts'; import { diffCliReader } from './diff.ts'; +import { captureCommandFamily } from './index.ts'; import { snapshotCliOutput } from './output.ts'; import { screenshotCliReader, screenshotDaemonWriter } from './screenshot.ts'; import { snapshotCliReader } from './snapshot.ts'; @@ -21,6 +22,25 @@ function expectInvalidArgs(fn: () => unknown, messageFragment: string) { } describe('capture command interface', () => { + test('reads inspect-point coordinate flags and writes its daemon request', () => { + const input = captureCommandFamily.cliReaders['inspect-point']( + [], + flags({ pointX: 14.5, pointY: 29 }), + ); + expect(input).toMatchObject({ point: { x: 14.5, y: 29 } }); + expect(captureCommandFamily.daemonWriters?.['inspect-point']?.(input)).toMatchObject({ + command: 'inspect-point', + positionals: ['14.5', '29'], + }); + }); + + test('rejects inspect-point when either coordinate is missing', () => { + expectInvalidArgs( + () => captureCommandFamily.cliReaders['inspect-point']([], flags({ pointX: 1 })), + '--x and --y', + ); + }); + test('reads snapshot flags', () => { expect( snapshotCliReader( diff --git a/src/commands/capture/index.ts b/src/commands/capture/index.ts index c908be42f6..c7d0b3aadc 100644 --- a/src/commands/capture/index.ts +++ b/src/commands/capture/index.ts @@ -1,4 +1,12 @@ -import { defineCommandFamilyFromFacets } from '../family/types.ts'; +import { defineCommandFacet, defineCommandFamilyFromFacets } from '../family/types.ts'; +import { PUBLIC_COMMANDS } from '@agent-device/command-registry/catalog'; +import type { InspectPointOptions, PointInspectionResult } from '@agent-device/contracts/client'; +import { AppError } from '@agent-device/kernel/errors'; +import { pointField, requiredField } from '../command-input.ts'; +import { commonInputFromFlags, direct } from '../cli-grammar/common.ts'; +import type { CliReader, DaemonWriter } from '../cli-grammar/types.ts'; +import { defineFieldCommandMetadata } from '../field-command-contract.ts'; +import { resultOutput } from '../output-common.ts'; import { alertCommandFacet } from './alert.ts'; import { diffCommandFacet } from './diff.ts'; import { screenshotCommandFacet } from './screenshot.ts'; @@ -6,8 +14,41 @@ import { settingsCommandFacet } from './settings.ts'; import { snapshotCommandFacet } from './snapshot.ts'; import { waitCommandFacet } from './wait.ts'; +const inspectPointMetadata = defineFieldCommandMetadata( + 'inspect-point', + 'Inspect the live accessibility elements containing one screen coordinate.', + { point: requiredField(pointField('Screen coordinate to inspect.')) }, +); + +const inspectPointCliReader: CliReader = (_positionals, flags) => { + if (typeof flags.pointX !== 'number' || typeof flags.pointY !== 'number') { + throw new AppError('INVALID_ARGS', 'inspect-point requires --x and --y'); + } + return { ...commonInputFromFlags(flags), point: { x: flags.pointX, y: flags.pointY } }; +}; + +const inspectPointDaemonWriter: DaemonWriter = direct(PUBLIC_COMMANDS['inspect-point'], (input) => { + const { point } = input as InspectPointOptions; + return [String(point.x), String(point.y)]; +}); + +const inspectPointCommandFacet = defineCommandFacet({ + name: 'inspect-point', + text: { summary: 'Inspect elements at a screen coordinate' }, + metadata: inspectPointMetadata, + run: (client, input) => client.capture.inspectPoint(input), + cliSchema: { allowedFlags: ['pointX', 'pointY'] }, + cliReader: inspectPointCliReader, + daemonWriter: inspectPointDaemonWriter, + cliOutputFormatter: resultOutput((result: PointInspectionResult) => ({ + data: result, + text: JSON.stringify(result, null, 2), + })), +}); + const captureCommandFacets = [ snapshotCommandFacet, + inspectPointCommandFacet, screenshotCommandFacet, diffCommandFacet, waitCommandFacet, diff --git a/src/daemon/__tests__/runtime-binding-conformance.ts b/src/daemon/__tests__/runtime-binding-conformance.ts index bfc125365c..f6c4731db7 100644 --- a/src/daemon/__tests__/runtime-binding-conformance.ts +++ b/src/daemon/__tests__/runtime-binding-conformance.ts @@ -18,6 +18,7 @@ import { resolveBoundBackRuntime } from '../back-runtime.ts'; import { resolveBoundFocusRuntime } from '../focus-runtime.ts'; import { resolveBoundFoldRuntime } from '../fold-runtime.ts'; import { resolveBoundGestureRuntime } from '../gesture-runtime.ts'; +import { resolveBoundInspectPointRuntime } from '../inspect-point-runtime.ts'; import { resolveBoundOrientationRuntime } from '../orientation-runtime.ts'; import type { ResolvedGenericExecution } from '../request-generic-dispatch.ts'; import type { @@ -109,6 +110,16 @@ export const conformedRuntimeBindings = { resolve: async (device, bindings) => refusable(await resolveBoundFocusRuntime({ device, positionals: ['40', '90'], ...bindings })), }, + 'inspect-point': { + resolve: async (device, bindings) => + refusable( + await resolveBoundInspectPointRuntime({ + device, + positionals: ['40', '90'], + ...bindings, + }), + ), + }, viewport: { resolve: async (device, bindings) => refusable( diff --git a/src/daemon/generic-runtime-execution.ts b/src/daemon/generic-runtime-execution.ts index cdd440102f..455ecf717e 100644 --- a/src/daemon/generic-runtime-execution.ts +++ b/src/daemon/generic-runtime-execution.ts @@ -13,6 +13,7 @@ import { resolveBoundOrientationRuntime } from './orientation-runtime.ts'; import { resolveBoundFoldRuntime } from './fold-runtime.ts'; import { resolveBoundTvRemoteRuntime } from './tv-remote-runtime.ts'; import { errorResponse } from '@agent-device/kernel/contracts'; +import { resolveBoundInspectPointRuntime } from './inspect-point-runtime.ts'; /** * The generic route's runtime-owned leaves (ADR 0019). Each one admits its own exact owner facts @@ -41,6 +42,13 @@ export async function resolveGenericRuntimeExecution( }); } switch (params.req.command) { + case 'inspect-point': + return await resolveBoundInspectPointRuntime({ + device: params.session.device, + positionals: params.req.positionals ?? [], + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }); case 'screenshot': return await resolveScreenshotGenericExecution(params); case 'focus': diff --git a/src/daemon/handlers/__tests__/install-source.test.ts b/src/daemon/handlers/__tests__/install-source.test.ts index ca21dfbcc4..5b930176f0 100644 --- a/src/daemon/handlers/__tests__/install-source.test.ts +++ b/src/daemon/handlers/__tests__/install-source.test.ts @@ -446,6 +446,7 @@ function sourceRuntimeFacts( ...gestureRuntimeOperationFacts({ unsupported: unavailable }), ...scrollRuntimeOperationFacts({ scroll: unavailable }), readTextAtPoint: unavailable, + inspectPoint: unavailable, back: unavailable, setOrientation: unavailable, tvRemote: unavailable, diff --git a/src/daemon/inspect-point-runtime.ts b/src/daemon/inspect-point-runtime.ts new file mode 100644 index 0000000000..d5065f2a49 --- /dev/null +++ b/src/daemon/inspect-point-runtime.ts @@ -0,0 +1,36 @@ +import { inspectPointRuntimeUse } from '@agent-device/contracts/platform-runtime-operations'; +import type { DeviceInfo } from '@agent-device/kernel/device'; +import { readPointPositionals } from '@agent-device/kernel/validation'; +import type { DaemonCommandContext } from './context.ts'; +import type { ResolvedGenericExecution } from './request-generic-dispatch.ts'; +import { resolveBoundGenericRuntime, type RuntimeAdmissionBindings } from './runtime-admission.ts'; +import { runtimeExecutionFromContext } from './snapshot-runtime-capture-input.ts'; + +export async function resolveBoundInspectPointRuntime( + params: { + device: DeviceInfo; + positionals: string[]; + } & RuntimeAdmissionBindings, +): Promise { + const point = readPointPositionals(params.positionals, 'inspect-point requires x y'); + return await resolveBoundGenericRuntime( + { + command: 'inspect-point', + device: params.device, + use: inspectPointRuntimeUse, + inspectFacts: params.inspectFacts, + bindDevice: params.bindDevice, + }, + async (runtime, context: DaemonCommandContext) => { + const result = await runtime.operations.inspectPoint({ + point, + ...(context.appBundleId ? { options: { appBundleId: context.appBundleId } } : {}), + execution: runtimeExecutionFromContext(context), + }); + const [first, ...rest] = result.elements; + return first === undefined + ? { status: 'no-element-at-point', point, elements: [] } + : { status: 'inspected', point, elements: [first, ...rest] }; + }, + ); +} diff --git a/src/mcp/command-output-schemas.ts b/src/mcp/command-output-schemas.ts index 742a5835db..affa6d25a9 100644 --- a/src/mcp/command-output-schemas.ts +++ b/src/mcp/command-output-schemas.ts @@ -864,6 +864,50 @@ const BASE_COMMAND_OUTPUT_SCHEMAS = { ), ], }, + 'inspect-point': { + type: 'object', + oneOf: [ + objectSchema( + { + status: constSchema('inspected'), + point: objectSchema({ x: numberSchema(), y: numberSchema() }, ['x', 'y']), + elements: { + type: 'array', + items: objectSchema( + { + text: stringSchema(), + label: stringSchema(), + identifier: stringSchema(), + type: stringSchema(), + role: stringSchema(), + value: stringSchema(), + frame: objectSchema( + { + x: numberSchema(), + y: numberSchema(), + width: numberSchema(), + height: numberSchema(), + }, + ['x', 'y', 'width', 'height'], + ), + hittable: booleanSchema(), + }, + ['frame'], + ), + }, + }, + ['status', 'point', 'elements'], + ), + objectSchema( + { + status: constSchema('no-element-at-point'), + point: objectSchema({ x: numberSchema(), y: numberSchema() }, ['x', 'y']), + elements: { type: 'array', maxItems: 0 }, + }, + ['status', 'point', 'elements'], + ), + ], + }, // A family that owns its commands authors their advertised response shape beside the // command surface and projects it here. This spread stays last: a hand-written entry for diff --git a/test/integration/command-coverage/declarations.ts b/test/integration/command-coverage/declarations.ts index b3adbb26c4..4f14a98c0d 100644 --- a/test/integration/command-coverage/declarations.ts +++ b/test/integration/command-coverage/declarations.ts @@ -1388,6 +1388,37 @@ const COMMAND_COVERAGE_DECLARATIONS = { ), linux: linux.gap('No Linux-specific source-install command evidence exists yet'), }, + [C['inspect-point']]: { + androidEmulator: androidEmulator.contract( + ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE, + 'the Android runtime reports point inspection unavailable instead of dispatching an unsupported command', + ), + iosSimulator: iosSimulator.contract( + 'packages/platform-apple/src/__tests__/interactor-runner-provider.test.ts', + 'inspectPoint returns valid runner descriptors in their reported order', + 'the Apple interactor preserves valid ordered point-inspection descriptors returned by the runner', + ), + macos: macos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the %s leaf explicitly', + 'the Apple runtime refuses point inspection outside iOS Simulator', + ), + tvos: tvos.contract( + 'packages/platform-apple/src/runtime.test.ts', + 'classifies the %s leaf explicitly', + 'the Apple runtime refuses point inspection outside iOS Simulator', + ), + web: web.contract( + 'packages/platform-web/src/runtime.test.ts', + 'preserves a narrow web provider dump including empty successful entries', + 'the web runtime reports point inspection unavailable', + ), + linux: linux.contract( + 'packages/platform-linux/src/runtime.test.ts', + 'classifies the Linux $name lifecycle denominator against the legacy dispatch cell', + 'the Linux runtime reports point inspection unavailable', + ), + }, } satisfies Record; /** diff --git a/test/integration/command-coverage/evidence.ts b/test/integration/command-coverage/evidence.ts index 42aed9865b..053e4ecb6f 100644 --- a/test/integration/command-coverage/evidence.ts +++ b/test/integration/command-coverage/evidence.ts @@ -29,7 +29,7 @@ export const ANDROID_APPLICATION_LIFECYCLE_CONTRACT_EVIDENCE: AndroidContractEvi export const ANDROID_HOVER_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = defineAndroidContractEvidence( 'packages/platform-android/src/runtime.test.ts', - [C.hover], + [C.hover, C['inspect-point']], 'classifies the Android %s runtime denominator', ); export const ANDROID_TV_REMOTE_RUNTIME_CONTRACT_EVIDENCE: AndroidContractEvidence = diff --git a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts index 4131ca25f8..61724973ac 100644 --- a/test/integration/provider-scenarios/apple-platform-output-guard.test.ts +++ b/test/integration/provider-scenarios/apple-platform-output-guard.test.ts @@ -74,6 +74,7 @@ const DRIVEN_COMMANDS: Record = { [PUBLIC_COMMANDS.boot]: () => one(), [PUBLIC_COMMANDS.prepare]: () => one(['ios-runner']), [PUBLIC_COMMANDS.snapshot]: () => one([], { snapshotInteractiveOnly: true }), + [PUBLIC_COMMANDS['inspect-point']]: () => one(['10', '10']), [PUBLIC_COMMANDS.perf]: () => [{ positionals: [] }, { positionals: ['frames'] }], [PUBLIC_COMMANDS.record]: ({ world, tmpDir }) => world === 'ios' @@ -142,11 +143,9 @@ const DRIVEN_COMMANDS: Record = { // --------------------------------------------------------------------------- // SKIPPED_COMMANDS — catalog commands that genuinely cannot be driven against the -// fake-provider harness, each with a reason. Intentionally EMPTY: every public -// command is driveable here (an orchestrator with no real workload simply returns a -// fast, still-scanned error response). Kept as an explicit, enforced set so a future -// undriveable command has a home and the partition test keeps a new command from -// escaping the guard silently. +// fake-provider harness, each with a reason. Kept as an explicit, enforced set so a future +// undriveable command has a home and the partition test keeps a new command from escaping the +// guard silently. // --------------------------------------------------------------------------- const SKIPPED_COMMANDS: Record = {}; @@ -248,6 +247,8 @@ function permissiveRunner(): AppleRunnerProvider { return { found: true, nodes: [richNodes()[2]] }; case 'findText': return { found: true }; + case 'readText': + return command.inspectPoint === true ? { elements: [] } : { done: true }; default: return { done: true }; } diff --git a/test/integration/provider-scenarios/ios-lifecycle.test.ts b/test/integration/provider-scenarios/ios-lifecycle.test.ts index 2becd3f0c5..957375f393 100644 --- a/test/integration/provider-scenarios/ios-lifecycle.test.ts +++ b/test/integration/provider-scenarios/ios-lifecycle.test.ts @@ -70,6 +70,36 @@ test('Provider-backed integration iOS Settings flow uses scripted simctl and run assert.equal(firstSnapshot.json?.result?.data?.nodes?.[0]?.ref, 'e1'); }, }, + { + name: 'inspect the accessibility elements at a settings coordinate', + command: 'inspect-point', + positionals: ['196', '122'], + expectData: { + status: 'inspected', + point: { x: 196, y: 122 }, + elements: [ + { + text: 'General', + label: 'General', + identifier: 'General', + type: 'Button', + value: 'General', + frame: { x: 146, y: 102, width: 100, height: 40 }, + hittable: true, + }, + ], + }, + }, + { + name: 'reports an honest miss for an empty settings coordinate', + command: 'inspect-point', + positionals: ['0', '0'], + expectData: { + status: 'no-element-at-point', + point: { x: 0, y: 0 }, + elements: [], + }, + }, { name: 'reopen existing session app', command: 'open', diff --git a/test/integration/provider-scenarios/ios-world.ts b/test/integration/provider-scenarios/ios-world.ts index a012acebb2..21eba2ab5c 100644 --- a/test/integration/provider-scenarios/ios-world.ts +++ b/test/integration/provider-scenarios/ios-world.ts @@ -40,6 +40,45 @@ export async function createIosSettingsWorld(): Promise { result: { uptimeMs: 42 }, }, runnerSnapshot(), + { + command: 'ios.runner.readText', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple', + request: { + command: 'readText', + inspectPoint: true, + x: 196, + y: 122, + appBundleId: 'com.apple.Preferences', + }, + result: { + text: 'General', + elements: [ + { + text: 'General', + label: 'General', + identifier: 'General', + type: 'Button', + value: 'General', + frame: { x: 146, y: 102, width: 100, height: 40 }, + hittable: true, + }, + ], + }, + }, + { + command: 'ios.runner.readText', + deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, + platform: 'apple', + request: { + command: 'readText', + inspectPoint: true, + x: 0, + y: 0, + appBundleId: 'com.apple.Preferences', + }, + result: { elements: [] }, + }, { command: 'ios.runner.tap', deviceId: PROVIDER_SCENARIO_IOS_SIMULATOR.id, diff --git a/test/integration/smoke-web-platform.test.ts b/test/integration/smoke-web-platform.test.ts index 7100ea9e6b..d08accadcf 100644 --- a/test/integration/smoke-web-platform.test.ts +++ b/test/integration/smoke-web-platform.test.ts @@ -62,7 +62,7 @@ test('web shutdown cleanup reaps the exact daemon that survived graceful shutdow const daemonPid = child.pid ?? 0; assert.ok(daemonPid > 0, 'expected the fake daemon to have a pid'); t.after(() => { - if (isProcessAlive(daemonPid)) process.kill(daemonPid, 'SIGKILL'); + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); rmSync(root, { recursive: true, force: true }); }); @@ -101,9 +101,27 @@ test('web shutdown cleanup reaps the exact daemon that survived graceful shutdow true, 'expected cleanup to escalate after the child ignored SIGTERM', ); + await waitForChildExit(child, 1_000); assert.equal(isProcessAlive(daemonPid), false); }); +async function waitForChildExit(child: ReturnType, timeoutMs: number): Promise { + if (child.exitCode !== null || child.signalCode !== null) return; + await new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + child.off('exit', onExit); + reject( + new Error(`child process ${child.pid ?? ''} did not exit within ${timeoutMs}ms`), + ); + }, timeoutMs); + const onExit = () => { + clearTimeout(timeout); + resolve(); + }; + child.once('exit', onExit); + }); +} + type StepRecord = { step: string; command: string; diff --git a/website/docs/docs/commands.md b/website/docs/docs/commands.md index d46a5df559..1f8d845af7 100644 --- a/website/docs/docs/commands.md +++ b/website/docs/docs/commands.md @@ -371,10 +371,16 @@ agent-device close ```bash agent-device snapshot [--diff] [-i] [--depth, -d ] [--scope, -s ] [--raw] [--actions] [--force-full] [--timeout ] agent-device diff snapshot [-i] [-d ] [-s ] [--raw] +agent-device inspect-point --x --y agent-device get text @e1 agent-device get attrs @e1 ``` +- `inspect-point` is available on iOS Simulator sessions. It returns a bounded list of the + accessibility elements containing the coordinate, ordered from the smallest frame to the largest. + An honest miss returns `no-element-at-point`; transport and runner failures still fail the command. + Inspection identifies a candidate target but is not itself a behavioral assertion. + - iOS snapshots use XCTest on simulators and physical devices. iOS `--raw` is the acquired tree on whichever backend serves the capture: it keeps offscreen nodes, decorations, and structural wrappers the default and `-i` views fold away, so a recovered raw capture shows the same hierarchy