Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
211 changes: 211 additions & 0 deletions apple/watch-helper/WatchControl.m
Original file line number Diff line number Diff line change
@@ -0,0 +1,211 @@
#import <Foundation/Foundation.h>
#import <CoreGraphics/CoreGraphics.h>
#import <objc/message.h>
#import <objc/runtime.h>
#import <dlfcn.h>
#import <mach/mach_time.h>
#import <unistd.h>
#import <math.h>

// Isolated host-side CoreSimulator input helper. The helper loads only the active Xcode's private
// frameworks, resolves every symbol dynamically, and exits after one verified transport send.
// The Indigo protocol and doubled contact shape follow Meta idb's MIT-licensed Simulator HID
// contract; no private framework header is compiled into the package.

typedef void *(*MouseMessageFn)(CGPoint *, CGPoint *, NSInteger, NSUInteger, CGSize, NSInteger);
typedef void *(*ButtonMessageFn)(int, int, int);
typedef void *(*CrownMessageFn)(double);

static const size_t kHeaderSize = 0x20;
static const size_t kPayloadSize = 0x90;
static const size_t kDoubledTouchSize = 0x140;
static const NSInteger kDigitizerTarget = 0x32;
static const int kHardwareTarget = 0x33;
static const int kButtonDown = 1;
static const int kButtonUp = 2;
static const int kHomeButtonSource = 0;

static void fail(NSString *message) {
fprintf(stderr, "%s\n", message.UTF8String);
}

static BOOL loadFramework(NSArray<NSString *> *candidates) {
for (NSString *path in candidates) {
if (dlopen(path.fileSystemRepresentation, RTLD_LAZY | RTLD_GLOBAL)) return YES;
}
return NO;
}

static NSString *developerDirectory(void) {
NSString *selected = NSProcessInfo.processInfo.environment[@"DEVELOPER_DIR"];
if (selected.length > 0) return selected.stringByStandardizingPath;
NSTask *task = [NSTask new];
task.executableURL = [NSURL fileURLWithPath:@"/usr/bin/xcode-select"];
task.arguments = @[@"-p"];
NSPipe *pipe = [NSPipe pipe];
task.standardOutput = pipe;
task.standardError = [NSPipe pipe];
if (![task launchAndReturnError:NULL]) return @"";
[task waitUntilExit];
NSData *data = [pipe.fileHandleForReading readDataToEndOfFile];
NSString *value = [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] ?: @"";
return [value stringByTrimmingCharactersInSet:NSCharacterSet.whitespaceAndNewlineCharacterSet];
}

static BOOL loadPrivateFrameworks(NSString *developerDir) {
NSString *contents = developerDir.stringByDeletingLastPathComponent;
NSArray *coreSimulator = @[
[developerDir stringByAppendingPathComponent:@"Library/PrivateFrameworks/CoreSimulator.framework/CoreSimulator"],
@"/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimulator",
];
NSArray *simulatorKit = @[
[developerDir stringByAppendingPathComponent:@"Library/PrivateFrameworks/SimulatorKit.framework/SimulatorKit"],
[contents stringByAppendingPathComponent:@"SharedFrameworks/SimulatorKit.framework/SimulatorKit"],
@"/Library/Developer/PrivateFrameworks/SimulatorKit.framework/SimulatorKit",
];
return loadFramework(coreSimulator) && loadFramework(simulatorKit);
}

static id simulatorDevice(NSString *developerDir, NSString *udid, NSError **error) {
Class contextClass = NSClassFromString(@"SimServiceContext");
SEL shared = NSSelectorFromString(@"sharedServiceContextForDeveloperDir:error:");
id context = ((id (*)(id, SEL, id, NSError **))objc_msgSend)(contextClass, shared, developerDir, error);
if (!context) return nil;
SEL defaultSet = NSSelectorFromString(@"defaultDeviceSetWithError:");
id deviceSet = ((id (*)(id, SEL, NSError **))objc_msgSend)(context, defaultSet, error);
if (!deviceSet) return nil;
NSArray *devices = ((id (*)(id, SEL))objc_msgSend)(deviceSet, NSSelectorFromString(@"devices"));
for (id device in devices) {
NSUUID *identifier = ((id (*)(id, SEL))objc_msgSend)(device, NSSelectorFromString(@"UDID"));
if ([identifier.UUIDString caseInsensitiveCompare:udid] == NSOrderedSame) return device;
}
return nil;
}

static id hidClient(id device, NSError **error) {
Class clientClass = NSClassFromString(@"SimulatorKit.SimDeviceLegacyHIDClient");
if (!clientClass) return nil;
id allocated = ((id (*)(id, SEL))objc_msgSend)(clientClass, sel_registerName("alloc"));
return ((id (*)(id, SEL, id, NSError **))objc_msgSend)(
allocated, NSSelectorFromString(@"initWithDevice:error:"), device, error);
}

static BOOL sendMessage(id client, void *message, NSError **error) {
if (!message) return NO;
dispatch_queue_t queue = dispatch_queue_create("agent-device.watch-hid", DISPATCH_QUEUE_SERIAL);
dispatch_semaphore_t finished = dispatch_semaphore_create(0);
__block NSError *completionError = nil;
void (^completion)(NSError *) = ^(NSError *inner) {
completionError = inner;
dispatch_semaphore_signal(finished);
};
SEL send = NSSelectorFromString(@"sendWithMessage:freeWhenDone:completionQueue:completion:");
((void (*)(id, SEL, void *, BOOL, dispatch_queue_t, id))objc_msgSend)(
client, send, message, YES, queue, completion);
if (dispatch_semaphore_wait(finished, dispatch_time(DISPATCH_TIME_NOW, NSEC_PER_SEC)) != 0) {
if (error) *error = [NSError errorWithDomain:@"agent-device.watch-hid" code:1
userInfo:@{NSLocalizedDescriptionKey: @"Simulator HID send timed out"}];
return NO;
}
if (completionError && error) *error = completionError;
return completionError == nil;
}

static void *touchMessage(MouseMessageFn builder, double x, double y, int phase) {
CGPoint point = CGPointMake(x, y);
void *base = builder(&point, NULL, kDigitizerTarget, phase, CGSizeMake(1, 1), 0);
if (!base) return NULL;
uint8_t *message = calloc(1, kDoubledTouchSize);
memcpy(message, base, kHeaderSize + kPayloadSize);
free(base);
*(uint32_t *)(message + 0x18) = (uint32_t)kPayloadSize;
message[0x1c] = 2;
*(uint32_t *)(message + 0x20) = 0x0b;
*(uint64_t *)(message + 0x24) = mach_absolute_time();
*(double *)(message + 0x3c) = x;
*(double *)(message + 0x44) = y;
memcpy(message + kHeaderSize + kPayloadSize, message + kHeaderSize, kPayloadSize);
uint8_t *secondEvent = message + kHeaderSize + kPayloadSize + 0x10;
*(uint32_t *)(secondEvent + 0) = 1;
*(uint32_t *)(secondEvent + 4) = 2;
return message;
}

static BOOL sendTouch(id client, MouseMessageFn builder, double x, double y, int phase, NSError **error) {
return sendMessage(client, touchMessage(builder, x, y, phase), error);
}

static BOOL tap(id client, MouseMessageFn builder, double x, double y, NSError **error) {
if (!sendTouch(client, builder, x, y, kButtonDown, error)) return NO;
usleep(60 * 1000);
return sendTouch(client, builder, x, y, kButtonUp, error);
}

static BOOL swipe(id client, MouseMessageFn builder, double x1, double y1, double x2, double y2,
NSUInteger durationMs, NSError **error) {
NSUInteger frames = MAX(2, MIN(120, durationMs / 16));
for (NSUInteger index = 0; index <= frames; index++) {
double progress = (double)index / (double)frames;
if (!sendTouch(client, builder, x1 + (x2 - x1) * progress,
y1 + (y2 - y1) * progress, kButtonDown, error)) return NO;
usleep((useconds_t)(durationMs * 1000 / frames));
}
return sendTouch(client, builder, x2, y2, kButtonUp, error);
}

static BOOL pressCrown(id client, ButtonMessageFn builder, NSError **error) {
if (!sendMessage(client, builder(kHomeButtonSource, kButtonDown, kHardwareTarget), error)) return NO;
usleep(50 * 1000);
return sendMessage(client, builder(kHomeButtonSource, kButtonUp, kHardwareTarget), error);
}

static BOOL readDouble(NSString *value, double minimum, double maximum, double *output) {
NSScanner *scanner = [NSScanner scannerWithString:value];
double number = 0;
if (![scanner scanDouble:&number] || !scanner.isAtEnd || !isfinite(number) ||
Comment thread
cubic-dev-ai[bot] marked this conversation as resolved.
number < minimum || number > maximum) return NO;
*output = number;
return YES;
}

int main(int argc, const char *argv[]) {
@autoreleasepool {
if (argc < 3) return 2;
NSString *developerDir = developerDirectory();
if (!loadPrivateFrameworks(developerDir)) {
fail(@"CoreSimulator or SimulatorKit could not be loaded from the selected Xcode");
return 1;
}
NSError *error = nil;
id device = simulatorDevice(developerDir, @(argv[1]), &error);
id client = device ? hidClient(device, &error) : nil;
if (!client) {
fail(error.localizedDescription ?: @"Watch Simulator HID client is unavailable");
return 1;
}
MouseMessageFn mouse = (MouseMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForMouseNSEvent");
ButtonMessageFn button = (ButtonMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForButton");
CrownMessageFn crown = (CrownMessageFn)dlsym(RTLD_DEFAULT, "IndigoHIDMessageForDigitalCrownEvent");
NSString *command = @(argv[2]);
BOOL sent = NO;
if ([command isEqual:@"tap"] && argc == 5 && mouse) {
double x = 0, y = 0;
sent = readDouble(@(argv[3]), 0, 1, &x) && readDouble(@(argv[4]), 0, 1, &y) &&
tap(client, mouse, x, y, &error);
} else if ([command isEqual:@"swipe"] && argc == 8 && mouse) {
double x1 = 0, y1 = 0, x2 = 0, y2 = 0, duration = 0;
sent = readDouble(@(argv[3]), 0, 1, &x1) && readDouble(@(argv[4]), 0, 1, &y1) &&
readDouble(@(argv[5]), 0, 1, &x2) && readDouble(@(argv[6]), 0, 1, &y2) &&
readDouble(@(argv[7]), 50, 5000, &duration) &&
swipe(client, mouse, x1, y1, x2, y2, (NSUInteger)duration, &error);
} else if ([command isEqual:@"crown-scroll"] && argc == 4 && crown) {
double delta = 0;
sent = readDouble(@(argv[3]), -10000, 10000, &delta) &&
sendMessage(client, crown(delta), &error);
} else if ([command isEqual:@"crown-press"] && argc == 3 && button) {
sent = pressCrown(client, button, &error);
}
if (!sent) fail(error.localizedDescription ?: @"Invalid or unsupported watch HID command");
return sent ? 0 : 2;
}
}
32 changes: 21 additions & 11 deletions docs/adr/0009-apple-platform-consolidation.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,11 @@ Accepted
> `src/platforms/**` location are superseded by the lazy platform-module registry and private
> `@agent-device/platform-*` packages. Execution cuts over command-atomically; shared physical
> mechanics move only through ADR 0019's legal injected transition or after their last legacy user.
>
> **Amended in 2026-09:** watchOS Simulator is no longer an unsupported sentinel. Discovery,
> application lifecycle, screenshots, accessibility snapshots, single-pointer input, Digital Crown
> scrolling, and Crown navigation are owned by an isolated CoreSimulator backend. Physical watchOS
> devices and operations outside that explicit fact set remain unsupported.

## Context

Expand Down Expand Up @@ -51,33 +56,38 @@ non-breaking `appleOs` discriminant — the groundwork for this — shipped in #
Adding a first-class Apple OS becomes cheap: a leaf module plus a runner-profile row. iOS/iPadOS/tvOS/macOS
are mostly relocate-and-rename (the engine never needed to know which Apple OS it drives); visionOS is scoped
net-new work (XCUITest supports it — a profile row, a build case, `#if os(visionOS)`, a widened discovery
filter, plus real spatial-input QA); watchOS is an explicit **unsupported sentinel** because XCUITest cannot
drive watchOS UI. macOS stays a distinct AppKit leaf (its helper binary and menubar/desktop surface model are
preserved). The tvOS focus-only interaction contract (no coordinate `tap`) must not be flattened across OSes,
and snapshot fidelity is uneven (the deep-RN AX-server fallback is iOS-simulator-only). The internal
filter, plus real spatial-input QA). watchOS Simulator uses an isolated CoreSimulator backend because XCUITest
cannot drive watchOS UI; its command facts are narrower than the iOS runner and are advertised only after the
selected runtime proves the required HID display. macOS stays a distinct AppKit leaf (its helper binary and
menubar/desktop surface model are preserved). The tvOS focus-only interaction contract (no coordinate `tap`)
must not be flattened across OSes, and snapshot fidelity is uneven. The internal
`Platform` collapse of `ios`+`macos` into `apple` was the last, highest-diff step; public leaf output
remains a separate compatibility projection.

This composes with ADR 0008 (the descriptor's capability facet) and ADR 0003.

Implementation status as of 2026-08:
Implementation status as of 2026-09:

- Shipped: the internal `Platform` collapse to `apple`; additive `appleOs` groundwork; the shared
Apple engine under `packages/platform-apple/src/core`; macOS leaf files under
`packages/platform-apple/src/os/macos`; a dedicated tvOS leaf under
`packages/platform-apple/src/os/tvos`;
direct internal imports to the Apple modules; the per-`AppleOS` runtime facts (including
`packages/platform-apple/src/gesture-facts.ts`; the former capability projection was retired after
its predicates moved into request-bound facts); the watchOS **unsupported sentinel** (reserved in the `AppleOS` type and interactor-rejected —
XCUITest cannot drive watchOS UI — `isSupportedAppleDeploymentLeaf`, the Apple interactor, and
gesture admission reject it — never produced by discovery); and visionOS profile/build/discovery
plus simulator-deployment evidence.
its predicates moved into request-bound facts); watchOS Simulator discovery and the isolated
CoreSimulator runtime for lifecycle, screenshots, host AX snapshots, and input/navigation only after
a successful LegacyHID display probe on the default simulator set; the physical-watchOS unsupported sentinel; and visionOS
profile/build/discovery plus simulator-deployment evidence.
- Decision-only support boundary: visionOS discovery and simulator deployment are supported and
unit-tested (`packages/platform-apple/src/inventory-classification.ts` and
`packages/platform-apple/src/deployment/runtime.test.ts`); app deployment is also admitted for
CoreDevice-backed physical devices, while XCTest-backed physical deployment is unsupported and
push remains simulator-only. No public-command coverage is claimed for visionOS or watchOS. This
boundary is recorded here without creating a command-coverage manifest for either leaf.
push remains simulator-only. watchOS simulator app deployment is admitted; `networkDump` remains
available on every Apple leaf. No public-command coverage is claimed for visionOS. watchOS public-command
coverage is limited to the Simulator operations named above; non-default simulator sets, text entry,
app switcher, orientation, settings, multi-touch, physical devices, and other unadvertised operations
fail closed. Crown scrolling is vertical amount-only; pixel-density screenshot normalization is not
supported by the watchOS backend.
- Retained compatibility: the public wire still emits `ios`/`macos` leaves through
`PUBLIC_PLATFORMS`; internal family ownership must not leak into that projection.
- Deferred: net-new visionOS spatial-input QA.
Expand Down
7 changes: 4 additions & 3 deletions docs/adr/0019-request-bound-platform-runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -324,9 +324,10 @@ not make a difficult legacy-supported cell disappear. Behavior changes require a

Apple coverage uses an exhaustive `AppleOS` fixture table and explicitly exercises iOS simulator and
physical backends where they differ, iPadOS, tvOS focus-only/no-coordinate behavior, macOS desktop,
visionOS deferred or supported cells, and the watchOS unsupported/discovery-absence sentinel. Every
fixture matches exactly one family. A loop over six families with one generic `platform: 'apple'`
device is not leaf coverage.
visionOS deferred or supported cells, the watchOS Simulator CoreSimulator/host-AX fact set gated by a
successful `simctl io <udid> enumerate` LegacyHID display probe, and the physical-watchOS unsupported
sentinel. Watch input is unavailable for non-default simulator sets. Every fixture matches exactly one family. A loop over six families
with one generic `platform: 'apple'` device is not leaf coverage.

### 3. Provider ownership is exact and fail-closed

Expand Down
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -215,6 +215,7 @@
"!apple/macos-helper/**/.build",
"apple/snapshot-bridge",
"apple/fold-helper",
"apple/watch-helper",
"android/snapshot-helper/dist",
"!android/snapshot-helper/dist/*.idsig",
"!android/snapshot-helper/README.md",
Expand Down
14 changes: 8 additions & 6 deletions packages/kernel/src/device.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ export {
// collapses to the single `apple` platform (ADR-0009 / issue #979).
export type ApplePlatform = 'ios' | 'macos';
// Explicit, stored Apple operating system. All six literals are reserved so the
// type is stable as platform support grows, but discovery only ever populates
// the four currently supported ones ('ios' | 'ipados' | 'tvos' | 'macos').
// type is stable as platform support grows. Local discovery populates iOS,
// iPadOS, tvOS, watchOS, visionOS, and macOS when the corresponding runtime is installed.
const APPLE_OS_VALUES = ['ios', 'ipados', 'tvos', 'watchos', 'visionos', 'macos'] as const;
export type AppleOS = (typeof APPLE_OS_VALUES)[number];
// Internal device platforms. Apple OSes collapse to a single `apple` platform; the
Expand Down Expand Up @@ -296,10 +296,12 @@ function resolveRunnerPlatformNameForAppleOs(
return 'macOS';
case 'visionos':
return 'visionOS';
// iOS and iPadOS share the single iOS runner profile/SDK. watchOS remains
// reserved in the type but is never produced by discovery; defaulting it to
// iOS keeps any future record on a valid runner profile without introducing
// watchOS support.
case 'watchos':
throw new AppError(
'UNSUPPORTED_PLATFORM',
'watchOS uses its isolated Simulator backend and has no XCTest runner platform.',
);
// iOS and iPadOS share the single iOS runner profile/SDK.
default:
return 'iOS';
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,21 @@ test('the runner/local partition covers the full provider-backed interactor surf
assert.deepEqual(Object.keys(interactor).sort(), classified);
});

test('watchOS local HID routing fails closed when a runner provider is injected', () => {
assert.throws(
() =>
createAppleInteractor(
{ ...IOS_SIMULATOR, id: 'watch-1', name: 'Apple Watch', appleOs: 'watchos' },
{},
recordingRunnerProvider([]),
),
(error: unknown) =>
error instanceof AppError &&
error.code === 'UNSUPPORTED_OPERATION' &&
error.message.includes('unavailable through runner providers'),
);
});

test('provider-backed interactor routes runner-command methods through the injected transport', async () => {
const calls: RecordedRunnerCall[] = [];
const interactor = createAppleInteractor(
Expand Down
16 changes: 13 additions & 3 deletions packages/platform-apple/src/__tests__/watchos-sentinel.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,8 @@ import type { DeviceInfo } from '@agent-device/kernel/device';
import type { RunnerContext } from '@agent-device/contracts/interactor-types';
import { AppError } from '@agent-device/kernel/errors';

// watchOS is an explicit unsupported sentinel: XCUITest cannot drive watchOS UI,
// so a `appleOs: 'watchos'` device must be rejected at interactor creation (the
// admission seam) rather than silently falling through to the iOS runner.
// Physical watchOS remains an explicit unsupported sentinel. Simulator watchOS uses
// the isolated CoreSimulator backend rather than silently falling through to XCTest.
const watchOsDevice: DeviceInfo = {
platform: 'apple',
id: 'watch-1',
Expand All @@ -29,6 +28,17 @@ test('createAppleInteractor rejects a watchOS device as UNSUPPORTED_PLATFORM', (
}
});

test('createAppleInteractor constructs the isolated backend for a watchOS simulator', () => {
const simulator = createAppleInteractor(
{ ...watchOsDevice, kind: 'simulator', booted: true },
{} as RunnerContext,
);
expect(simulator.tap).toBeTypeOf('function');
expect(simulator.screenshot).toBeTypeOf('function');
expect(simulator.home).toBeTypeOf('function');
expect(simulator.appSwitcher).toBeUndefined();
});

test('a non-watchOS appleOs does not trigger the watchOS sentinel', () => {
// A tvOS device must pass the watchOS admission guard. Interactor creation
// succeeds on an empty runner context, and pinning that success keeps this
Expand Down
4 changes: 2 additions & 2 deletions packages/platform-apple/src/deployment/runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -192,9 +192,9 @@ test.each([
'unsupported-platform-leaf',
],
[
'watchOS sentinel',
'watchOS simulator',
appleDevice({ appleOs: 'watchos' }),
false,
true,
false,
'unsupported-platform-leaf',
],
Expand Down
Loading