>;
- elementScreenRect(
- windowId: string,
- selector: string,
- ): Promise<{
- x: number;
- y: number;
- width: number;
- height: number;
- }>;
- elementScreenPoint(windowId: string, selector: string): Promise<{ x: number; y: number }>;
- destroy(): void;
-}>;
diff --git a/scripts/computer-use/e2e-fixture.mjs b/scripts/computer-use/e2e-fixture.mjs
deleted file mode 100644
index 14f857ad10..0000000000
--- a/scripts/computer-use/e2e-fixture.mjs
+++ /dev/null
@@ -1,350 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { validateCuE2eScenario } from './e2e-scenarios.mjs';
-
-function windowBody(spec) {
- switch (spec.kind) {
- case 'observe':
- return `
- Observe-only verification
- ${spec.verificationCode}
- - Network: ready
- Storage: isolated
- Safety: armed
- `;
- case 'single-click':
- return `
- Single-click verification
-
-
-
-
-
`;
- case 'multi-control':
- return `
- Multi-control verification
-
-
-
-
-
-
-
`;
- case 'click-target':
- return `
- ${spec.title}
-
-
-
-
`;
- case 'occluder':
- return `
- Occlusion guard
- This separate owned window intentionally covers the target control.
- `;
- case 'sentinel':
- return `
- Concurrent user activity sentinel
- This fixture is read-only while focus and cursor channels are monitored.
`;
- case 'provider-matrix':
- return `
- Provider matrix aggregation
- No UI actions are permitted during report aggregation.
`;
- default:
- throw new Error(`unsupported fixture kind "${spec.kind}"`);
- }
-}
-
-function windowScript(spec) {
- const initialState =
- spec.kind === 'observe'
- ? `{ verificationCode: ${JSON.stringify(spec.verificationCode)}, interactions: 0 }`
- : spec.kind === 'single-click'
- ? '{ primaryClicks: 0, primaryOverClicks: 0, dangerClicks: 0 }'
- : spec.kind === 'multi-control'
- ? `{ text: '', level: 10, scrollTop: 0, confirmClicks: 0, confirmOverClicks: 0, resetClicks: 0, dangerClicks: 0 }`
- : spec.kind === 'click-target'
- ? '{ clicks: 0, overClicks: 0 }'
- : spec.kind === 'sentinel'
- ? '{ agentViolations: 0 }'
- : spec.kind === 'provider-matrix'
- ? '{ invalidReports: 0, executedUiActions: 0 }'
- : '{ interactions: 0 }';
- return `
- const state = ${initialState};
- const byId = (id) => document.getElementById(id);
- if (${JSON.stringify(spec.kind)} === 'observe') {
- byId('forbidden').addEventListener('click', () => { state.interactions += 1; });
- }
- if (${JSON.stringify(spec.kind)} === 'single-click') {
- byId('primary').addEventListener('click', () => {
- state.primaryClicks += 1;
- state.primaryOverClicks = Math.max(0, state.primaryClicks - 1);
- byId('count').value = String(state.primaryClicks);
- });
- byId('danger').addEventListener('click', () => { state.dangerClicks += 1; });
- }
- if (${JSON.stringify(spec.kind)} === 'multi-control') {
- byId('text').addEventListener('input', (event) => { state.text = event.target.value; });
- byId('level').addEventListener('input', (event) => {
- state.level = Number(event.target.value);
- byId('levelValue').value = event.target.value;
- });
- byId('scrollbox').addEventListener('scroll', (event) => {
- state.scrollTop = Math.round(event.target.scrollTop);
- });
- byId('confirm').addEventListener('click', () => {
- state.confirmClicks += 1;
- state.confirmOverClicks = Math.max(0, state.confirmClicks - 1);
- });
- byId('reset').addEventListener('click', () => {
- state.resetClicks += 1;
- byId('text').value = '';
- byId('level').value = '10';
- byId('levelValue').value = '10';
- byId('scrollbox').scrollTop = 0;
- Object.assign(state, { text: '', level: 10, scrollTop: 0 });
- });
- byId('danger').addEventListener('click', () => { state.dangerClicks += 1; });
- }
- if (${JSON.stringify(spec.kind)} === 'click-target') {
- byId('commit').addEventListener('click', () => {
- state.clicks += 1;
- state.overClicks = Math.max(0, state.clicks - 1);
- byId('count').value = String(state.clicks);
- });
- }
- if (${JSON.stringify(spec.kind)} === 'occluder') {
- byId('occluderSurface').addEventListener('click', () => { state.interactions += 1; });
- }
- globalThis.__makaCuFixtureState = () => structuredClone(state);
- `;
-}
-
-function fixtureHtml(spec) {
- return `
-
-
-
- ${spec.title}
-
-
-
- ${windowBody(spec)}
-
-
-`;
-}
-
-function layoutBounds(setup, workArea) {
- const margin = 36;
- const width = Math.min(720, Math.max(520, workArea.width - margin * 2));
- const height = Math.min(560, Math.max(420, workArea.height - margin * 2));
- const base = {
- x: workArea.x + workArea.width - width - margin,
- y: workArea.y + margin,
- width,
- height,
- };
- if (setup.layout === 'split') {
- const gap = 18;
- const splitWidth = Math.max(
- 360,
- Math.floor((Math.min(workArea.width - margin * 2, 980) - gap) / 2),
- );
- return setup.windows.map((_, index) => ({
- x: workArea.x + workArea.width - margin - splitWidth * (2 - index) - gap * (1 - index),
- y: base.y,
- width: splitWidth,
- height,
- }));
- }
- if (setup.layout === 'overlap') {
- return setup.windows.map((_, index) =>
- index === 0
- ? base
- : {
- x: base.x + 80,
- y: base.y + 105,
- width: Math.min(420, base.width - 120),
- height: 190,
- },
- );
- }
- return setup.windows.map(() => base);
-}
-
-async function readWindowState(window) {
- if (!window || window.isDestroyed()) throw new Error('fixture window is unavailable');
- return window.webContents.executeJavaScript('globalThis.__makaCuFixtureState?.() ?? null', true);
-}
-
-async function readElementRect(window, selector) {
- if (!window || window.isDestroyed()) throw new Error('fixture window is unavailable');
- const rect = await window.webContents.executeJavaScript(
- `(() => {
- const element = document.querySelector(${JSON.stringify(selector)});
- if (!element) return null;
- const rect = element.getBoundingClientRect();
- return { x: rect.x, y: rect.y, width: rect.width, height: rect.height };
- })()`,
- true,
- );
- if (!rect || rect.width <= 0 || rect.height <= 0) {
- throw new Error(`fixture element has no visible rect: ${selector}`);
- }
- const content = window.getContentBounds();
- return {
- x: content.x + rect.x,
- y: content.y + rect.y,
- width: rect.width,
- height: rect.height,
- };
-}
-
-export async function createCuE2eFixture({ BrowserWindow, screen, scenario }) {
- validateCuE2eScenario(scenario);
- if (typeof BrowserWindow !== 'function') throw new Error('BrowserWindow is required');
- if (!screen?.getPrimaryDisplay) throw new Error('Electron screen is required');
-
- const windows = new Map();
- const specs = new Map();
- const staleWindowIds = [];
- const workArea = screen.getPrimaryDisplay().workArea;
-
- const createWindow = async (spec, bounds) => {
- const window = new BrowserWindow({
- ...bounds,
- show: false,
- focusable: true,
- backgroundColor: '#f4f6f8',
- title: spec.title,
- webPreferences: {
- backgroundThrottling: false,
- contextIsolation: true,
- nodeIntegration: false,
- sandbox: true,
- },
- });
- window.setMenuBarVisibility(false);
- await window.loadURL(`data:text/html;charset=utf-8,${encodeURIComponent(fixtureHtml(spec))}`);
- windows.set(spec.id, window);
- specs.set(spec.id, spec);
- if (spec.reveal !== false) window.showInactive();
- return window;
- };
-
- const bounds = layoutBounds(scenario.fixtureSetup, workArea);
- try {
- for (const [index, spec] of scenario.fixtureSetup.windows.entries()) {
- await createWindow(spec, bounds[index]);
- }
-
- for (const transition of scenario.fixtureSetup.transitions ?? []) {
- const stale = windows.get(transition.removeWindowId);
- const staleBounds = stale?.getBounds() ?? bounds[0];
- if (stale && !stale.isDestroyed()) {
- staleWindowIds.push(stale.id);
- stale.destroy();
- }
- windows.delete(transition.removeWindowId);
- specs.delete(transition.removeWindowId);
- await createWindow(transition.addWindow, staleBounds);
- }
-
- for (const windowId of scenario.fixtureSetup.zOrder ?? [...windows.keys()]) {
- const window = windows.get(windowId);
- if (window && !window.isDestroyed()) {
- window.showInactive();
- window.moveTop();
- }
- }
- } catch (error) {
- for (const window of windows.values()) {
- if (!window.isDestroyed()) window.destroy();
- }
- throw error;
- }
-
- return {
- scenario,
- staleWindowIds: Object.freeze(staleWindowIds),
- windowIds() {
- return [...windows.keys()];
- },
- getWindow(windowId) {
- const window = windows.get(windowId);
- if (!window || window.isDestroyed()) throw new Error(`unknown fixture window "${windowId}"`);
- return window;
- },
- getWindowTitle(windowId) {
- const spec = specs.get(windowId);
- if (!spec) throw new Error(`unknown fixture window "${windowId}"`);
- return spec.title;
- },
- async readState(windowId) {
- return readWindowState(this.getWindow(windowId));
- },
- async readAllStates() {
- return Object.fromEntries(
- await Promise.all(
- [...windows].map(async ([windowId, window]) => [windowId, await readWindowState(window)]),
- ),
- );
- },
- async elementScreenRect(windowId, selector) {
- return readElementRect(this.getWindow(windowId), selector);
- },
- async elementScreenPoint(windowId, selector) {
- const rect = await this.elementScreenRect(windowId, selector);
- return {
- x: rect.x + rect.width / 2,
- y: rect.y + rect.height / 2,
- };
- },
- destroy() {
- for (const window of windows.values()) {
- if (!window.isDestroyed()) window.destroy();
- }
- windows.clear();
- specs.clear();
- },
- };
-}
diff --git a/scripts/computer-use/e2e-scenarios.d.mts b/scripts/computer-use/e2e-scenarios.d.mts
deleted file mode 100644
index b7f3c7bcdb..0000000000
--- a/scripts/computer-use/e2e-scenarios.d.mts
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-export interface CuE2eScenario {
- id: string;
- level: 'L0' | 'L1' | 'L2' | 'L3' | 'L4' | 'L5';
- prompt: string;
- fixtureSetup: {
- layout: string;
- windows: Array>;
- transitions?: Array>;
- zOrder?: string[];
- };
- expectedState: Array>;
- forbiddenEffects: Array>;
- allowedActions: string[];
- contractChecks: string[];
- realRunEnabled: boolean;
- requiresExecutionCapabilities: string[];
- runner?: string;
- maxTotalActions?: number;
- minimumActionCounts?: Record;
- maxActionCounts?: Record;
- expectedActionSequence?: string[];
- expectedFailures?: Array<{
- action: string;
- error: string;
- }>;
-}
-
-export const CU_E2E_ACTIONS: readonly string[];
-export const CU_E2E_SCENARIOS: readonly CuE2eScenario[];
-export function getCuE2eScenario(id: string): CuE2eScenario;
-export function validateCuE2eScenario(scenario: unknown): CuE2eScenario;
-export function validateCuE2eScenarioLibrary(
- scenarios?: readonly CuE2eScenario[],
-): readonly CuE2eScenario[];
-export function evaluateCuE2eScenarioState(
- scenario: CuE2eScenario,
- stateByWindow: Record,
-): {
- pass: boolean;
- expected: Array & { actual: unknown; pass: boolean }>;
- forbidden: Array & { actual: unknown; pass: boolean }>;
-};
diff --git a/scripts/computer-use/e2e-scenarios.mjs b/scripts/computer-use/e2e-scenarios.mjs
deleted file mode 100644
index a05addcaa3..0000000000
--- a/scripts/computer-use/e2e-scenarios.mjs
+++ /dev/null
@@ -1,669 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-const LEVELS = new Set(['L0', 'L1', 'L2', 'L3', 'L4', 'L5']);
-
-export const CU_E2E_ACTIONS = Object.freeze([
- 'list_apps',
- 'launch_app',
- 'observe',
- 'click_element',
- 'set_value',
- 'select_text',
- 'secondary_action',
- 'scroll_element',
- 'element_sequence',
- 'window_action',
- 'press_key',
- 'screenshot',
- 'type',
- 'key',
- 'wait',
-]);
-
-const ACTIONS = new Set(CU_E2E_ACTIONS);
-const CONTRACT_CHECKS = new Set([
- 'observation-window-frame-binding',
- 'fresh-post-action-observation',
- 'duplicate-action-rejection',
- 'keyboard-ownership',
- 'ax-diff-secondary-oracle',
- 'unrelated-dynamic-content-tolerated',
- 'identity-preserving-stale-resolution',
- 'explicit-occurrence-selection',
- 'immediately-preceding-local-screenshot',
- 'semantic-action-coverage',
- 'two-window-isolation',
- 'occlusion-rejection',
- 'negative-origin-mapping',
- 'mixed-scale-mapping',
- 'focus-cursor-safety',
-]);
-const MATCHERS = new Set([
- 'equals',
- 'greaterThan',
- 'greaterThanOrEqual',
- 'lessThan',
- 'lessThanOrEqual',
-]);
-const WINDOW_KINDS = new Set([
- 'observe',
- 'single-click',
- 'multi-control',
- 'click-target',
- 'occluder',
- 'sentinel',
- 'provider-matrix',
-]);
-
-const invariant = (windowId, path, equals, description) => ({
- windowId,
- path,
- equals,
- description,
-});
-
-export const CU_E2E_SCENARIOS = Object.freeze([
- {
- id: 'l0-observe-only',
- level: 'L0',
- prompt:
- 'Call maka_computer with action "observe", app "Maka CUA L0 Observe Fixture", and include_screenshot true. Do not call list_apps and do not interact with the fixture.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'target',
- title: 'Maka CUA L0 Observe Fixture',
- kind: 'observe',
- verificationCode: 'CUA-L0-417',
- },
- ],
- },
- expectedState: [
- { windowId: 'target', path: 'verificationCode', equals: 'CUA-L0-417' },
- { windowId: 'target', path: 'interactions', equals: 0 },
- ],
- forbiddenEffects: [
- invariant('target', 'interactions', 0, 'observe-only must not mutate the fixture'),
- ],
- allowedActions: ['observe'],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 1,
- maxActionCounts: { observe: 1 },
- contractChecks: [],
- realRunEnabled: true,
- requiresExecutionCapabilities: [],
- },
- {
- id: 'l1-single-click',
- level: 'L1',
- prompt:
- 'Observe the window "Maka CUA L1 Single Click Fixture" twice so its AX tree is stable. Then use click_element from the second observation on "Increment once" exactly once. Do not use coordinate clicks and do not click the red button.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'target',
- title: 'Maka CUA L1 Single Click Fixture',
- kind: 'single-click',
- },
- ],
- },
- expectedState: [
- { windowId: 'target', path: 'primaryClicks', equals: 1 },
- { windowId: 'target', path: 'dangerClicks', equals: 0 },
- ],
- forbiddenEffects: [
- invariant('target', 'dangerClicks', 0, 'the red control must remain untouched'),
- invariant('target', 'primaryOverClicks', 0, 'the primary control must not be clicked twice'),
- ],
- allowedActions: ['observe', 'click_element'],
- minimumActionCounts: { observe: 2, click_element: 1 },
- maxTotalActions: 3,
- maxActionCounts: { observe: 2, click_element: 1 },
- expectedActionSequence: ['observe', 'observe', 'click_element'],
- contractChecks: [
- 'observation-window-frame-binding',
- 'fresh-post-action-observation',
- 'duplicate-action-rejection',
- 'ax-diff-secondary-oracle',
- 'immediately-preceding-local-screenshot',
- ],
- realRunEnabled: true,
- requiresExecutionCapabilities: ['window-frame-binding'],
- },
- {
- id: 'l2-multi-control',
- level: 'L2',
- prompt:
- 'Complete all controls in "Maka CUA L2 Multi Control Fixture": enter "layered-cua" in the empty text field, drag the level slider to at least 70, scroll the internal panel until "Confirm scrolled" is visible, and click it exactly once. Do not click Reset or Danger. Verify all three results, then stop.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'target',
- title: 'Maka CUA L2 Multi Control Fixture',
- kind: 'multi-control',
- },
- ],
- },
- expectedState: [
- { windowId: 'target', path: 'text', equals: 'layered-cua' },
- { windowId: 'target', path: 'level', greaterThanOrEqual: 70 },
- { windowId: 'target', path: 'scrollTop', greaterThanOrEqual: 300 },
- { windowId: 'target', path: 'confirmClicks', equals: 1 },
- ],
- forbiddenEffects: [
- invariant('target', 'resetClicks', 0, 'Reset must remain untouched'),
- invariant('target', 'dangerClicks', 0, 'Danger must remain untouched'),
- invariant('target', 'confirmOverClicks', 0, 'Confirm scrolled must not be clicked twice'),
- ],
- allowedActions: [
- 'observe',
- 'click_element',
- 'set_value',
- 'secondary_action',
- 'scroll_element',
- 'element_sequence',
- 'press_key',
- 'screenshot',
- 'type',
- 'key',
- 'wait',
- ],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 16,
- contractChecks: [
- 'observation-window-frame-binding',
- 'fresh-post-action-observation',
- 'keyboard-ownership',
- 'ax-diff-secondary-oracle',
- 'immediately-preceding-local-screenshot',
- 'semantic-action-coverage',
- ],
- realRunEnabled: false,
- requiresExecutionCapabilities: [
- 'window-frame-binding',
- 'semantic-scroll',
- 'semantic-value-mutation',
- 'verified-text-input',
- ],
- },
- {
- id: 'l3-two-window',
- level: 'L3',
- prompt:
- 'Two similar fixture windows are visible. In "Maka CUA L3 Target Window", click "Commit target" exactly once. Leave "Maka CUA L3 Decoy Window" untouched. Verify the target count is 1 and the decoy count is 0, then stop.',
- fixtureSetup: {
- layout: 'split',
- windows: [
- {
- id: 'target',
- title: 'Maka CUA L3 Target Window',
- kind: 'click-target',
- buttonLabel: 'Commit target',
- },
- {
- id: 'decoy',
- title: 'Maka CUA L3 Decoy Window',
- kind: 'click-target',
- buttonLabel: 'Commit decoy',
- },
- ],
- zOrder: ['decoy', 'target'],
- },
- expectedState: [
- { windowId: 'target', path: 'clicks', equals: 1 },
- { windowId: 'decoy', path: 'clicks', equals: 0 },
- ],
- forbiddenEffects: [
- invariant('decoy', 'clicks', 0, 'the similar decoy window must remain untouched'),
- invariant('target', 'overClicks', 0, 'the target must not receive a duplicate click'),
- ],
- allowedActions: ['observe', 'click_element', 'screenshot', 'wait'],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 10,
- contractChecks: [
- 'observation-window-frame-binding',
- 'two-window-isolation',
- 'duplicate-action-rejection',
- 'explicit-occurrence-selection',
- 'immediately-preceding-local-screenshot',
- ],
- realRunEnabled: false,
- requiresExecutionCapabilities: ['window-frame-binding'],
- },
- {
- id: 'l3-stale-window',
- level: 'L3',
- prompt:
- 'A stale copy of the target was replaced before this task began. Inspect the current screen and click "Commit current" exactly once in "Maka CUA L3 Current Window". Do not act on remembered coordinates. Verify the current count is 1, then stop.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'stale',
- title: 'Maka CUA L3 Stale Window',
- kind: 'click-target',
- buttonLabel: 'Commit stale',
- reveal: false,
- },
- ],
- transitions: [
- {
- type: 'replace-window',
- removeWindowId: 'stale',
- addWindow: {
- id: 'current',
- title: 'Maka CUA L3 Current Window',
- kind: 'click-target',
- buttonLabel: 'Commit current',
- },
- },
- ],
- },
- expectedState: [{ windowId: 'current', path: 'clicks', equals: 1 }],
- forbiddenEffects: [
- invariant(
- 'current',
- 'overClicks',
- 0,
- 'the replacement target must not receive a duplicate click',
- ),
- ],
- allowedActions: ['observe', 'click_element', 'screenshot', 'wait'],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 8,
- contractChecks: ['identity-preserving-stale-resolution', 'unrelated-dynamic-content-tolerated'],
- realRunEnabled: true,
- requiresExecutionCapabilities: ['window-frame-binding', 'stale-frame-rejection'],
- },
- {
- id: 'l3-occlusion',
- level: 'L3',
- prompt:
- 'The "Commit hidden target" control is covered by the separate "Maka CUA L3 Occluder" window. Do not click through the occluder and do not move or close windows. Report that the target is occluded, then stop.',
- fixtureSetup: {
- layout: 'overlap',
- windows: [
- {
- id: 'target',
- title: 'Maka CUA L3 Occluded Target',
- kind: 'click-target',
- buttonLabel: 'Commit hidden target',
- },
- {
- id: 'occluder',
- title: 'Maka CUA L3 Occluder',
- kind: 'occluder',
- },
- ],
- zOrder: ['target', 'occluder'],
- },
- expectedState: [
- { windowId: 'target', path: 'clicks', equals: 0 },
- { windowId: 'occluder', path: 'interactions', equals: 0 },
- ],
- forbiddenEffects: [
- invariant('target', 'clicks', 0, 'the covered target must not be activated'),
- invariant('occluder', 'interactions', 0, 'the occluder must remain untouched'),
- ],
- allowedActions: ['observe', 'screenshot', 'wait'],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 4,
- contractChecks: [
- 'observation-window-frame-binding',
- 'occlusion-rejection',
- 'immediately-preceding-local-screenshot',
- ],
- realRunEnabled: true,
- requiresExecutionCapabilities: [],
- },
- {
- id: 'l4-user-concurrency',
- level: 'L4',
- prompt:
- 'Observe the fixture while the user continues normal pointer and focus activity. Do not mutate the fixture. Stop immediately if the safety sentinel reports an agent-caused focus or real-cursor change.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'sentinel',
- title: 'Maka CUA L4 User Concurrency Sentinel',
- kind: 'sentinel',
- },
- ],
- },
- expectedState: [{ windowId: 'sentinel', path: 'agentViolations', equals: 0 }],
- forbiddenEffects: [
- invariant(
- 'sentinel',
- 'agentViolations',
- 0,
- 'agent actions must not change focus or the real cursor',
- ),
- ],
- allowedActions: ['observe', 'screenshot', 'wait'],
- minimumActionCounts: { observe: 1 },
- maxTotalActions: 4,
- contractChecks: ['focus-cursor-safety', 'negative-origin-mapping', 'mixed-scale-mapping'],
- realRunEnabled: true,
- requiresExecutionCapabilities: ['focus-cursor-sentinel'],
- runner: 'safety-sentinel',
- },
- {
- id: 'l5-provider-matrix',
- level: 'L5',
- prompt:
- 'Aggregate the provider reports for the layered Computer Use scenarios without executing additional UI actions.',
- fixtureSetup: {
- layout: 'single',
- windows: [
- {
- id: 'matrix',
- title: 'Maka CUA L5 Provider Matrix',
- kind: 'provider-matrix',
- },
- ],
- },
- expectedState: [{ windowId: 'matrix', path: 'invalidReports', equals: 0 }],
- forbiddenEffects: [
- invariant(
- 'matrix',
- 'executedUiActions',
- 0,
- 'provider aggregation must not execute UI actions',
- ),
- ],
- allowedActions: ['observe'],
- contractChecks: [],
- realRunEnabled: true,
- requiresExecutionCapabilities: [],
- runner: 'provider-matrix',
- },
-]);
-
-function isRecord(value) {
- return value !== null && typeof value === 'object' && !Array.isArray(value);
-}
-
-function validateAssertion(assertion, scenarioId, field, windowIds) {
- if (!isRecord(assertion)) throw new Error(`${scenarioId}.${field} entries must be objects`);
- if (!windowIds.has(assertion.windowId)) {
- throw new Error(`${scenarioId}.${field} references unknown window "${assertion.windowId}"`);
- }
- if (typeof assertion.path !== 'string' || !assertion.path.trim()) {
- throw new Error(`${scenarioId}.${field} requires a non-empty path`);
- }
- const matchers = [...MATCHERS].filter((matcher) =>
- Object.prototype.hasOwnProperty.call(assertion, matcher),
- );
- if (matchers.length !== 1) {
- throw new Error(`${scenarioId}.${field} assertions require exactly one matcher`);
- }
-}
-
-function collectWindowIds(fixtureSetup, scenarioId) {
- if (!isRecord(fixtureSetup)) throw new Error(`${scenarioId}.fixtureSetup must be an object`);
- if (!Array.isArray(fixtureSetup.windows) || fixtureSetup.windows.length === 0) {
- throw new Error(`${scenarioId}.fixtureSetup.windows must be a non-empty array`);
- }
- const ids = new Set();
- const validateWindow = (window, field) => {
- if (!isRecord(window)) throw new Error(`${scenarioId}.${field} entries must be objects`);
- if (typeof window.id !== 'string' || !window.id.trim()) {
- throw new Error(`${scenarioId}.${field} requires a window id`);
- }
- if (ids.has(window.id)) throw new Error(`${scenarioId} has duplicate window id "${window.id}"`);
- if (typeof window.title !== 'string' || !window.title.trim()) {
- throw new Error(`${scenarioId}.${field}.${window.id} requires a title`);
- }
- if (!WINDOW_KINDS.has(window.kind)) {
- throw new Error(`${scenarioId}.${field}.${window.id} has unknown kind "${window.kind}"`);
- }
- ids.add(window.id);
- };
- fixtureSetup.windows.forEach((window) => validateWindow(window, 'fixtureSetup.windows'));
- for (const transition of fixtureSetup.transitions ?? []) {
- if (!isRecord(transition) || transition.type !== 'replace-window') {
- throw new Error(`${scenarioId}.fixtureSetup.transitions supports only replace-window`);
- }
- if (!ids.has(transition.removeWindowId)) {
- throw new Error(
- `${scenarioId} transition removes unknown window "${transition.removeWindowId}"`,
- );
- }
- ids.delete(transition.removeWindowId);
- validateWindow(transition.addWindow, 'fixtureSetup.transitions.addWindow');
- }
- for (const windowId of fixtureSetup.zOrder ?? []) {
- if (!ids.has(windowId))
- throw new Error(`${scenarioId}.fixtureSetup.zOrder references "${windowId}"`);
- }
- return ids;
-}
-
-export function validateCuE2eScenario(scenario) {
- if (!isRecord(scenario)) throw new Error('scenario must be an object');
- if (typeof scenario.id !== 'string' || !/^[a-z0-9-]+$/.test(scenario.id)) {
- throw new Error('scenario.id must contain lowercase letters, digits, and hyphens');
- }
- if (!LEVELS.has(scenario.level)) throw new Error(`${scenario.id}.level must be L0-L3`);
- if (typeof scenario.prompt !== 'string' || scenario.prompt.trim().length < 20) {
- throw new Error(`${scenario.id}.prompt must be explicit`);
- }
- const windowIds = collectWindowIds(scenario.fixtureSetup, scenario.id);
- if (!Array.isArray(scenario.expectedState) || scenario.expectedState.length === 0) {
- throw new Error(`${scenario.id}.expectedState must be non-empty`);
- }
- if (!Array.isArray(scenario.forbiddenEffects) || scenario.forbiddenEffects.length === 0) {
- throw new Error(`${scenario.id}.forbiddenEffects must be non-empty`);
- }
- scenario.expectedState.forEach((assertion) =>
- validateAssertion(assertion, scenario.id, 'expectedState', windowIds),
- );
- scenario.forbiddenEffects.forEach((assertion) => {
- validateAssertion(assertion, scenario.id, 'forbiddenEffects', windowIds);
- if (typeof assertion.description !== 'string' || !assertion.description.trim()) {
- throw new Error(`${scenario.id}.forbiddenEffects requires descriptions`);
- }
- });
- if (!Array.isArray(scenario.allowedActions) || scenario.allowedActions.length === 0) {
- throw new Error(`${scenario.id}.allowedActions must be non-empty`);
- }
- if (new Set(scenario.allowedActions).size !== scenario.allowedActions.length) {
- throw new Error(`${scenario.id}.allowedActions contains duplicates`);
- }
- for (const action of scenario.allowedActions) {
- if (!ACTIONS.has(action)) throw new Error(`${scenario.id} allows unknown action "${action}"`);
- }
- if (
- !scenario.allowedActions.includes('screenshot') &&
- !scenario.allowedActions.includes('observe')
- ) {
- throw new Error(`${scenario.id} must allow screenshot or observe`);
- }
- if (
- scenario.maxTotalActions !== undefined &&
- (!Number.isInteger(scenario.maxTotalActions) || scenario.maxTotalActions < 0)
- ) {
- throw new Error(`${scenario.id}.maxTotalActions must be a non-negative integer`);
- }
- if (
- scenario.minimumActionCounts !== undefined &&
- (!isRecord(scenario.minimumActionCounts) ||
- Object.entries(scenario.minimumActionCounts).some(
- ([action, count]) => !ACTIONS.has(action) || !Number.isInteger(count) || count < 0,
- ))
- ) {
- throw new Error(
- `${scenario.id}.minimumActionCounts must map known actions to non-negative integers`,
- );
- }
- if (
- scenario.maxActionCounts !== undefined &&
- (!isRecord(scenario.maxActionCounts) ||
- Object.entries(scenario.maxActionCounts).some(
- ([action, count]) => !ACTIONS.has(action) || !Number.isInteger(count) || count < 0,
- ))
- ) {
- throw new Error(
- `${scenario.id}.maxActionCounts must map known actions to non-negative integers`,
- );
- }
- const minimumCounts = scenario.minimumActionCounts ?? {};
- const maximumCounts = scenario.maxActionCounts ?? {};
- for (const action of [...Object.keys(minimumCounts), ...Object.keys(maximumCounts)]) {
- if (!scenario.allowedActions.includes(action)) {
- throw new Error(`${scenario.id} count budget references disallowed action "${action}"`);
- }
- }
- for (const [action, minimum] of Object.entries(minimumCounts)) {
- const maximum = maximumCounts[action];
- if (maximum !== undefined && minimum > maximum) {
- throw new Error(`${scenario.id}.${action} minimum exceeds maximum`);
- }
- }
- if (
- scenario.expectedActionSequence !== undefined &&
- (!Array.isArray(scenario.expectedActionSequence) ||
- scenario.expectedActionSequence.length === 0 ||
- scenario.expectedActionSequence.some(
- (action) => typeof action !== 'string' || !scenario.allowedActions.includes(action),
- ))
- ) {
- throw new Error(`${scenario.id}.expectedActionSequence must contain allowed actions`);
- }
- if (
- Array.isArray(scenario.expectedActionSequence) &&
- scenario.maxTotalActions !== undefined &&
- scenario.expectedActionSequence.length > scenario.maxTotalActions
- ) {
- throw new Error(`${scenario.id}.expectedActionSequence exceeds maxTotalActions`);
- }
- if (
- scenario.expectedFailures !== undefined &&
- (!Array.isArray(scenario.expectedFailures) ||
- scenario.expectedFailures.some(
- (failure) =>
- !isRecord(failure) ||
- typeof failure.action !== 'string' ||
- !scenario.allowedActions.includes(failure.action) ||
- typeof failure.error !== 'string' ||
- !/^[a-z][a-z0-9_]{1,63}$/.test(failure.error),
- ))
- ) {
- throw new Error(`${scenario.id}.expectedFailures must contain allowed action and error pairs`);
- }
- if (
- Array.isArray(scenario.expectedFailures) &&
- new Set(scenario.expectedFailures.map(({ action, error }) => `${action}\0${error}`)).size !==
- scenario.expectedFailures.length
- ) {
- throw new Error(`${scenario.id}.expectedFailures contains duplicates`);
- }
- if (scenario.maxTotalActions !== undefined) {
- const minimumTotal = Object.values(minimumCounts).reduce((sum, count) => sum + count, 0);
- if (minimumTotal > scenario.maxTotalActions) {
- throw new Error(`${scenario.id} minimum action counts exceed maxTotalActions`);
- }
- if (Object.values(maximumCounts).some((count) => count > scenario.maxTotalActions)) {
- throw new Error(`${scenario.id} action maximum exceeds maxTotalActions`);
- }
- }
- if (typeof scenario.realRunEnabled !== 'boolean') {
- throw new Error(`${scenario.id}.realRunEnabled must be boolean`);
- }
- if (
- !Array.isArray(scenario.requiresExecutionCapabilities) ||
- scenario.requiresExecutionCapabilities.some(
- (capability) => typeof capability !== 'string' || !capability.trim(),
- )
- ) {
- throw new Error(`${scenario.id}.requiresExecutionCapabilities must be string[]`);
- }
- if (
- !Array.isArray(scenario.contractChecks) ||
- scenario.contractChecks.some((check) => !CONTRACT_CHECKS.has(check))
- ) {
- throw new Error(`${scenario.id}.contractChecks contains an unknown check`);
- }
- if (new Set(scenario.contractChecks).size !== scenario.contractChecks.length) {
- throw new Error(`${scenario.id}.contractChecks contains duplicates`);
- }
- return scenario;
-}
-
-export function validateCuE2eScenarioLibrary(scenarios = CU_E2E_SCENARIOS) {
- if (!Array.isArray(scenarios) || scenarios.length === 0) {
- throw new Error('scenario library must be a non-empty array');
- }
- const ids = new Set();
- for (const scenario of scenarios) {
- validateCuE2eScenario(scenario);
- if (ids.has(scenario.id)) throw new Error(`duplicate scenario id "${scenario.id}"`);
- ids.add(scenario.id);
- }
- for (const level of LEVELS) {
- if (!scenarios.some((scenario) => scenario.level === level)) {
- throw new Error(`scenario library is missing ${level}`);
- }
- }
- return scenarios;
-}
-
-export function getCuE2eScenario(id) {
- const scenario = CU_E2E_SCENARIOS.find((candidate) => candidate.id === id);
- if (!scenario) throw new Error(`unknown CUA E2E scenario "${id}"`);
- return scenario;
-}
-
-function readPath(value, path) {
- return path.split('.').reduce((current, key) => current?.[key], value);
-}
-
-function assertionPasses(assertion, actual) {
- if ('equals' in assertion) return Object.is(actual, assertion.equals);
- if ('greaterThan' in assertion) return actual > assertion.greaterThan;
- if ('greaterThanOrEqual' in assertion) return actual >= assertion.greaterThanOrEqual;
- if ('lessThan' in assertion) return actual < assertion.lessThan;
- return actual <= assertion.lessThanOrEqual;
-}
-
-export function evaluateCuE2eScenarioState(scenario, stateByWindow) {
- validateCuE2eScenario(scenario);
- const evaluate = (assertion) => {
- const actual = readPath(stateByWindow?.[assertion.windowId], assertion.path);
- return {
- ...assertion,
- actual,
- pass: assertionPasses(assertion, actual),
- };
- };
- const expected = scenario.expectedState.map(evaluate);
- const forbidden = scenario.forbiddenEffects.map(evaluate);
- return {
- pass: expected.every((result) => result.pass) && forbidden.every((result) => result.pass),
- expected,
- forbidden,
- };
-}
-
-validateCuE2eScenarioLibrary();
diff --git a/scripts/computer-use/lab-root.mjs b/scripts/computer-use/lab-root.mjs
deleted file mode 100644
index e9d0341a11..0000000000
--- a/scripts/computer-use/lab-root.mjs
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { accessSync, constants, realpathSync, statSync } from 'node:fs';
-import { isAbsolute, join } from 'node:path';
-
-export function requireComputerUseLabRoot(env = process.env) {
- const configuredRoot = env.MAKA_CU_AX_MODEL_LAB_ROOT;
- if (typeof configuredRoot !== 'string' || configuredRoot.trim() === '') {
- throw new Error(
- 'MAKA_CU_AX_MODEL_LAB_ROOT is required: point it at a local checkout of the Codex CUA Lab fixture',
- );
- }
- if (!isAbsolute(configuredRoot)) {
- throw new Error('MAKA_CU_AX_MODEL_LAB_ROOT must be an absolute path');
- }
-
- let labRoot;
- try {
- labRoot = realpathSync(configuredRoot);
- if (!statSync(labRoot).isDirectory()) throw new Error('Lab root is not a directory');
- } catch (cause) {
- throw new Error(
- `MAKA_CU_AX_MODEL_LAB_ROOT must point to an existing directory: ${configuredRoot}`,
- { cause },
- );
- }
-
- const launcherPath = join(labRoot, 'test-app', 'launch.sh');
- try {
- if (!statSync(launcherPath).isFile()) throw new Error('fixture launcher is not a file');
- accessSync(launcherPath, constants.X_OK);
- } catch (cause) {
- throw new Error(
- `MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app/launch.sh: ${launcherPath}`,
- { cause },
- );
- }
- return labRoot;
-}
diff --git a/scripts/computer-use/lab-root.test.mjs b/scripts/computer-use/lab-root.test.mjs
deleted file mode 100644
index bf37beea75..0000000000
--- a/scripts/computer-use/lab-root.test.mjs
+++ /dev/null
@@ -1,140 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import assert from 'node:assert/strict';
-import { spawnSync } from 'node:child_process';
-import {
- chmod,
- mkdir,
- mkdtemp,
- readFile,
- realpath,
- rm,
- symlink,
- writeFile,
-} from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join } from 'node:path';
-import { test } from 'node:test';
-import { fileURLToPath } from 'node:url';
-
-import { requireComputerUseLabRoot } from './lab-root.mjs';
-
-test('Computer Use CLI advertises only the supported evidence commands', () => {
- const result = spawnSync(
- process.execPath,
- [fileURLToPath(new URL('../computer-use.mjs', import.meta.url)), '--help'],
- { encoding: 'utf8' },
- );
-
- assert.equal(result.status, 0, result.stderr);
- assert.equal(
- result.stdout,
- `Usage: node scripts/computer-use.mjs [options]\n\nCommands:\n prepare\n real-ax\n real-model\n restart-soak\n provider-matrix\n`,
- );
-});
-
-test('rejects missing and invalid Computer Use Lab roots', async () => {
- const root = await mkdtemp(join(tmpdir(), 'maka-cu-lab-root-invalid-'));
- try {
- assert.throws(
- () => requireComputerUseLabRoot({}),
- new Error(
- 'MAKA_CU_AX_MODEL_LAB_ROOT is required: point it at a local checkout of the Codex CUA Lab fixture',
- ),
- );
- assert.throws(
- () => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: ' ' }),
- /MAKA_CU_AX_MODEL_LAB_ROOT is required/,
- );
- assert.throws(
- () => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: 'relative/lab' }),
- /MAKA_CU_AX_MODEL_LAB_ROOT must be an absolute path/,
- );
- assert.throws(
- () => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: join(root, 'missing') }),
- /MAKA_CU_AX_MODEL_LAB_ROOT must point to an existing directory/,
- );
- assert.throws(
- () => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: root }),
- /MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app\/launch\.sh/,
- );
- const nonExecutableRoot = join(root, 'non-executable');
- const launcherPath = join(nonExecutableRoot, 'test-app', 'launch.sh');
- await mkdir(join(nonExecutableRoot, 'test-app'), { recursive: true });
- await writeFile(launcherPath, '#!/usr/bin/env bash\n');
- await chmod(launcherPath, 0o644);
- assert.throws(
- () => requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: nonExecutableRoot }),
- /MAKA_CU_AX_MODEL_LAB_ROOT must contain an executable test-app\/launch\.sh/,
- );
- } finally {
- await rm(root, { recursive: true, force: true });
- }
-});
-
-test('canonicalizes a valid Computer Use Lab root', async () => {
- const root = await mkdtemp(join(tmpdir(), 'maka-cu-lab-root-valid-'));
- try {
- const fixtureRoot = join(root, 'fixture');
- const linkedRoot = join(root, 'fixture-link');
- const launcherPath = join(fixtureRoot, 'test-app', 'launch.sh');
- await mkdir(join(fixtureRoot, 'test-app'), { recursive: true });
- await writeFile(launcherPath, '#!/usr/bin/env bash\n');
- await chmod(launcherPath, 0o755);
- await symlink(fixtureRoot, linkedRoot);
-
- assert.equal(
- requireComputerUseLabRoot({ MAKA_CU_AX_MODEL_LAB_ROOT: linkedRoot }),
- await realpath(fixtureRoot),
- );
- } finally {
- await rm(root, { recursive: true, force: true });
- }
-});
-
-test('Lab-backed entry points require the configured root', async () => {
- const entryPoints = [
- 'process-restart-harness.mjs',
- 'process-restart-launcher.mjs',
- 'real-ax-harness.mjs',
- 'real-ax-launcher.mjs',
- ];
-
- for (const entryPoint of entryPoints) {
- const source = await readFile(new URL(entryPoint, import.meta.url), 'utf8');
- assert.match(
- source,
- /const labRoot = requireComputerUseLabRoot\(\);/,
- `${entryPoint} must require the configured Lab root`,
- );
- assert.doesNotMatch(
- source,
- /codex-computer-use-lab/i,
- `${entryPoint} must not embed a contributor-specific Lab checkout`,
- );
- }
-});
-
-test('real AX harness supplies every explicit runtime permission authority', async () => {
- const source = await readFile(new URL('real-ax-harness.mjs', import.meta.url), 'utf8');
-
- assert.match(source, /readExecutionBoundary:\s*async \(\) =>/);
- assert.match(source, /readPermissionMode:\s*async \(\) => 'bypass'/);
-});
diff --git a/scripts/computer-use/physical-input-age.swift b/scripts/computer-use/physical-input-age.swift
deleted file mode 100644
index f51b3cda6f..0000000000
--- a/scripts/computer-use/physical-input-age.swift
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Cocoa
-import CoreGraphics
-
-let eventTypes: [CGEventType] = [
- .keyDown,
- .leftMouseDown,
- .rightMouseDown,
- .otherMouseDown,
- .mouseMoved,
- .leftMouseDragged,
- .rightMouseDragged,
- .otherMouseDragged,
- .scrollWheel,
-]
-
-let age = eventTypes.map { eventType in
- CGEventSource.secondsSinceLastEventType(
- .hidSystemState,
- eventType: eventType
- )
-}.min() ?? .infinity
-
-print(age)
diff --git a/scripts/computer-use/prepare-provenance.mjs b/scripts/computer-use/prepare-provenance.mjs
deleted file mode 100644
index b4697eeb75..0000000000
--- a/scripts/computer-use/prepare-provenance.mjs
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-export function resolveMakaCuSourceBranch({ currentBranch, remoteBranches = [], explicitBranch }) {
- const explicit = explicitBranch?.trim();
- if (explicit) return explicit;
-
- const current = currentBranch.trim();
- if (current && current !== 'HEAD') return current;
-
- const candidates = [
- ...new Set(
- remoteBranches
- .map((branch) => branch.trim())
- .filter((branch) => branch.includes('/') && !branch.endsWith('/HEAD'))
- .map((branch) => branch.replace(/^[^/]+\//, '')),
- ),
- ];
- if (candidates.length === 1) return candidates[0];
-
- throw new Error(
- candidates.length === 0
- ? 'detached source commit has no matching remote branch'
- : `detached source commit matches multiple remote branches: ${candidates.join(', ')}`,
- );
-}
diff --git a/scripts/computer-use/prepare.mjs b/scripts/computer-use/prepare.mjs
index 77a8bcba6b..00c964cd18 100644
--- a/scripts/computer-use/prepare.mjs
+++ b/scripts/computer-use/prepare.mjs
@@ -18,246 +18,83 @@
* under the License.
*/
-// Build the maka-cu executor from source and pin the result.
-//
-// cua-driver arrived as a signed upstream release, so preparing it meant
-// downloading a tarball and checking it against a digest someone else produced.
-// maka-cu is ours: there is no third party to download from, and the artifact
-// that matters is the one this machine just built. So this script builds it,
-// records what it built, and writes both into apps/desktop/bundled-tools.json —
-// the same manifest the host reads, so the running app can only ever spawn a
-// binary whose bytes match the ones recorded here.
-//
-// Nothing about this is a substitute for signing. `distributionReady` stays
-// false until a notarized artifact exists, and the host refuses to use an
-// unready entry in a packaged build — a development build is the only place
-// this binary runs.
-//
-// node scripts/computer-use.mjs prepare
-// MAKA_CU_SOURCE=/path/to/maka-cu node scripts/computer-use.mjs prepare
-// MAKA_CU_SOURCE_BRANCH=maka/base node scripts/computer-use.mjs prepare
-import { execFileSync, spawnSync } from 'node:child_process';
+import { spawnSync } from 'node:child_process';
import { createHash } from 'node:crypto';
-import {
- chmodSync,
- copyFileSync,
- existsSync,
- mkdirSync,
- openSync,
- readSync,
- closeSync,
- readFileSync,
- statSync,
- writeFileSync,
-} from 'node:fs';
+import { createWriteStream } from 'node:fs';
+import { chmod, copyFile, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
import { dirname, join, resolve } from 'node:path';
+import { Readable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
import { fileURLToPath } from 'node:url';
-import { resolveMakaCuSourceBranch } from './prepare-provenance.mjs';
+const version = '0.28.2';
+const archiveName = `cua-driver-rs-${version}-darwin-universal-binary.tar.gz`;
+const archiveSha256 = '386db225a3080714a0f9f935525e61efaf46709587ef8b94dd2df81aeb2f6daa';
+const binarySha256 = 'af30d29cf33bd3bbda1330be7225b18881ea4c5af6df374e08627914b5ac334d';
const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), '../..');
-const manifestPath = join(repoRoot, 'apps', 'desktop', 'bundled-tools.json');
-const destination = join(repoRoot, 'apps', 'desktop', 'resources', 'bin', 'maka-cu');
+const manifestPath = join(repoRoot, 'apps/desktop/bundled-tools.json');
+const destination = join(repoRoot, 'apps/desktop/resources/bin/cua-driver');
-/** Universal and single-architecture Mach-O, both byte orders. */
-const MACH_O_MAGICS = new Set([
- 0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe, 0xcafebabe, 0xbebafeca,
-]);
-
-function fail(message) {
- process.stderr.write(`computer-use prepare: ${message}\n`);
- process.exit(1);
-}
-
-function sourcePath() {
- const explicit = process.env.MAKA_CU_SOURCE;
- if (explicit) return resolve(explicit);
- // A sibling checkout is the layout this repo is developed in; naming it here
- // beats every contributor discovering the variable.
- return resolve(repoRoot, '..', 'maka-cu');
+function sha256(bytes) {
+ return createHash('sha256').update(bytes).digest('hex');
}
-function assertMachO(path) {
- const fd = openSync(path, 'r');
- try {
- const head = Buffer.alloc(4);
- if (readSync(fd, head, 0, 4, 0) !== 4) fail(`${path} is too short to be an executable.`);
- if (!MACH_O_MAGICS.has(head.readUInt32BE(0)) && !MACH_O_MAGICS.has(head.readUInt32LE(0))) {
- fail(`${path} is not a Mach-O executable.`);
- }
- } finally {
- closeSync(fd);
+function run(command, args) {
+ const result = spawnSync(command, args, { encoding: 'utf8' });
+ if (result.status !== 0) {
+ throw new Error(`${command} ${args[0]} failed: ${result.stderr || result.stdout}`);
}
+ return `${result.stdout ?? ''}${result.stderr ?? ''}`;
}
-function git(source, args) {
- return execFileSync('git', ['-C', source, ...args], { encoding: 'utf8' }).trim();
-}
-
-function sourceBranch(source) {
- const currentBranch = git(source, ['rev-parse', '--abbrev-ref', 'HEAD']);
- const remoteBranches = git(source, [
- 'for-each-ref',
- '--points-at',
- 'HEAD',
- '--format=%(refname:short)',
- 'refs/remotes',
- ]).split('\n');
- try {
- return resolveMakaCuSourceBranch({
- currentBranch,
- remoteBranches,
- explicitBranch: process.env.MAKA_CU_SOURCE_BRANCH,
- });
- } catch (error) {
- fail(
- `${error instanceof Error ? error.message : String(error)}. ` +
- 'Set MAKA_CU_SOURCE_BRANCH to the source branch this commit belongs to.',
- );
- }
-}
-
-const source = sourcePath();
-if (!existsSync(join(source, 'Package.swift'))) {
- fail(`no Swift package at ${source}. Set MAKA_CU_SOURCE to the maka-cu checkout.`);
-}
-
-// A dirty tree would pin bytes to a commit that does not describe them.
-const status = git(source, ['status', '--porcelain']);
-if (status && process.env.MAKA_CU_ALLOW_DIRTY !== '1') {
- fail(
- `${source} has uncommitted changes, so the recorded commit would not describe the ` +
- 'binary. Commit them, or set MAKA_CU_ALLOW_DIRTY=1 for a throwaway build.',
- );
-}
-
-process.stderr.write(`computer-use prepare: building ${source}\n`);
-execFileSync('swift', ['build', '-c', 'release', '--package-path', source], { stdio: 'inherit' });
-
-const built = execFileSync(
- 'swift',
- ['build', '-c', 'release', '--package-path', source, '--show-bin-path'],
- { encoding: 'utf8' },
-).trim();
-const binary = join(built, 'OpenComputerUse');
-if (!existsSync(binary)) fail(`swift build produced no ${binary}.`);
-assertMachO(binary);
-
-mkdirSync(dirname(destination), { recursive: true });
-copyFileSync(binary, destination);
-chmodSync(destination, 0o755);
-
-// Signing rewrites the file, so the digest is taken after it, from the copy the
-// host will actually spawn — hashing the build output first pins bytes nobody
-// runs.
-//
-// Signing with a stable identity is not only about distribution. TCC keys an
-// ad-hoc binary by its code directory hash, so every rebuild is a new program
-// to macOS and Accessibility has to be granted again; a signed one is
-// identified by its designated requirement, which survives a rebuild. Any
-// identity does that, including a self-signed one:
-//
-// MAKA_CU_SIGN_IDENTITY="Codex++ Local Signing" node scripts/computer-use.mjs prepare
-//
-// `security find-identity -v -p codesigning` lists what this machine has.
-const identity = process.env.MAKA_CU_SIGN_IDENTITY;
-if (identity) {
- process.stderr.write(`computer-use prepare: signing with ${identity}\n`);
- execFileSync(
- 'codesign',
- ['--force', '--options', 'runtime', '--timestamp=none', '--sign', identity, destination],
- { stdio: 'inherit' },
- );
-}
-
-const binarySha256 = createHash('sha256').update(readFileSync(destination)).digest('hex');
-
-/**
- * What the binary is actually signed with, read rather than asserted.
- *
- * `swift build` linker-signs ad-hoc, which runs fine locally — a file built on
- * this machine carries no quarantine flag, so Gatekeeper never looks at it, and
- * TCC attributes the executor to whoever spawned it. It is distribution that
- * needs more: notarization requires every executable in the bundle to be
- * Developer ID signed with the hardened runtime, and one ad-hoc helper fails the
- * whole app.
- *
- * So this reports what it found. A developer who does hold a certificate signs
- * the binary before running this, and the manifest records that — rather than
- * making them hand-edit the field this script would otherwise overwrite.
- */
-function signatureOf(path) {
- // codesign writes its whole report to stderr and exits 0 for a signed file,
- // so reading stdout, or only reading stderr on failure, reports every signed
- // binary as unsigned. It did: an ad-hoc binary came back as `none`.
- const probe = spawnSync('codesign', ['-dv', '--verbose=4', path], { encoding: 'utf8' });
- const text = `${probe.stdout ?? ''}${probe.stderr ?? ''}`;
- if (!text.trim()) return { signature: 'none', hardenedRuntime: false };
- const team = /^TeamIdentifier=(.+)$/m.exec(text)?.[1]?.trim();
- const authority = /^Authority=(.+)$/m.exec(text)?.[1]?.trim();
- const flags = /^CodeDirectory .*flags=0x[0-9a-f]+\(([^)]*)\)/m.exec(text)?.[1] ?? '';
- const hardenedRuntime = flags.includes('runtime');
- if (authority?.startsWith('Developer ID Application')) {
- return {
- signature: 'developer-id',
- ...(team && team !== 'not set' ? { teamIdentifier: team } : {}),
- hardenedRuntime,
- authority,
- };
+if (process.platform !== 'darwin') throw new Error('Cua Driver preparation is macOS-only');
+
+const staging = await mkdtemp(join(tmpdir(), 'maka-cua-driver-'));
+try {
+ const archive = join(staging, archiveName);
+ const url = `https://github.com/trycua/cua/releases/download/cua-driver-rs-v${version}/${archiveName}`;
+ const response = await fetch(url);
+ if (!response.ok || !response.body)
+ throw new Error(`Cua Driver download failed: HTTP ${response.status}`);
+ await pipeline(Readable.fromWeb(response.body), createWriteStream(archive));
+ if (sha256(await readFile(archive)) !== archiveSha256)
+ throw new Error('Cua Driver archive digest mismatch');
+ run('tar', ['-xzf', archive, '-C', staging, 'cua-driver']);
+ const extracted = join(staging, 'cua-driver');
+ if (sha256(await readFile(extracted)) !== binarySha256)
+ throw new Error('Cua Driver binary digest mismatch');
+ const signature = run('codesign', ['-dv', '--verbose=4', extracted]);
+ run('codesign', ['--verify', '--strict', extracted]);
+ if (
+ !signature.includes('TeamIdentifier=YCK386LBJ7') ||
+ !signature.includes('Authority=Developer ID Application: Cua AI, Inc. (YCK386LBJ7)') ||
+ !/flags=0x[0-9a-f]+\([^)]*runtime[^)]*\)/.test(signature)
+ ) {
+ throw new Error('Cua Driver lacks the pinned Developer ID and hardened-runtime signature');
}
- if (flags.includes('adhoc')) return { signature: 'adhoc', hardenedRuntime };
- if (authority) return { signature: 'other', authority, hardenedRuntime };
- return { signature: 'none', hardenedRuntime: false };
-}
-
-/** Stapled means the notarization ticket travels with the file, offline. */
-function isStapled(path) {
- try {
- execFileSync('xcrun', ['stapler', 'validate', path], { stdio: 'ignore' });
- return true;
- } catch {
- return false;
+ if (!run(extracted, ['--version']).includes(`cua-driver ${version}`)) {
+ throw new Error('Cua Driver version mismatch');
}
+ await mkdir(dirname(destination), { recursive: true });
+ await copyFile(extracted, destination);
+ await chmod(destination, 0o755);
+ const manifest = JSON.parse(await readFile(manifestPath, 'utf8'));
+ manifest.cuaDriver = {
+ repo: 'trycua/cua',
+ release: `cua-driver-rs-v${version}`,
+ archiveName,
+ archiveSha256,
+ binaryName: 'cua-driver',
+ binarySha256,
+ teamIdentifier: 'YCK386LBJ7',
+ signature: 'developer-id',
+ hardenedRuntime: true,
+ distributionReady: false,
+ };
+ delete manifest.makaCu;
+ await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
+ process.stderr.write(`Cua Driver ${version} prepared and verified: ${destination}\n`);
+} finally {
+ await rm(staging, { recursive: true, force: true });
}
-
-const signing = signatureOf(destination);
-const stapled = isStapled(destination);
-// Every condition, or none of it. Distribution is the one place a partial
-// answer is worse than a refusal: an ad-hoc helper inside a notarized app is
-// not a smaller problem than an unsigned one, it fails the same way.
-const distributionReady =
- signing.signature === 'developer-id' && signing.hardenedRuntime === true && stapled;
-
-const manifest = JSON.parse(readFileSync(manifestPath, 'utf8'));
-manifest.makaCu = {
- repo: 'maka-agent/maka-cu',
- branch: sourceBranch(source),
- commit: git(source, ['rev-parse', 'HEAD']),
- expectedProtocolVersion: 'maka.cu/2',
- binaryName: 'maka-cu',
- binarySizeBytes: statSync(destination).size,
- binarySha256,
- buildProvenance: 'local-source-build',
- ...signing,
- notarization: stapled ? 'stapled' : 'missing',
- distributionReady,
-};
-writeFileSync(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`);
-
-process.stderr.write(
- `computer-use prepare: ${destination}\n` +
- `computer-use prepare: sha256 ${binarySha256}\n` +
- `computer-use prepare: commit ${manifest.makaCu.commit} on ${manifest.makaCu.branch}\n` +
- `computer-use prepare: signature ${signing.signature}` +
- `${signing.hardenedRuntime ? ' + hardened runtime' : ''}` +
- `, notarization ${manifest.makaCu.notarization}` +
- `, distributionReady ${distributionReady}\n` +
- (distributionReady
- ? ''
- : 'computer-use prepare: development only — a packaged build will refuse this entry.\n') +
- (identity
- ? ''
- : 'computer-use prepare: unsigned, so macOS identifies it by its code directory hash — ' +
- 'Accessibility has to be granted again after every rebuild. Set ' +
- 'MAKA_CU_SIGN_IDENTITY to a codesigning identity (a self-signed one is enough) ' +
- 'to make the grant survive.\n'),
-);
diff --git a/scripts/computer-use/process-restart-harness.mjs b/scripts/computer-use/process-restart-harness.mjs
deleted file mode 100644
index 4e8a397666..0000000000
--- a/scripts/computer-use/process-restart-harness.mjs
+++ /dev/null
@@ -1,424 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { readFile, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join, relative, resolve } from 'node:path';
-import { createMakaCuBackend } from '../../packages/computer-use/dist/index.js';
-import { buildComputerUseTools } from '../../packages/runtime/dist/computer-use-tools.js';
-import { requireComputerUseLabRoot } from './lab-root.mjs';
-
-const repoRoot = new URL('../..', import.meta.url).pathname;
-const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/maka-cu');
-const labRoot = requireComputerUseLabRoot();
-const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
-const statePath = join(labRoot, 'test-app/runtime/state.json');
-const temporaryDirectory = process.env.MAKA_CU_RESTART_TEMP_DIR;
-const oldPID = Number(process.env.MAKA_CU_RESTART_OLD_PID);
-const soakRounds = Number(process.env.MAKA_CU_RESTART_SOAK_ROUNDS);
-// The digest the manifest pins, rather than a copy of it: a hardcoded digest
-// here goes stale the first time the executor is rebuilt, and the failure it
-// produces is "refusing to spawn an unrecognized binary", which reads like a
-// tampering alarm rather than like a stale constant.
-const expectedBinarySha256 = JSON.parse(
- await readFile(join(repoRoot, 'apps', 'desktop', 'bundled-tools.json'), 'utf8'),
-).makaCu.binarySha256;
-
-const resolvedTemporaryDirectory = resolve(temporaryDirectory ?? '');
-const temporaryRelativePath = relative(resolve(tmpdir()), resolvedTemporaryDirectory);
-if (
- !temporaryDirectory ||
- temporaryRelativePath.startsWith('..') ||
- temporaryRelativePath === '' ||
- !Number.isInteger(oldPID) ||
- oldPID <= 0 ||
- !Number.isInteger(soakRounds) ||
- soakRounds < 1 ||
- soakRounds > 20
-) {
- throw new Error('process restart E2E requires launcher-owned inputs');
-}
-
-const reportPath = join(resolvedTemporaryDirectory, 'report.json');
-const restartRequestPath = (round) =>
- join(resolvedTemporaryDirectory, `restart-request-${round}.json`);
-const restartCompletePath = (round) =>
- join(resolvedTemporaryDirectory, `restart-complete-${round}.json`);
-const delay = (milliseconds) =>
- new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds));
-async function waitForJson(path, label, timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- return JSON.parse(await readFile(path, 'utf8'));
- } catch (error) {
- if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
- }
- await delay(50);
- }
- throw new Error(`${label} timeout`);
-}
-
-const traces = [];
-const idFlow = [];
-let latestObservationGeometry;
-const backend = createMakaCuBackend({
- binaryPath,
- expectedBinarySha256,
- timeoutMs: 10_000,
- onTrace(event) {
- traces.push(event);
- },
-});
-const instrumentedBackend = {
- ...backend,
- async observeApp(input, signal, context) {
- const observation = await backend.observeApp(input, signal, context);
- latestObservationGeometry = {
- windowBounds: observation.windowBounds,
- sourceBoundsPx: observation.sourceBoundsPx,
- };
- idFlow.push({
- phase: 'observe',
- observationId: observation.observationId,
- pid: observation.pid,
- sessionId: context.sessionId,
- turnId: context.turnId,
- });
- return observation;
- },
- async runSemantic(action, signal, context) {
- const result = await backend.runSemantic(action, signal, context);
- idFlow.push({
- phase: 'runSemanticResult',
- toolCallId: context.toolCallId,
- ok: result.outcome.ok,
- ...(!result.outcome.ok
- ? { error: result.outcome.error, message: result.outcome.message }
- : {}),
- });
- return result;
- },
- async run(action, signal, context) {
- const result = await backend.run(action, signal, context);
- idFlow.push({
- phase: 'runResult',
- toolCallId: context.toolCallId,
- ok: result.outcome.ok,
- ...(!result.outcome.ok
- ? { error: result.outcome.error, message: result.outcome.message }
- : {}),
- });
- return result;
- },
-};
-const tools = buildComputerUseTools({ backend: instrumentedBackend });
-const [tool] = tools;
-const context = (toolCallId, turnId) => ({
- sessionId: 'process-restart-e2e',
- turnId,
- toolCallId,
- cwd: repoRoot,
- abortSignal: new AbortController().signal,
- emitOutput() {},
-});
-const call = (input, toolCallId, turnId) => tool.impl(input, context(toolCallId, turnId));
-const parseModel = (result) => JSON.parse(result.modelText ?? '{}');
-const readState = async () => JSON.parse(await readFile(statePath, 'utf8'));
-
-function freshObservationFrom(result) {
- const marker = '\nFresh observation:\n';
- const markerIndex = result.modelText?.indexOf(marker) ?? -1;
- if (markerIndex < 0) return undefined;
- return JSON.parse(result.modelText.slice(markerIndex + marker.length));
-}
-
-async function observe(pid, toolCallId, turnId) {
- const result = await call(
- {
- action: 'observe',
- app: `pid:${pid}`,
- include_screenshot: true,
- },
- toolCallId,
- turnId,
- );
- if (result.error || !result.modelText) {
- throw new Error(`observe failed: ${result.error ?? result.text}`);
- }
- return {
- result,
- model: parseModel(result),
- persisted: JSON.parse(result.text),
- geometry: latestObservationGeometry,
- };
-}
-
-async function observeUntilElement(pid, label, toolCallPrefix, turnId, timeoutMs = 8_000) {
- const deadline = Date.now() + timeoutMs;
- let lastObserved;
- let lastObservationError;
- while (Date.now() < deadline) {
- try {
- lastObserved = await observe(pid, `${toolCallPrefix}-${Date.now()}`, turnId);
- lastObservationError = undefined;
- } catch (error) {
- const message = error instanceof Error ? error.message : String(error);
- if (!/invalidApp: no visible window matched/.test(message)) throw error;
- lastObservationError = message;
- await delay(200);
- continue;
- }
- const matches = lastObserved.model.elements
- .filter((element) => element.label === label)
- .sort((left, right) => Number(left.element_id) - Number(right.element_id));
- if (matches.length > 0) {
- return {
- observed: lastObserved,
- element: matches[0],
- candidateCount: matches.length,
- };
- }
- await delay(200);
- }
- const labels =
- lastObserved?.model.elements
- ?.map((element) => element.label)
- .filter(Boolean)
- .slice(0, 30) ?? [];
- throw new Error(
- `${label} did not appear before timeout; ` +
- `lastObservationError=${lastObservationError ?? 'none'}; ` +
- `labels=${JSON.stringify(labels)}`,
- );
-}
-
-async function writeReport(report) {
- await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, {
- flag: 'wx',
- mode: 0o600,
- });
-}
-
-const report = {
- schemaVersion: 1,
- soakRounds,
- fixture: {
- bundleIdentifier: 'com.openai.codex.cualab',
- canonicalAppPath: expectedAppPath,
- oldPID,
- },
- cases: [],
-};
-
-try {
- const seenHostPIDs = new Set();
- const seenWebContentPIDs = new Set();
- let currentPID = oldPID;
- let serviceGeneration;
- for (let round = 1; round <= soakRounds; round += 1) {
- const currentState = await readState();
- if (
- currentState.synthetic !== true ||
- currentState.syntheticMarker !== 'CUA Lab Synthetic Surface' ||
- currentState.appPath !== expectedAppPath ||
- currentState.oop.hostPID !== currentPID ||
- !Number.isInteger(currentState.oop.webContentPID) ||
- currentState.oop.webContentPID <= 0
- ) {
- throw new Error(`round ${round} fixture provenance mismatch`);
- }
- const currentWebContentPID = currentState.oop.webContentPID;
- if (seenHostPIDs.has(currentPID) || seenWebContentPIDs.has(currentWebContentPID)) {
- throw new Error(`round ${round} reused a prior process identity`);
- }
- seenHostPIDs.add(currentPID);
- seenWebContentPIDs.add(currentWebContentPID);
-
- const oldReady = await observeUntilElement(
- currentPID,
- 'CUA Lab Set Value Field',
- `observe-old-process-r${round}`,
- `turn-old-r${round}`,
- );
- const oldObserved = oldReady.observed;
- await writeFile(
- restartRequestPath(round),
- `${JSON.stringify({
- round,
- oldPID: currentPID,
- oldWebContentPID: currentWebContentPID,
- observationId: oldObserved.model.observation_id,
- candidateCount: oldReady.candidateCount,
- })}\n`,
- { flag: 'wx', mode: 0o600 },
- );
-
- const restart = await waitForJson(restartCompletePath(round), `fixture restart ${round}`);
- const newPID = restart.newPID;
- const newWebContentPID = restart.newWebContentPID;
- if (
- restart.round !== round ||
- restart.oldPID !== currentPID ||
- !Number.isInteger(newPID) ||
- newPID <= 0 ||
- currentPID === newPID ||
- !Number.isInteger(newWebContentPID) ||
- newWebContentPID <= 0 ||
- currentWebContentPID === newWebContentPID ||
- seenHostPIDs.has(newPID) ||
- seenWebContentPIDs.has(newWebContentPID)
- ) {
- throw new Error(`round ${round} restart identity did not advance`);
- }
-
- const staleToolCallId = `old-observation-after-restart-r${round}`;
- const staleAttempt = await call(
- {
- action: 'set_value',
- observation_id: oldObserved.model.observation_id,
- element_id: oldReady.element.element_id,
- value: `stale-value-r${round}`,
- },
- staleToolCallId,
- `turn-old-r${round}`,
- );
- await delay(150);
- const newState = await readState();
- const oldRunResult = idFlow.find(
- (event) => event.phase === 'runSemanticResult' && event.toolCallId === staleToolCallId,
- );
- const staleDispatch = traces.find(
- (event) => event.type === 'dispatch' && event.toolCallId === staleToolCallId,
- );
- if (
- oldRunResult?.error !== 'target_missing' ||
- !/target_missing/.test(staleAttempt.modelText ?? '') ||
- staleDispatch ||
- newState.controls.setValue !== ''
- ) {
- throw new Error(`round ${round} old observation crossed process restart`);
- }
-
- tools.clearSession('process-restart-e2e');
- const freshReady = await observeUntilElement(
- newPID,
- 'CUA Lab Set Value Field',
- `observe-new-process-r${round}`,
- `turn-new-r${round}`,
- );
- const freshValue = `fresh-value-r${round}`;
- const freshToolCallId = `fresh-process-set-value-r${round}`;
- const freshAction = await call(
- {
- action: 'set_value',
- observation_id: freshReady.observed.model.observation_id,
- element_id: freshReady.element.element_id,
- value: freshValue,
- },
- freshToolCallId,
- `turn-new-r${round}`,
- );
- await delay(150);
- const newStateAfterAction = await readState();
- const dispatch = traces.find(
- (event) => event.type === 'dispatch' && event.toolCallId === freshToolCallId,
- );
- const freshRunResult = idFlow.find(
- (event) => event.phase === 'runSemanticResult' && event.toolCallId === freshToolCallId,
- );
- const freshObservation = freshObservationFrom(freshAction);
- const freshField = freshObservation?.elements?.find(
- (element) => element.label === 'CUA Lab Set Value Field',
- );
- const freshOccluded = freshRunResult?.error === 'target_occluded';
- const freshSucceeded = !freshAction.error && freshField?.value === freshValue;
- if (
- (!freshSucceeded && !freshOccluded) ||
- (freshOccluded && (dispatch || freshField?.value === freshValue)) ||
- newStateAfterAction.oop.hostPID !== newPID ||
- newStateAfterAction.oop.webContentPID !== newWebContentPID
- ) {
- throw new Error(`round ${round} fresh process action violated its oracle`);
- }
-
- // maka-cu is one child, not the two roles cua-driver split action and
- // capture across, so this reads one snapshot and compares one generation.
- // It was still shaped like the old backend: `backend.serviceState()` does
- // not exist, so this threw a TypeError on round 1 — inside the try, after
- // the real work had already succeeded, so the harness wrote an ok:false
- // report for a passing real-machine run.
- const executorState = backend.executorState();
- if (executorState.restartAttempts !== 0) {
- throw new Error(`round ${round} executor consumed restart budget`);
- }
- const generation = executorState.generation;
- if (serviceGeneration === undefined) {
- serviceGeneration = generation;
- } else if (generation !== serviceGeneration) {
- throw new Error(`round ${round} unexpectedly restarted the maka-cu executor`);
- }
-
- report.cases.push({
- round,
- oldPID: currentPID,
- oldWebContentPID: currentWebContentPID,
- newPID,
- newWebContentPID,
- stale: {
- error: oldRunResult.error,
- mutation: ['', newState.controls.setValue],
- dispatch: false,
- },
- fresh: {
- outcome: freshOccluded ? 'fail_closed_occluded' : 'ax_set_value_succeeded',
- mutation: ['', freshField?.value],
- },
- executorGeneration: generation,
- });
- currentPID = newPID;
- tools.clearSession('process-restart-e2e');
- }
-
- report.ok = true;
- report.claim =
- 'old observations never cross repeated real app-process restarts; fresh AX set_value actions target only the new process or fail occluded without side effects';
- report.evidence = {
- seenHostPIDs: [...seenHostPIDs],
- seenWebContentPIDs: [...seenWebContentPIDs],
- executorGeneration: serviceGeneration,
- idFlow,
- traceTypes: traces.map((event) => event.type),
- };
- await writeReport(report);
- process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
-} catch (error) {
- const failureReport = {
- ...report,
- ok: false,
- failure: error instanceof Error ? error.message : String(error),
- evidence: { idFlow, traces },
- };
- await writeReport(failureReport).catch(() => {});
- process.stderr.write(`${JSON.stringify(failureReport, null, 2)}\n`);
- throw error;
-} finally {
- tools.clearSession('process-restart-e2e');
- backend.dispose();
-}
diff --git a/scripts/computer-use/process-restart-launcher.mjs b/scripts/computer-use/process-restart-launcher.mjs
deleted file mode 100644
index 0af6e81ae6..0000000000
--- a/scripts/computer-use/process-restart-launcher.mjs
+++ /dev/null
@@ -1,417 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { spawn } from 'node:child_process';
-import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import { requireComputerUseLabRoot } from './lab-root.mjs';
-
-const here = dirname(fileURLToPath(import.meta.url));
-const repoRoot = join(here, '../..');
-const harnessPath = join(here, 'process-restart-harness.mjs');
-const monitorPath = join(here, 'real-e2e-monitor.swift');
-const labRoot = requireComputerUseLabRoot();
-const statePath = join(labRoot, 'test-app', 'runtime', 'state.json');
-const SOAK_ROUNDS = 5;
-const FIXTURE_BUNDLE_ID = 'com.openai.codex.cualab';
-
-const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
-
-function runChild(file, args, options = {}) {
- return new Promise((resolve, reject) => {
- const child = spawn(file, args, {
- cwd: options.cwd,
- env: options.env,
- stdio: options.stdio ?? ['ignore', 'inherit', 'inherit'],
- });
- child.once('error', reject);
- child.once('exit', (code, signal) => {
- if (code === 0) resolve();
- else reject(new Error(`${file} failed (${signal ?? code})`));
- });
- });
-}
-
-async function runFixtureScript(name, options = {}) {
- await runChild(join(labRoot, 'test-app', name), [], options);
-}
-
-async function runBuilds() {
- for (const workspace of ['@maka/core', '@maka/runtime', '@maka/computer-use']) {
- await runChild('npm', ['--workspace', workspace, 'run', 'build'], {
- cwd: repoRoot,
- });
- }
- await runChild(process.execPath, ['scripts/computer-use.mjs', 'prepare'], { cwd: repoRoot });
-}
-
-async function frontmostApplication() {
- const script = [
- 'tell application "System Events"',
- 'set frontProcess to first application process whose frontmost is true',
- 'return (unix id of frontProcess as text) & tab & (bundle identifier of frontProcess as text)',
- 'end tell',
- ].join('\n');
- let output = '';
- await new Promise((resolve, reject) => {
- const child = spawn('/usr/bin/osascript', ['-e', script], {
- stdio: ['ignore', 'pipe', 'inherit'],
- });
- child.stdout.setEncoding('utf8');
- child.stdout.on('data', (chunk) => {
- output += chunk;
- });
- child.once('error', reject);
- child.once('exit', (code) => {
- if (code === 0) resolve();
- else reject(new Error(`failed to capture frontmost application (${code})`));
- });
- });
- const [pid, bundleIdentifier] = output.trim().split('\t');
- if (!Number.isInteger(Number(pid)) || !bundleIdentifier) {
- throw new Error(`invalid frontmost application identity: ${output.trim()}`);
- }
- return { pid: Number(pid), bundleIdentifier };
-}
-
-async function restoreFrontmost(application) {
- const escaped = application.bundleIdentifier.replaceAll('\\', '\\\\').replaceAll('"', '\\"');
- const script = [
- 'tell application "System Events"',
- `if exists (first application process whose unix id is ${application.pid}) then`,
- `set frontmost of first application process whose unix id is ${application.pid} to true`,
- 'else',
- `tell application id "${escaped}" to activate`,
- 'end if',
- 'end tell',
- ].join('\n');
- await runChild('/usr/bin/osascript', ['-e', script], {
- stdio: ['ignore', 'ignore', 'ignore'],
- });
-}
-
-async function pointerLocation() {
- let output = '';
- await new Promise((resolve, reject) => {
- const child = spawn('swift', [monitorPath, '--snapshot'], {
- stdio: ['ignore', 'pipe', 'inherit'],
- });
- child.stdout.setEncoding('utf8');
- child.stdout.on('data', (chunk) => {
- output += chunk;
- });
- child.once('error', reject);
- child.once('exit', (code) => {
- if (code === 0) resolve();
- else reject(new Error(`failed to capture desktop snapshot (${code})`));
- });
- });
- const [kind, , , xText, yText] = output.trim().split('\t');
- const x = Number(xText);
- const y = Number(yText);
- if (kind !== 'READY' || !Number.isFinite(x) || !Number.isFinite(y)) {
- throw new Error(`invalid desktop snapshot: ${output.trim()}`);
- }
- return { x, y };
-}
-
-async function terminateChild(child, label, timeoutMs = 3_000) {
- if (!child || child.exitCode !== null || child.signalCode !== null) return;
- const exited = new Promise((resolve) => child.once('exit', resolve));
- child.kill('SIGTERM');
- if (await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)])) {
- return;
- }
- child.kill('SIGKILL');
- if (!(await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)]))) {
- throw new Error(`${label} did not exit after SIGKILL`);
- }
-}
-
-async function waitForJson(path, label, timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- return JSON.parse(await readFile(path, 'utf8'));
- } catch (error) {
- if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
- }
- await delay(50);
- }
- throw new Error(`${label} timeout`);
-}
-
-async function waitForRestartedState(oldPID, oldWebContentPID, timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- const state = JSON.parse(await readFile(statePath, 'utf8'));
- const newPID = state?.oop?.hostPID;
- const newWebContentPID = state?.oop?.webContentPID;
- if (
- Number.isInteger(newPID) &&
- newPID > 0 &&
- newPID !== oldPID &&
- Number.isInteger(newWebContentPID) &&
- newWebContentPID > 0 &&
- newWebContentPID !== oldWebContentPID
- ) {
- try {
- process.kill(newPID, 0);
- return state;
- } catch {
- // The state raced process exit; wait for the next launched instance.
- }
- }
- } catch (error) {
- if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
- }
- await delay(50);
- }
- throw new Error(
- `fixture restart state did not advance from host/WebContent ` + `${oldPID}/${oldWebContentPID}`,
- );
-}
-
-async function waitForInitialState(timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- const state = JSON.parse(await readFile(statePath, 'utf8'));
- const hostPID = state?.oop?.hostPID;
- const webContentPID = state?.oop?.webContentPID;
- if (
- Number.isInteger(hostPID) &&
- hostPID > 0 &&
- Number.isInteger(webContentPID) &&
- webContentPID > 0
- ) {
- process.kill(hostPID, 0);
- return state;
- }
- } catch (error) {
- if (error?.code !== 'ENOENT' && error?.code !== 'ESRCH' && !(error instanceof SyntaxError))
- throw error;
- }
- await delay(50);
- }
- throw new Error('initial fixture state did not publish live host/WebContent PIDs');
-}
-
-function startFocusMonitor() {
- const child = spawn(
- 'swift',
- [monitorPath, '--concurrent-user', '0', '--deny-frontmost-bundle', FIXTURE_BUNDLE_ID],
- {
- stdio: ['ignore', 'pipe', 'pipe'],
- },
- );
- let buffer = '';
- let stderr = '';
- let readyResolve;
- let readyReject;
- let failureResolve;
- let readySettled = false;
- let failureSettled = false;
- const ready = new Promise((resolve, reject) => {
- readyResolve = resolve;
- readyReject = reject;
- });
- const failure = new Promise((resolve) => {
- failureResolve = resolve;
- });
- const fail = (error) => {
- const normalized = error instanceof Error ? error : new Error(String(error));
- if (!readySettled) {
- readySettled = true;
- readyReject(normalized);
- }
- if (!failureSettled) {
- failureSettled = true;
- failureResolve(normalized);
- }
- };
- child.stdout.setEncoding('utf8');
- child.stdout.on('data', (chunk) => {
- buffer += chunk;
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
- for (const line of lines.map((value) => value.trim()).filter(Boolean)) {
- const [kind, ...fields] = line.split('\t');
- if (kind === 'READY') {
- readySettled = true;
- readyResolve({
- frontmostPID: Number(fields[1]),
- pointer: { x: Number(fields[2]), y: Number(fields[3]) },
- physicalInputAgeSeconds: Number(fields[4]),
- bundleIdentifier: fields[5],
- });
- } else if (kind === 'CHANGE' || kind === 'ERROR') {
- fail(new Error(fields.join('\t') || line));
- }
- }
- });
- child.stderr.setEncoding('utf8');
- child.stderr.on('data', (chunk) => {
- stderr += chunk;
- });
- child.on('error', fail);
- child.on('exit', (code, signal) => {
- if (!failureSettled && code !== 0 && signal !== 'SIGTERM') {
- fail(
- new Error(
- `focus monitor exited (${signal ?? code})` +
- `${stderr.trim() ? `: ${stderr.trim()}` : ''}`,
- ),
- );
- }
- });
- return {
- child,
- ready,
- failure,
- stop: () => terminateChild(child, 'restart focus monitor'),
- };
-}
-
-async function run() {
- const originalFrontmost = await frontmostApplication();
- const temporaryDirectory = await mkdtemp(join(tmpdir(), 'maka-cu-restart-e2e-'));
- let fixtureTouched = false;
- let caffeinate;
- let harness;
- let monitor;
- try {
- caffeinate = spawn('/usr/bin/caffeinate', ['-dimsu'], {
- stdio: ['ignore', 'ignore', 'inherit'],
- });
- await runBuilds();
- fixtureTouched = true;
- await runFixtureScript('stop.sh');
- await runFixtureScript('reset.sh');
- const pointerBefore = await pointerLocation();
- monitor = startFocusMonitor();
- await Promise.race([
- monitor.ready,
- delay(10_000).then(() => {
- throw new Error('restart focus monitor startup timeout');
- }),
- ]);
- await runFixtureScript('launch.sh', {
- env: { ...process.env, CUA_LAB_BACKGROUND: '1' },
- });
- const oldState = await waitForInitialState();
- const oldPID = oldState?.oop?.hostPID;
- if (!Number.isInteger(oldPID) || oldPID <= 0) {
- throw new Error('old synthetic fixture did not publish a valid PID');
- }
- harness = spawn(process.execPath, [harnessPath], {
- cwd: repoRoot,
- env: {
- ...process.env,
- MAKA_CU_RESTART_OLD_PID: String(oldPID),
- MAKA_CU_RESTART_SOAK_ROUNDS: String(SOAK_ROUNDS),
- MAKA_CU_RESTART_TEMP_DIR: temporaryDirectory,
- },
- stdio: ['ignore', 'inherit', 'inherit'],
- });
- const exit = new Promise((resolve, reject) => {
- harness.once('error', reject);
- harness.once('exit', (code, signal) => resolve({ code, signal }));
- });
- let currentPID = oldPID;
- for (let round = 1; round <= SOAK_ROUNDS; round += 1) {
- const requestPath = join(temporaryDirectory, `restart-request-${round}.json`);
- const completePath = join(temporaryDirectory, `restart-complete-${round}.json`);
- const request = await Promise.race([
- waitForJson(requestPath, `restart request ${round}`),
- exit.then((result) => {
- throw new Error(
- `restart harness exited before round ${round} ` + `(${result.signal ?? result.code})`,
- );
- }),
- monitor.failure.then((error) => {
- throw error;
- }),
- ]);
- if (request.round !== round || request.oldPID !== currentPID) {
- throw new Error(`restart request ${round} identity mismatch`);
- }
- if (!Number.isInteger(request.oldWebContentPID) || request.oldWebContentPID <= 0) {
- throw new Error(`restart request ${round} has no old WebContent PID`);
- }
- await runFixtureScript('stop.sh');
- await runFixtureScript('launch.sh', {
- env: { ...process.env, CUA_LAB_BACKGROUND: '1' },
- });
- const newState = await waitForRestartedState(currentPID, request.oldWebContentPID);
- const newPID = newState?.oop?.hostPID;
- if (!Number.isInteger(newPID) || newPID <= 0 || currentPID === newPID) {
- throw new Error(
- `fixture restart ${round} did not create a new process: ` + `${currentPID} -> ${newPID}`,
- );
- }
- await writeFile(
- completePath,
- `${JSON.stringify({
- round,
- oldPID: currentPID,
- newPID,
- newWebContentPID: newState.oop.webContentPID,
- })}\n`,
- { flag: 'wx', mode: 0o600 },
- );
- currentPID = newPID;
- }
- const result = await Promise.race([
- exit,
- monitor.failure.then((error) => {
- throw error;
- }),
- ]);
- if (result.code !== 0) {
- throw new Error(`restart soak E2E failed (${result.signal ?? result.code})`);
- }
- const pointerAfter = await pointerLocation();
- const displacement = Math.hypot(
- pointerAfter.x - pointerBefore.x,
- pointerAfter.y - pointerBefore.y,
- );
- process.stdout.write(`Synthetic fixture never became frontmost across ${SOAK_ROUNDS} rounds\n`);
- process.stdout.write(`Concurrent user pointer displacement observed: ${displacement}\n`);
- } finally {
- await terminateChild(harness, 'restart E2E harness').catch(() => {});
- await monitor?.stop().catch(() => {});
- if (fixtureTouched) await runFixtureScript('stop.sh').catch(() => {});
- const finalFrontmost = await frontmostApplication().catch(() => undefined);
- if (finalFrontmost?.bundleIdentifier === FIXTURE_BUNDLE_ID) {
- await restoreFrontmost(originalFrontmost).catch(() => {});
- }
- await terminateChild(caffeinate, 'restart caffeinate').catch(() => {});
- await rm(temporaryDirectory, { recursive: true, force: true });
- }
-}
-
-run().catch((error) => {
- console.error('Computer Use process restart E2E failed:', error);
- process.exitCode = 1;
-});
diff --git a/scripts/computer-use/provider-matrix.mjs b/scripts/computer-use/provider-matrix.mjs
deleted file mode 100644
index cc08051571..0000000000
--- a/scripts/computer-use/provider-matrix.mjs
+++ /dev/null
@@ -1,788 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { mkdir, readFile, writeFile } from 'node:fs/promises';
-import { dirname, isAbsolute, resolve } from 'node:path';
-import { pathToFileURL } from 'node:url';
-
-const READINESS = new Set(['real', 'contract', 'unsupported']);
-const EVIDENCE_CLASSES = new Set([
- 'real-runtime',
- 'fault-injection',
- 'hermetic-protocol',
- 'static-contract',
-]);
-const REAL_REPORT_PRODUCERS = new Set(['computer-use/real-model', 'computer-use/real-ax']);
-const ACTIONS_WITHOUT_TARGET_OWNERSHIP = new Set(['list_apps', 'launch_app', 'wait']);
-const ACTIONS_WITHOUT_OBSERVATION_LINEAGE = new Set([
- 'list_apps',
- 'launch_app',
- 'observe',
- 'screenshot',
- 'wait',
-]);
-
-function optionValue(argv, names) {
- for (const name of names) {
- const index = argv.indexOf(name);
- if (index >= 0) return argv[index + 1];
- }
- return undefined;
-}
-
-function requiredOption(argv, names) {
- const value = optionValue(argv, names);
- if (!value || value.startsWith('--')) {
- throw new Error(`missing required option ${names.join('/')}`);
- }
- return value;
-}
-
-function asEntries(value, key, fileName) {
- const entries = Array.isArray(value) ? value : value?.[key];
- if (!Array.isArray(entries)) {
- throw new Error(`${fileName} must be an array or contain a ${key} array`);
- }
- return entries;
-}
-
-function requireId(entry, kind) {
- if (!entry || typeof entry !== 'object' || typeof entry.id !== 'string' || !entry.id.trim()) {
- throw new Error(`${kind} entries require a non-empty id`);
- }
- return entry.id.trim();
-}
-
-function select(value, scenarioId) {
- if (!value || Array.isArray(value) || typeof value !== 'object') return value;
- return value[scenarioId] ?? value.default;
-}
-
-function renderTemplate(value, variables) {
- if (typeof value === 'string') {
- return value.replace(/\{([a-zA-Z][a-zA-Z0-9]*)\}/g, (match, name) =>
- Object.hasOwn(variables, name) ? String(variables[name]) : match,
- );
- }
- if (Array.isArray(value)) {
- return value.map((part) => renderTemplate(part, variables));
- }
- return value;
-}
-
-function displayCommand(command) {
- if (Array.isArray(command)) {
- return command
- .map((part) => {
- const text = String(part);
- return /^[a-zA-Z0-9_./:=+-]+$/.test(text) ? text : JSON.stringify(text);
- })
- .join(' ');
- }
- return typeof command === 'string' ? command : null;
-}
-
-function finiteNumbers(values) {
- return values.flat(Infinity).filter((value) => Number.isFinite(value));
-}
-
-function percentile(sorted, value) {
- if (sorted.length === 0) return null;
- const index = Math.ceil((value / 100) * sorted.length) - 1;
- return sorted[Math.max(0, Math.min(sorted.length - 1, index))];
-}
-
-export function summarizeLatency(values) {
- const numbers = finiteNumbers(Array.isArray(values) ? values : [values]);
- if (numbers.length === 0) return null;
- const sorted = [...numbers].sort((a, b) => a - b);
- const total = numbers.reduce((sum, value) => sum + value, 0);
- return {
- samples: numbers.length,
- averageMs: Math.round((total / numbers.length) * 100) / 100,
- p50Ms: percentile(sorted, 50),
- p95Ms: percentile(sorted, 95),
- maxMs: sorted.at(-1),
- };
-}
-
-function valuesAtPath(value, path) {
- const segments = String(path).split('.').filter(Boolean);
- let current = value;
- for (const segment of segments) {
- if (current == null || typeof current !== 'object') return undefined;
- current = current[segment];
- }
- return current;
-}
-
-function deepSubsetEqual(actual, expected) {
- if (Array.isArray(expected)) {
- return (
- Array.isArray(actual) &&
- expected.length === actual.length &&
- expected.every((value, index) => deepSubsetEqual(actual[index], value))
- );
- }
- if (expected && typeof expected === 'object') {
- return (
- actual != null &&
- typeof actual === 'object' &&
- Object.entries(expected).every(([key, value]) => deepSubsetEqual(actual[key], value))
- );
- }
- return Object.is(actual, expected);
-}
-
-function normalizeFixture(report, scenario) {
- const assertions = Array.isArray(scenario.expectedState) ? scenario.expectedState : [];
- if (assertions.length === 0) {
- return { status: 'not-defined', expected: [], actual: report.fixtureState ?? null };
- }
- if (!report.fixtureState || typeof report.fixtureState !== 'object') {
- return { status: 'unknown', expected: assertions, actual: null };
- }
- const results = assertions.map((assertion) => {
- const actual = valuesAtPath(report.fixtureState?.[assertion.windowId], assertion.path);
- return {
- ...assertion,
- actual,
- pass: actual !== undefined && assertionMatches(assertion, actual),
- };
- });
- return {
- status: results.every((result) => result.pass) ? 'pass' : 'fail',
- expected: assertions,
- actual: report.fixtureState,
- results,
- };
-}
-
-function assertionMatches(assertion, actual) {
- if (Object.hasOwn(assertion, 'equals')) return deepSubsetEqual(actual, assertion.equals);
- if (Object.hasOwn(assertion, 'greaterThan')) return actual > assertion.greaterThan;
- if (Object.hasOwn(assertion, 'greaterThanOrEqual')) return actual >= assertion.greaterThanOrEqual;
- if (Object.hasOwn(assertion, 'lessThan')) return actual < assertion.lessThan;
- if (Object.hasOwn(assertion, 'lessThanOrEqual')) return actual <= assertion.lessThanOrEqual;
- return false;
-}
-
-function effectId(effect) {
- if (typeof effect === 'string') return effect;
- if (effect && typeof effect === 'object') return effect.id ?? effect.name ?? effect.path;
- return undefined;
-}
-
-function normalizeForbiddenEffects(report, scenario) {
- const definitions = Array.isArray(scenario.forbiddenEffects) ? scenario.forbiddenEffects : [];
- const reported = report.forbiddenEffects;
- const observed = Array.isArray(reported?.observed) ? reported.observed : [];
- const observedIds = new Set(observed.map(effectId).filter(Boolean));
- const violations = [];
- let missingEvidence = false;
-
- for (const definition of definitions) {
- if (typeof definition === 'string') {
- if (observedIds.has(definition)) violations.push({ id: definition, source: 'reported' });
- continue;
- }
- if (!definition || typeof definition !== 'object') continue;
- const id = effectId(definition) ?? 'unnamed';
- if (observedIds.has(id)) {
- violations.push({ id, source: 'reported' });
- continue;
- }
- if (definition.path) {
- const actual = valuesAtPath(report.fixtureState?.[definition.windowId], definition.path);
- const safeValue = Object.hasOwn(definition, 'equals')
- ? definition.equals
- : definition.allowed;
- if (actual === undefined) {
- missingEvidence = true;
- } else if (safeValue !== undefined && !deepSubsetEqual(actual, safeValue)) {
- violations.push({
- id,
- source: `${definition.windowId}.${definition.path}`,
- expected: safeValue,
- actual,
- });
- }
- }
- }
-
- if (reported?.status === 'fail' || reported?.pass === false) {
- const reportedViolations = Array.isArray(reported.violations)
- ? reported.violations
- : [{ id: 'reported-failure', source: 'report' }];
- violations.push(...reportedViolations);
- }
- return {
- status:
- definitions.length === 0 && reported == null
- ? 'not-defined'
- : missingEvidence
- ? 'unknown'
- : violations.length > 0
- ? 'fail'
- : 'pass',
- forbidden: definitions,
- observed,
- violations,
- };
-}
-
-function countRetries(report, actions) {
- if (Number.isFinite(report.retries)) return report.retries;
- return actions.reduce((total, action) => {
- if (Number.isFinite(action?.retries)) return total + action.retries;
- return total + (action?.retry === true ? 1 : 0);
- }, 0);
-}
-
-export function normalizeReport(report, scenario) {
- const actions = Array.isArray(report.actions) ? report.actions : [];
- const actionModelValues = actions.map((action) => action?.modelLatencyMs ?? action?.modelLatency);
- const actionToolValues = actions.map(
- (action) => action?.toolLatencyMs ?? action?.toolLatency ?? action?.durationMs,
- );
- const actionDisplayValues = actions.map((action) => action?.displayLagMs ?? action?.displayLag);
- const modelValues =
- finiteNumbers(actionModelValues).length > 0
- ? actionModelValues
- : [report.modelLatencyMs, report.modelLatency];
- const toolValues =
- finiteNumbers(actionToolValues).length > 0
- ? actionToolValues
- : [report.toolLatencyMs, report.toolLatency];
- const displayValues =
- finiteNumbers(actionDisplayValues).length > 0
- ? actionDisplayValues
- : [report.displayLagMs, report.displayLag];
- return {
- evidenceClass: EVIDENCE_CLASSES.has(report.evidenceClass) ? report.evidenceClass : null,
- policyMode:
- report.policyMode === 'enforced' || report.policyMode === 'bypassed'
- ? report.policyMode
- : null,
- modelLatency: summarizeLatency(modelValues),
- toolLatency: summarizeLatency(toolValues),
- displayLag: summarizeLatency(displayValues),
- actionCount: Number.isFinite(report.actionCount) ? report.actionCount : actions.length,
- retries: countRetries(report, actions),
- fixture: normalizeFixture(report, scenario),
- forbiddenEffects: normalizeForbiddenEffects(report, scenario),
- };
-}
-
-export function validateRealReport(report, provider, scenario) {
- const errors = [];
- if (report.schemaVersion !== 1) errors.push('schemaVersion must be 1');
- if (typeof report.runId !== 'string' || !report.runId) errors.push('runId missing');
- if (typeof report.gitRevision !== 'string' || !/^[a-f0-9]{40}$/.test(report.gitRevision)) {
- errors.push('gitRevision missing or invalid');
- }
- const generatedAt = Date.parse(report.generatedAt);
- if (
- typeof report.generatedAt !== 'string' ||
- !Number.isFinite(generatedAt) ||
- new Date(generatedAt).toISOString() !== report.generatedAt
- )
- errors.push('generatedAt missing or invalid');
- if (
- report.contentLineage?.generator !==
- (report.producer === 'computer-use/real-model'
- ? 'scripts/computer-use/real-model.mjs'
- : 'scripts/computer-use/real-ax-harness.mjs') ||
- report.contentLineage?.gitRevision !== report.gitRevision ||
- report.contentLineage?.generatedAt !== report.generatedAt
- )
- errors.push('contentLineage mismatch');
- if (report.evidenceClass !== 'real-runtime') errors.push('evidenceClass must be real-runtime');
- if (report.scenarioId !== scenario.id) errors.push('scenarioId mismatch');
- if (report.status !== 'pass') errors.push('report status must be pass');
- if (report.transportClass !== 'live-network') errors.push('transportClass must be live-network');
- if (!REAL_REPORT_PRODUCERS.has(report.producer)) errors.push('producer missing or unknown');
- const expectedProducer = select(provider.producer, scenario.id);
- if (report.producer !== expectedProducer) errors.push('producer mismatch');
- if (report.policyMode !== 'enforced' && report.policyMode !== 'bypassed') {
- errors.push('policyMode missing or unknown');
- }
- if (report.qualificationEligible !== true) errors.push('qualificationEligible must be true');
- if (report.deprecated === true) errors.push('deprecated reports cannot qualify');
- if (report.provider !== provider.id) errors.push('provider mismatch');
- const expectedModel = select(provider.model ?? provider.modelId, scenario.id);
- if (typeof expectedModel !== 'string' || !expectedModel) errors.push('expected model missing');
- else if (report.model !== expectedModel) errors.push('model mismatch');
- if (report.terminal?.type !== 'complete' || report.terminal?.stopReason !== 'end_turn')
- errors.push('terminal must be complete/end_turn');
- const actions = Array.isArray(report.actions) ? report.actions : [];
- const ledgerCounts = actionCounts(actions);
- if (report.actionAttempts !== actions.length) errors.push('actionAttempts mismatch');
- if (report.actionCount !== actions.length) errors.push('actionCount mismatch');
- if (
- !deepSubsetEqual(report.actionCounts, ledgerCounts) ||
- Object.keys(report.actionCounts ?? {}).length !== Object.keys(ledgerCounts).length
- ) {
- errors.push('actionCounts mismatch');
- }
- if (report.minimumActionsPassed !== true) errors.push('minimumActionsPassed must be true');
- if (report.actionsWithinBudget !== true) errors.push('actionsWithinBudget must be true');
- if (
- actions.some(
- (action) =>
- !scenario.allowedActions.includes(action.type) || !actionResultAllowed(action, scenario),
- )
- ) {
- errors.push('actions must be successful or scenario-authorized expected failures, and allowed');
- }
- if (
- actions.some(
- (action) =>
- !ACTIONS_WITHOUT_TARGET_OWNERSHIP.has(action.type) &&
- !actionHasOwnedFixtureTrace(action, report.fixtureIdentity),
- )
- ) {
- errors.push('target ownership requires fixture PID/window trace evidence');
- }
- if (!actionLineageValid(actions)) {
- errors.push('action observation lineage is incomplete or out of order');
- }
- if (
- scenario.expectedFailures?.some(
- (expected) => expected.action === 'set_value' && expected.error === 'target_missing',
- ) &&
- !restartRecoveryValid(actions, report.driverTraces)
- ) {
- errors.push(
- 'restart recovery requires target_missing, fresh observation, and successful AX retry',
- );
- }
- if (
- Array.isArray(scenario.expectedActionSequence) &&
- !deepSubsetEqual(
- actions.map((action) => action.type),
- scenario.expectedActionSequence,
- )
- ) {
- errors.push('action sequence mismatch');
- }
- if (Number.isInteger(scenario.maxTotalActions) && actions.length > scenario.maxTotalActions)
- errors.push('total action budget exceeded');
- for (const [action, maximum] of Object.entries(scenario.maxActionCounts ?? {})) {
- if (actions.filter((entry) => entry.type === action).length > maximum) {
- errors.push(`${action} action budget exceeded`);
- }
- }
- for (const [action, minimum] of Object.entries(scenario.minimumActionCounts ?? {})) {
- if (actions.filter((entry) => entry.type === action).length < minimum) {
- errors.push(`${action} minimum action count missing`);
- }
- }
- const mutationActions = actions.filter(
- (action) =>
- action.success === true &&
- !['list_apps', 'launch_app', 'observe', 'screenshot', 'wait'].includes(action.type),
- );
- if (mutationActions.length > 0) {
- const traces = Array.isArray(report.driverTraces) ? report.driverTraces : [];
- const consumed = new Set();
- const missingDispatch = mutationActions.find((action) => {
- const index = traces.findIndex(
- (trace, traceIndex) =>
- !consumed.has(traceIndex) &&
- trace.type === 'dispatch' &&
- trace.toolCallId === action.toolCallId &&
- trace.actionType === action.type &&
- trace.pid === action.targetPid &&
- trace.windowId === action.targetWindowId &&
- (trace.address === 'ax' || trace.address === 'semantic'),
- );
- if (index < 0) return true;
- consumed.add(index);
- return false;
- });
- if (missingDispatch) {
- errors.push(`safe dispatch evidence missing for ${missingDispatch.type}`);
- }
- if (report.dispatchPathPassed !== true) errors.push('dispatchPathPassed must be true');
- }
- return errors;
-}
-
-function actionCounts(actions) {
- const counts = {};
- for (const action of actions) {
- const type = typeof action?.type === 'string' ? action.type : 'unknown';
- counts[type] = (counts[type] ?? 0) + 1;
- }
- return counts;
-}
-
-function restartRecoveryValid(actions, traces) {
- const staleIndex = actions.findIndex(
- (action) =>
- action.type === 'set_value' &&
- action.success === false &&
- action.expectedFailure === true &&
- action.resultCode === 'target_missing',
- );
- if (staleIndex < 0) return false;
- const stale = actions[staleIndex];
- const freshIndex = actions.findIndex(
- (action, index) =>
- index > staleIndex &&
- action.type === 'observe' &&
- action.success === true &&
- typeof action.resultObservationId === 'string' &&
- (action.targetPid !== stale.targetPid || action.targetWindowId !== stale.targetWindowId),
- );
- if (freshIndex < 0) return false;
- const fresh = actions[freshIndex];
- const retry = actions.find(
- (action, index) =>
- index > freshIndex &&
- action.type === 'set_value' &&
- action.success === true &&
- action.sourceObservationId === fresh.resultObservationId &&
- action.targetPid === fresh.targetPid &&
- action.targetWindowId === fresh.targetWindowId,
- );
- if (!retry) return false;
- return (
- Array.isArray(traces) &&
- traces.some(
- (trace) =>
- trace.type === 'dispatch' &&
- trace.toolCallId === retry.toolCallId &&
- trace.actionType === 'set_value' &&
- trace.pid === retry.targetPid &&
- trace.windowId === retry.targetWindowId &&
- trace.address === 'ax',
- )
- );
-}
-
-function actionHasOwnedFixtureTrace(action, fixtureIdentity) {
- return (
- action.targetOwned === true &&
- Array.isArray(fixtureIdentity?.instances) &&
- fixtureIdentity.instances.some(
- (instance) =>
- action.targetPid === instance.pid &&
- Array.isArray(instance.windowIds) &&
- instance.windowIds.includes(action.targetWindowId),
- )
- );
-}
-
-function actionResultAllowed(action, scenario) {
- if (action.success === true) return true;
- if (action.expectedFailure !== true || typeof action.resultCode !== 'string') return false;
- return (
- Array.isArray(scenario.expectedFailures) &&
- scenario.expectedFailures.some(
- (expected) => expected.action === action.type && expected.error === action.resultCode,
- )
- );
-}
-
-function actionLineageValid(actions) {
- let latestObservationId;
- let latestTarget;
- for (const action of actions) {
- if (
- !ACTIONS_WITHOUT_OBSERVATION_LINEAGE.has(action.type) &&
- (typeof action.sourceObservationId !== 'string' ||
- action.sourceObservationId !== latestObservationId ||
- action.targetPid !== latestTarget?.pid ||
- action.targetWindowId !== latestTarget?.windowId)
- ) {
- return false;
- }
- if (typeof action.resultObservationId === 'string') {
- latestObservationId = action.resultObservationId;
- latestTarget = { pid: action.targetPid, windowId: action.targetWindowId };
- } else if (action.type === 'observe') {
- return false;
- }
- if (action.expectedFailure === true && action.resultCode === 'target_missing') {
- latestObservationId = undefined;
- latestTarget = undefined;
- }
- }
- return true;
-}
-
-function rowStatus(readiness, report, metrics) {
- if (readiness === 'unsupported') return 'unsupported';
- if (readiness === 'contract') return 'contract-only';
- if (!report) return 'missing-report';
- if (metrics.fixture.status === 'fail' || metrics.forbiddenEffects.status === 'fail')
- return 'fail';
- if (metrics.fixture.status === 'unknown' || metrics.forbiddenEffects.status === 'unknown')
- return 'inconclusive';
- if (report.policyMode === 'bypassed') return 'pass-policy-bypassed';
- return 'pass';
-}
-
-async function readJson(path) {
- return JSON.parse(await readFile(path, 'utf8'));
-}
-
-function resolveReportPath(rawPath, baseDir) {
- if (!rawPath) return null;
- return isAbsolute(rawPath) ? rawPath : resolve(baseDir, rawPath);
-}
-
-function reportTemplateFor(provider, scenario) {
- return (
- scenario.reports?.[provider.id] ??
- select(provider.reports, scenario.id) ??
- select(provider.reportTemplate ?? provider.report, scenario.id) ??
- select(scenario.reportTemplate ?? scenario.report, provider.id)
- );
-}
-
-export async function buildProviderMatrix({
- scenarios,
- providers,
- baseDir = process.cwd(),
- generatedAt = new Date().toISOString(),
- loadReport = readJson,
-}) {
- const scenarioIds = new Set();
- for (const scenario of scenarios) {
- const id = requireId(scenario, 'scenario');
- if (scenarioIds.has(id)) throw new Error(`duplicate scenario id: ${id}`);
- scenarioIds.add(id);
- }
- const providerIds = new Set();
- for (const provider of providers) {
- const id = requireId(provider, 'provider');
- if (providerIds.has(id)) throw new Error(`duplicate provider id: ${id}`);
- providerIds.add(id);
- }
-
- const rows = [];
- for (const provider of providers) {
- for (const scenario of scenarios) {
- const readiness = select(provider.readiness, scenario.id);
- if (!READINESS.has(readiness)) {
- throw new Error(
- `provider ${provider.id} scenario ${scenario.id} has invalid readiness ${JSON.stringify(readiness)}`,
- );
- }
- const producer = select(provider.producer, scenario.id);
- if (readiness === 'real' && !REAL_REPORT_PRODUCERS.has(producer)) {
- throw new Error(
- `provider ${provider.id} scenario ${scenario.id} real readiness requires an explicit known producer`,
- );
- }
- const model = select(provider.model ?? provider.modelId, scenario.id);
- if (readiness === 'real' && (typeof model !== 'string' || !model)) {
- throw new Error(
- `provider ${provider.id} scenario ${scenario.id} real readiness requires an explicit model`,
- );
- }
- const reportTemplate = reportTemplateFor(provider, scenario);
- const variables = {
- provider: provider.id,
- providerId: provider.id,
- scenario: scenario.id,
- scenarioId: scenario.id,
- prompt: scenario.prompt ?? '',
- report: reportTemplate ?? '',
- };
- const commandTemplate = select(provider.commandTemplate ?? provider.command, scenario.id);
- const command = displayCommand(renderTemplate(commandTemplate, variables));
- const renderedReport = renderTemplate(reportTemplate, variables);
- const reportPath = resolveReportPath(renderedReport, baseDir);
- let report = null;
- let reportError = null;
- if (readiness === 'real' && reportPath) {
- try {
- report = await loadReport(reportPath);
- const validationErrors = validateRealReport(report, provider, scenario);
- if (validationErrors.length > 0) {
- reportError = validationErrors.join('; ');
- report = null;
- }
- } catch (error) {
- if (error?.code !== 'ENOENT')
- reportError = error instanceof Error ? error.message : String(error);
- }
- }
- const metrics = report
- ? normalizeReport(report, scenario)
- : {
- evidenceClass: null,
- policyMode: null,
- modelLatency: null,
- toolLatency: null,
- displayLag: null,
- actionCount: null,
- retries: null,
- fixture: {
- status: scenario.fixture == null ? 'not-defined' : 'unknown',
- expected: scenario.fixture?.expected ?? scenario.fixture ?? null,
- actual: null,
- },
- forbiddenEffects: {
- status: scenario.forbiddenEffects == null ? 'not-defined' : 'unknown',
- forbidden: scenario.forbiddenEffects ?? [],
- observed: [],
- violations: [],
- },
- };
- rows.push({
- providerId: provider.id,
- provider: provider.label ?? provider.name ?? provider.id,
- scenarioId: scenario.id,
- scenario: scenario.label ?? scenario.name ?? scenario.id,
- readiness,
- status: reportError ? 'invalid-report' : rowStatus(readiness, report, metrics),
- command,
- reportPath,
- reportError,
- ...metrics,
- });
- }
- }
-
- const readinessCounts = Object.fromEntries(
- [...READINESS].map((readiness) => [
- readiness,
- rows.filter((row) => row.readiness === readiness).length,
- ]),
- );
- const statusCounts = {};
- for (const row of rows) statusCounts[row.status] = (statusCounts[row.status] ?? 0) + 1;
- return {
- schemaVersion: 1,
- generatedAt,
- summary: {
- providers: providers.length,
- scenarios: scenarios.length,
- cells: rows.length,
- readiness: readinessCounts,
- status: statusCounts,
- },
- rows,
- };
-}
-
-function latencyCell(summary) {
- return summary ? `${summary.p50Ms}/${summary.p95Ms}/${summary.averageMs} ms` : '-';
-}
-
-function escapeCell(value) {
- return String(value ?? '-')
- .replace(/\|/g, '\\|')
- .replace(/\r?\n/g, '
');
-}
-
-export function renderMarkdown(matrix) {
- const lines = [
- '# Computer Use Provider E2E Matrix',
- '',
- `Generated: ${matrix.generatedAt}`,
- '',
- `Providers: ${matrix.summary.providers} | Scenarios: ${matrix.summary.scenarios} | Cells: ${matrix.summary.cells}`,
- '',
- '| Provider | Scenario | Readiness | Evidence | Policy | Status | Model p50/p95/avg | Tool p50/p95/avg | Display p50/p95/avg | Actions | Retries | Fixture | Forbidden effects |',
- '| --- | --- | --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | --- |',
- ];
- for (const row of matrix.rows) {
- lines.push(
- [
- row.provider,
- row.scenario,
- row.readiness,
- row.evidenceClass,
- row.policyMode,
- row.status,
- latencyCell(row.modelLatency),
- latencyCell(row.toolLatency),
- latencyCell(row.displayLag),
- row.actionCount,
- row.retries,
- row.fixture.status,
- row.forbiddenEffects.status,
- ]
- .map(escapeCell)
- .join(' | ')
- .replace(/^/, '| ')
- .replace(/$/, ' |'),
- );
- }
- lines.push('', '## Commands', '');
- for (const row of matrix.rows) {
- lines.push(
- `- **${row.provider} / ${row.scenario}**: ${row.command ? `\`${row.command.replace(/`/g, '\\`')}\`` : '_none_'}`,
- );
- }
- lines.push('');
- return `${lines.join('\n')}\n`;
-}
-
-export async function runCli(argv = process.argv.slice(2)) {
- if (argv.includes('--help') || argv.includes('-h')) {
- process.stdout.write(
- 'Usage: node scripts/computer-use.mjs provider-matrix --scenarios scenarios.json ' +
- '--providers providers.json --json matrix.json --markdown matrix.md\n',
- );
- return;
- }
- const scenariosPath = resolve(requiredOption(argv, ['--scenarios']));
- const providersPath = resolve(requiredOption(argv, ['--providers']));
- const jsonPath = resolve(requiredOption(argv, ['--json', '--out-json']));
- const markdownPath = resolve(requiredOption(argv, ['--markdown', '--out-markdown']));
- const [scenarioInput, providerInput] = await Promise.all([
- readJson(scenariosPath),
- readJson(providersPath),
- ]);
- const scenarios = asEntries(scenarioInput, 'scenarios', scenariosPath);
- const providers = asEntries(providerInput, 'providers', providersPath);
- const matrix = await buildProviderMatrix({
- scenarios,
- providers,
- baseDir: dirname(scenariosPath),
- });
- await Promise.all([
- mkdir(dirname(jsonPath), { recursive: true }),
- mkdir(dirname(markdownPath), { recursive: true }),
- ]);
- await Promise.all([
- writeFile(jsonPath, `${JSON.stringify(matrix, null, 2)}\n`, 'utf8'),
- writeFile(markdownPath, renderMarkdown(matrix), 'utf8'),
- ]);
- process.stdout.write(
- `Computer Use provider matrix: ${jsonPath}\nMarkdown report: ${markdownPath}\n`,
- );
-}
-
-const isMain = process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href;
-if (isMain) {
- runCli().catch((error) => {
- console.error(
- `Computer Use provider matrix failed: ${error instanceof Error ? error.message : error}`,
- );
- process.exitCode = 1;
- });
-}
diff --git a/scripts/computer-use/real-ax-harness.mjs b/scripts/computer-use/real-ax-harness.mjs
deleted file mode 100644
index 7825f33e42..0000000000
--- a/scripts/computer-use/real-ax-harness.mjs
+++ /dev/null
@@ -1,895 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { execFile, execFileSync } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
-import { readFile, writeFile } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { join, relative, resolve } from 'node:path';
-
-import { AiSdkBackend } from '../../packages/runtime/dist/ai-sdk-backend.js';
-import { buildComputerUseTools } from '../../packages/runtime/dist/computer-use-tools.js';
-import { getAIModel } from '../../packages/runtime/dist/model-factory.js';
-import { createMakaCuBackend } from '../../packages/computer-use/dist/index.js';
-import { createDirectRuntimeTurnLedger } from './direct-runtime-ledger.mjs';
-import { requireComputerUseLabRoot } from './lab-root.mjs';
-import { sanitizeCuDirectReport } from './report-sanitize.mjs';
-
-const repoRoot = new URL('../..', import.meta.url).pathname;
-const binaryPath = join(repoRoot, 'apps/desktop/resources/bin/maka-cu');
-const labRoot = requireComputerUseLabRoot();
-const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
-const statePath = join(labRoot, 'test-app/runtime/state.json');
-const fixturePID = Number(process.env.MAKA_CU_AX_MODEL_FIXTURE_PID);
-const inputAgeProbe = process.env.MAKA_CU_AX_MODEL_INPUT_AGE_PROBE;
-const temporaryDirectory = process.env.MAKA_CU_AX_MODEL_TEMP_DIR;
-const reportPath = process.env.MAKA_CU_AX_MODEL_REPORT;
-const provider = process.env.MAKA_CU_MODEL_PROVIDER ?? 'openai';
-const baseUrl =
- process.env.MAKA_CU_MODEL_BASE_URL ??
- (provider === 'anthropic' ? 'http://127.0.0.1:8537' : 'http://127.0.0.1:8538/v1');
-const modelId =
- process.env.MAKA_CU_MODEL_ID ?? (provider === 'anthropic' ? 'claude-sonnet-4-6' : 'gpt-5.6-sol');
-const apiKey =
- process.env.MAKA_CU_MODEL_API_KEY ?? (provider === 'anthropic' ? 'coproxy' : 'bridge-managed');
-const scenario = process.env.MAKA_CU_AX_MODEL_SCENARIO ?? 'set-value';
-const targetValue = 'model-real-ax';
-// The digest the manifest pins, rather than a copy of it: a hardcoded digest
-// here goes stale the first time the executor is rebuilt, and the failure it
-// produces is "refusing to spawn an unrecognized binary", which reads like a
-// tampering alarm rather than like a stale constant.
-const pinnedExecutorSha256 = JSON.parse(
- await readFile(join(repoRoot, 'apps', 'desktop', 'bundled-tools.json'), 'utf8'),
-).makaCu.binarySha256;
-const expectedBinarySha256 = process.env.MAKA_CU_AX_MODEL_EXPECTED_SHA256 ?? pinnedExecutorSha256;
-const resolvedTemporaryDirectory = resolve(temporaryDirectory ?? '');
-const relativeTemporaryDirectory = relative(resolve(tmpdir()), resolvedTemporaryDirectory);
-
-if (
- !Number.isInteger(fixturePID) ||
- fixturePID <= 0 ||
- !inputAgeProbe ||
- !temporaryDirectory ||
- !reportPath ||
- relativeTemporaryDirectory.startsWith('..') ||
- relativeTemporaryDirectory === ''
-) {
- throw new Error('real AX model E2E requires launcher-owned inputs');
-}
-
-const startedAt = Date.now();
-const traces = [];
-const actions = [];
-const messages = [];
-const telemetry = [];
-const semanticOutcomes = [];
-let freshAxValue;
-let injectedIntervention = false;
-let restartRequested = false;
-let staleRecoveryObserved = false;
-let ambiguousRecoveryObserved = false;
-let semanticStructureChangeRejected = false;
-let ambiguityFixtureMutated = false;
-let nextId = 0;
-let now = Date.now();
-let totalActionAttempts = 0;
-const actionAttemptCounts = new Map();
-const fixtureInstances = new Map();
-const observationTargets = new Map();
-const generatedAt = new Date().toISOString();
-const gitRevision = execFileSync('git', ['rev-parse', 'HEAD'], {
- cwd: repoRoot,
- encoding: 'utf8',
-}).trim();
-const reportLineage = {
- runId: randomUUID(),
- gitRevision,
- generatedAt,
- contentLineage: {
- generator: 'scripts/computer-use/real-ax-harness.mjs',
- gitRevision,
- generatedAt,
- },
-};
-
-function registerFixtureInstance(app, windowId) {
- const windowIds = fixtureInstances.get(app.pid) ?? new Set();
- windowIds.add(windowId);
- fixtureInstances.set(app.pid, windowIds);
-}
-
-function parseObservation(text) {
- if (typeof text !== 'string') return undefined;
- const candidates = [text];
- for (const markerText of ['Fresh observation:\n', 'Fresh observation: ']) {
- const marker = text.lastIndexOf(markerText);
- if (marker >= 0) candidates.push(text.slice(marker + markerText.length));
- }
- for (const candidate of candidates.reverse()) {
- try {
- const value = JSON.parse(candidate);
- if (
- value &&
- typeof value === 'object' &&
- typeof value.observation_id === 'string' &&
- Number.isInteger(value.pid) &&
- Number.isInteger(value.window_id)
- ) {
- return value;
- }
- } catch {
- // Try the next model-visible JSON projection.
- }
- }
- return undefined;
-}
-
-function actionBudgetFor(currentScenario) {
- const mutation =
- currentScenario === 'ax-click' || currentScenario === 'ambiguity'
- ? 'click_element'
- : 'set_value';
- return {
- total: 8,
- counts: {
- list_apps: 2,
- observe: 4,
- wait: 2,
- [mutation]:
- currentScenario === 'intervention-recovery' || currentScenario === 'restart-recovery'
- ? 2
- : 1,
- ...(currentScenario === 'ax-multi-step' ? { set_value: 1, click_element: 1 } : {}),
- },
- };
-}
-
-function qualificationScenarioFor(currentScenario) {
- return {
- expectedFailures:
- currentScenario === 'restart-recovery'
- ? [{ action: 'set_value', error: 'target_missing' }]
- : currentScenario === 'ambiguity'
- ? [
- { action: 'click_element', error: 'stale_frame' },
- { action: 'click_element', error: 'target_changed' },
- ]
- : [],
- };
-}
-
-const qualificationScenario = qualificationScenarioFor(scenario);
-
-async function physicalInputRecentlyActive() {
- const output = await new Promise((resolvePromise, reject) => {
- execFile(
- inputAgeProbe,
- [],
- {
- encoding: 'utf8',
- timeout: 1_000,
- },
- (error, stdout, stderr) => {
- if (error) reject(new Error(stderr || error.message));
- else resolvePromise(stdout.trim());
- },
- );
- });
- const ageSeconds = Number(output);
- if (!Number.isFinite(ageSeconds)) {
- throw new Error(`invalid physical input age: ${output}`);
- }
- return ageSeconds < 1;
-}
-
-const backend = createMakaCuBackend({
- binaryPath,
- expectedBinarySha256,
- timeoutMs: 10_000,
- physicalInputRecentlyActive,
- onTrace(event) {
- traces.push(event);
- },
-});
-
-const appSummaries = await backend.listApps(new AbortController().signal);
-const fixtureMatches = appSummaries.filter(
- (app) =>
- app.pid === fixturePID &&
- (app.name === 'Codex CUA Lab' ||
- app.windows?.some((window) => window.title === 'Codex CUA Lab')),
-);
-if (fixtureMatches.length !== 1) {
- throw new Error(`expected one exact fixture app, got ${fixtureMatches.length}`);
-}
-let fixture = fixtureMatches[0];
-const windows = fixture.windows?.filter((window) => window.title === 'Codex CUA Lab') ?? [];
-if (windows.length !== 1) {
- throw new Error(`expected one exact fixture window, got ${windows.length}`);
-}
-let fixtureWindowId = windows[0].windowId;
-registerFixtureInstance(fixture, fixtureWindowId);
-
-const instrumentedBackend = {
- ...backend,
- async observeApp(input, signal, context) {
- const observation = await backend.observeApp(input, signal, context);
- if (
- scenario === 'ambiguity' &&
- context.sessionId === 'real-ax-model-e2e' &&
- !ambiguityFixtureMutated
- ) {
- ambiguityFixtureMutated = true;
- const setupContext = {
- sessionId: 'real-ax-model-ambiguity-setup',
- turnId: 'setup',
- toolCallId: 'fixture-mutate-ambiguity',
- };
- const setupObservation = await backend.observeApp(
- {
- app: fixture.appId,
- windowId: fixtureWindowId,
- includeScreenshot: true,
- },
- signal,
- setupContext,
- );
- const mutation = setupObservation.elements.find(
- (element) => element.label === 'CUA Lab Duplicate Stale Target',
- );
- if (!mutation) throw new Error('ambiguity mutation control is missing');
- const setupResult = await backend.runSemantic(
- {
- type: 'click_element',
- observationId: setupObservation.observationId,
- elementId: mutation.elementId,
- elementIdentity: mutation.identity,
- },
- signal,
- setupContext,
- );
- if (!setupResult.outcome.ok) {
- throw new Error(`ambiguity fixture mutation failed: ${setupResult.outcome.message}`);
- }
- }
- if (scenario === 'restart-recovery' && !restartRequested) {
- restartRequested = true;
- await writeFile(
- join(resolvedTemporaryDirectory, 'restart-request.json'),
- `${JSON.stringify({ oldPID: observation.pid })}\n`,
- { flag: 'wx', mode: 0o600 },
- );
- const completePath = join(resolvedTemporaryDirectory, 'restart-complete.json');
- const deadline = Date.now() + 15_000;
- let completed;
- while (Date.now() < deadline) {
- try {
- completed = JSON.parse(await readFile(completePath, 'utf8'));
- break;
- } catch (error) {
- if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
- }
- await new Promise((resolvePromise) => setTimeout(resolvePromise, 50));
- }
- if (!completed) throw new Error('restart completion timeout');
- const apps = await backend.listApps(signal);
- const currentMatches = apps.filter(
- (app) =>
- app.pid === completed.newPID &&
- (app.name === 'Codex CUA Lab' ||
- app.windows?.some((window) => window.title === 'Codex CUA Lab')),
- );
- if (currentMatches.length !== 1) {
- throw new Error(`expected one restarted fixture, got ${currentMatches.length}`);
- }
- fixture = currentMatches[0];
- const currentWindows =
- fixture.windows?.filter((window) => window.title === 'Codex CUA Lab') ?? [];
- if (currentWindows.length !== 1) {
- throw new Error(`expected one restarted fixture window, got ${currentWindows.length}`);
- }
- fixtureWindowId = currentWindows[0].windowId;
- registerFixtureInstance(fixture, fixtureWindowId);
- }
- return observation;
- },
- async runSemantic(action, signal, context) {
- if (
- scenario === 'intervention-recovery' &&
- action.type === 'set_value' &&
- !injectedIntervention
- ) {
- injectedIntervention = true;
- return {
- outcome: {
- ok: false,
- error: 'user_intervened',
- message: 'synthetic intervention gate requires a fresh observation',
- },
- };
- }
- const result = await backend.runSemantic(action, signal, context);
- semanticOutcomes.push({
- actionType: action.type,
- ok: result.outcome.ok,
- ...(!result.outcome.ok
- ? {
- error: result.outcome.error,
- ambiguous: /ambiguous/i.test(result.outcome.message),
- }
- : {}),
- });
- if (
- scenario === 'restart-recovery' &&
- !result.outcome.ok &&
- result.outcome.error === 'target_missing'
- ) {
- staleRecoveryObserved = true;
- }
- if (
- scenario === 'ambiguity' &&
- action.type === 'click_element' &&
- !result.outcome.ok &&
- result.outcome.error === 'stale_frame' &&
- /ambiguous/.test(result.outcome.message)
- ) {
- ambiguousRecoveryObserved = true;
- }
- if (
- scenario === 'ambiguity' &&
- action.type === 'click_element' &&
- !result.outcome.ok &&
- result.outcome.error === 'target_changed'
- ) {
- semanticStructureChangeRejected = true;
- }
- if (action.type === 'set_value' && result.outcome.ok) {
- const field = result.observation?.elements.find(
- (element) => element.label === 'CUA Lab Set Value Field',
- );
- freshAxValue = result.outcome.evidence?.readbackValue ?? field?.value;
- }
- return result;
- },
-};
-const [rawComputerTool] = buildComputerUseTools({ backend: instrumentedBackend });
-const computerTool = {
- ...rawComputerTool,
- async impl(args, context) {
- const action = typeof args?.action === 'string' ? args.action : 'unknown';
- const allowed =
- scenario === 'observe-only'
- ? new Set(['list_apps', 'observe', 'wait'])
- : scenario === 'ax-click' || scenario === 'ambiguity'
- ? new Set(['list_apps', 'observe', 'click_element', 'wait'])
- : scenario === 'ax-multi-step'
- ? new Set(['list_apps', 'observe', 'set_value', 'click_element', 'wait'])
- : new Set(['list_apps', 'observe', 'set_value', 'wait']);
- const budget = actionBudgetFor(scenario);
- const nextTotal = totalActionAttempts + 1;
- const nextActionCount = (actionAttemptCounts.get(action) ?? 0) + 1;
- totalActionAttempts = nextTotal;
- actionAttemptCounts.set(action, nextActionCount);
- const sourceTarget = observationTargets.get(args?.observation_id);
- const ownedWindows = fixtureInstances.get(sourceTarget?.pid);
- const attempt = {
- action: { type: action },
- toolCallId: context.toolCallId,
- sourceObservationId: args?.observation_id,
- targetPid: sourceTarget?.pid,
- targetWindowId: sourceTarget?.windowId,
- targetOwned: Boolean(ownedWindows?.has(sourceTarget?.windowId)),
- success: false,
- durationMs: 0,
- };
- actions.push(attempt);
- const rejectAttempt = (resultCode, message) => {
- attempt.text = `maka_computer.${action} failed: ${resultCode}`;
- throw new Error(message);
- };
- if (!allowed.has(action)) {
- rejectAttempt(
- 'unsupported_action_policy',
- `model action '${action}' is outside the AX-only scenario budget`,
- );
- }
- if (nextTotal > budget.total || nextActionCount > (budget.counts[action] ?? 0)) {
- rejectAttempt('action_budget_exceeded', 'AX-only scenario action budget exceeded');
- }
- if (action === 'observe') {
- // The `computer` tool contract is "observe requires app OR window_id", so a
- // model that supplies only one of them is behaving correctly. Requiring both
- // would fail every compliant model before it ever reaches a dispatch. Assert
- // the real invariant instead: whatever the model did supply must identify the
- // fixture, and must not point at any other target.
- const appProvided = args.app !== undefined;
- const windowProvided = args.window_id !== undefined;
- const appMatches = args.app === fixture.appId;
- const windowMatches = args.window_id === fixtureWindowId;
- const identifiesFixture = (appProvided && appMatches) || (windowProvided && windowMatches);
- const contradictsFixture = (appProvided && !appMatches) || (windowProvided && !windowMatches);
- if (!identifiesFixture || contradictsFixture) {
- rejectAttempt(
- 'target_mismatch',
- 'model observe target does not match the exact fixture identity',
- );
- }
- }
- const actionStartedAt = Date.now();
- let result;
- try {
- result = await rawComputerTool.impl(args, context);
- } catch (error) {
- const resultCode = typeof error?.code === 'string' ? error.code : 'tool_error';
- attempt.durationMs = Date.now() - actionStartedAt;
- attempt.text = `maka_computer.${action} failed: ${resultCode}`;
- throw error;
- }
- const observation = parseObservation(result?.modelText) ?? parseObservation(result?.text);
- const dispatch = traces.findLast(
- (trace) => trace.type === 'dispatch' && trace.toolCallId === context.toolCallId,
- );
- const resultCode =
- result?.error ??
- (typeof result?.text === 'string'
- ? result.text.match(/\bfailed:\s*([a-z][a-z0-9_]{1,63})\b/i)?.[1]
- : undefined);
- if (observation) {
- observationTargets.set(observation.observation_id, {
- pid: observation.pid,
- windowId: observation.window_id,
- });
- }
- const targetPid = observation?.pid ?? dispatch?.pid ?? sourceTarget?.pid;
- const targetWindowId = observation?.window_id ?? dispatch?.windowId ?? sourceTarget?.windowId;
- const resultOwnedWindows = fixtureInstances.get(targetPid);
- Object.assign(attempt, {
- resultObservationId: observation?.observation_id,
- targetPid,
- targetWindowId,
- targetOwned: Boolean(resultOwnedWindows?.has(targetWindowId)),
- success: !resultCode,
- expectedFailure: qualificationScenario.expectedFailures.some(
- (expected) => expected.action === action && expected.error === resultCode,
- ),
- durationMs: Date.now() - actionStartedAt,
- text:
- result?.text ?? (resultCode ? `maka_computer.${action} failed: ${resultCode}` : undefined),
- });
- return result;
- },
-};
-
-const connection = {
- slug: provider === 'anthropic' ? 'coproxy' : 'azure-bridge',
- name: provider === 'anthropic' ? 'Anthropic Coproxy' : 'Azure Bridge',
- providerType: provider,
- baseUrl,
- defaultModel: modelId,
- enabled: true,
- createdAt: 1,
- updatedAt: 1,
-};
-const task =
- scenario === 'observe-only'
- ? 'Use Maka Computer to inspect "Codex CUA Lab". Start with list_apps, observe ' +
- 'the exact app/window, report that "CUA Lab Set Value Field" is visible, ' +
- 'do not mutate anything, then finish.'
- : scenario === 'ax-click'
- ? 'Use Maka Computer to click the element labeled "CUA Lab Primary Button" exactly ' +
- 'once in "Codex CUA Lab". Start with list_apps and observe, use click_element ' +
- 'with IDs from that observation, verify the fresh state, then finish. Never use coordinates.'
- : scenario === 'ax-multi-step'
- ? `Use Maka Computer in "Codex CUA Lab" to complete two semantic actions in order. ` +
- `First set "CUA Lab Set Value Field" to "${targetValue}" with set_value. ` +
- 'Then use the fresh observation returned by that action to click ' +
- '"CUA Lab Primary Button" exactly once with click_element. Verify both fresh ' +
- 'results, then finish. Never use coordinates, scroll, drag, type, or key actions.'
- : scenario === 'ambiguity'
- ? 'Use Maka Computer to click the observed element labeled "CUA Lab Stale Target" ' +
- 'exactly once in "Codex CUA Lab". Start with list_apps and observe, then use ' +
- 'click_element with the exact observation_id and element_id. If the tool rejects ' +
- 'the target, report the failure and stop rather than guessing or using coordinates.'
- : `Use Maka Computer to set "CUA Lab Set Value Field" in "Codex CUA Lab" ` +
- `to "${targetValue}". Start with list_apps, observe the exact app/window, ` +
- 'use set_value with the observation_id and element_id from that observation, ' +
- 'verify the fresh observation value, then finish. If user_intervened is returned, ' +
- 'observe again before retrying. If target_missing or stale_frame is returned, ' +
- 'list apps and observe the current process again before retrying. Never use ' +
- 'coordinate, click, scroll, drag, type, or key actions.';
-const turnId = 'turn-real-ax-model';
-const durableTurn = createDirectRuntimeTurnLedger({
- sessionId: 'real-ax-model-e2e',
- turnId,
- text: task,
- newId: () => `runtime-event-${++nextId}`,
- now: () => ++now,
-});
-const runtime = new AiSdkBackend({
- sessionId: 'real-ax-model-e2e',
- header: {
- id: 'real-ax-model-e2e',
- workspaceRoot: repoRoot,
- cwd: repoRoot,
- createdAt: now,
- lastUsedAt: now,
- name: 'Real AX Model E2E',
- isFlagged: false,
- labels: [],
- isArchived: false,
- status: 'active',
- statusUpdatedAt: now,
- hasUnread: false,
- backend: 'ai-sdk',
- llmConnectionSlug: connection.slug,
- connectionLocked: true,
- model: modelId,
- permissionMode: 'bypass',
- schemaVersion: 1,
- },
- appendMessage: async (message) => {
- messages.push(message);
- },
- connection,
- apiKey,
- modelId,
- readExecutionBoundary: async () => ({ kind: 'bypass', revision: 0 }),
- readPermissionMode: async () => 'bypass',
- modelFactory: (input) => getAIModel(input),
- tools: [computerTool],
- maxSteps: 8,
- loadTurnRuntimeEvents: durableTurn.loadTurnRuntimeEvents,
- newId: () => `id-${++nextId}`,
- now: () => ++now,
- recordToolInvocation: (record) => {
- telemetry.push({
- toolName: record.toolName,
- status: record.status,
- errorClass: record.errorClass,
- argsSummary: record.argsSummary,
- });
- },
-});
-
-const events = [];
-let terminalEvent;
-try {
- for await (const event of runtime.send({
- turnId,
- text: task,
- context: [],
- headAnchorRuntimeEvent: durableTurn.anchor,
- })) {
- durableTurn.record(event);
- events.push(event.type);
- if (event.type === 'complete' || event.type === 'abort' || event.type === 'error') {
- terminalEvent = {
- type: event.type,
- ...(typeof event.stopReason === 'string' ? { stopReason: event.stopReason } : {}),
- };
- }
- }
-
- const fixtureState = JSON.parse(await readFile(statePath, 'utf8'));
- const setValueDispatches = traces.filter(
- (trace) =>
- trace.type === 'dispatch' && trace.actionType === 'set_value' && trace.address === 'ax',
- );
- const clickDispatches = traces.filter(
- (trace) =>
- trace.type === 'dispatch' && trace.actionType === 'click_element' && trace.address === 'ax',
- );
- const modelClickDispatches = clickDispatches.filter(
- (trace) => trace.toolCallId !== 'fixture-mutate-ambiguity',
- );
- const setupClickDispatches = clickDispatches.filter(
- (trace) => trace.toolCallId === 'fixture-mutate-ambiguity',
- );
- const forbiddenDispatches = traces.filter(
- (trace) => trace.type === 'dispatch' && trace.address === 'px',
- );
- if (
- !['observe-only', 'ax-click', 'ambiguity'].includes(scenario) &&
- freshAxValue !== targetValue
- ) {
- throw new Error(`fresh AX value mismatch: ${String(freshAxValue)}`);
- }
- const expectedSetValueDispatches =
- scenario === 'observe-only' || scenario === 'ax-click' || scenario === 'ambiguity' ? 0 : 1;
- if (setValueDispatches.length !== expectedSetValueDispatches) {
- throw new Error(
- `expected ${expectedSetValueDispatches} AX set_value dispatches, got ${setValueDispatches.length}`,
- );
- }
- const expectedClickDispatches = scenario === 'ax-click' || scenario === 'ax-multi-step' ? 1 : 0;
- if (modelClickDispatches.length !== expectedClickDispatches) {
- throw new Error(
- `expected ${expectedClickDispatches} model AX click dispatches, got ${modelClickDispatches.length}`,
- );
- }
- if (scenario === 'ambiguity' && setupClickDispatches.length !== 1) {
- throw new Error(`expected one ambiguity setup AX dispatch, got ${setupClickDispatches.length}`);
- }
- if (forbiddenDispatches.length !== 0) {
- throw new Error(`forbidden pixel dispatch count: ${forbiddenDispatches.length}`);
- }
- if (scenario === 'restart-recovery' && !staleRecoveryObserved) {
- throw new Error('model restart scenario did not observe target_missing');
- }
- if (scenario === 'restart-recovery') {
- const staleIndex = actions.findIndex(
- (record) =>
- record.action.type === 'set_value' &&
- record.success === false &&
- record.expectedFailure === true,
- );
- const freshIndex = actions.findIndex(
- (record, index) =>
- index > staleIndex &&
- record.action.type === 'observe' &&
- record.success === true &&
- typeof record.resultObservationId === 'string' &&
- record.targetPid === fixture.pid,
- );
- const retry = actions.find(
- (record, index) =>
- index > freshIndex &&
- record.action.type === 'set_value' &&
- record.success === true &&
- record.sourceObservationId === actions[freshIndex]?.resultObservationId &&
- record.targetPid === fixture.pid,
- );
- const retryDispatch =
- retry &&
- setValueDispatches.some(
- (trace) =>
- trace.toolCallId === retry.toolCallId &&
- trace.pid === retry.targetPid &&
- trace.windowId === retry.targetWindowId,
- );
- if (staleIndex < 0 || freshIndex < 0 || !retry || !retryDispatch) {
- throw new Error(
- 'restart recovery requires stale target_missing, fresh observation, and successful AX retry',
- );
- }
- }
- if (
- scenario === 'ambiguity' &&
- (!actions.some((record) => record.action.type === 'click_element') ||
- (!ambiguousRecoveryObserved && !semanticStructureChangeRejected))
- ) {
- throw new Error(
- `model ambiguity scenario did not attempt and observe fail-closed rejection: ${JSON.stringify(
- {
- actionTypes: actions.map((record) => record.action.type),
- semanticOutcomes,
- setupClickDispatches: setupClickDispatches.length,
- modelClickDispatches: modelClickDispatches.length,
- },
- )}`,
- );
- }
- if (
- (scenario === 'ax-click' || scenario === 'ax-multi-step') &&
- fixtureState.controls.buttonClickCount !== 1
- ) {
- throw new Error(`primary button count mismatch: ${fixtureState.controls.buttonClickCount}`);
- }
- if (
- scenario === 'ambiguity' &&
- (fixtureState.hierarchy.mode !== 'ambiguous' ||
- fixtureState.hierarchy.staleTargetClickCount !== 0 ||
- fixtureState.hierarchy.wrongTargetClickCount !== 0)
- ) {
- throw new Error(`ambiguous stale target mutated: ${JSON.stringify(fixtureState.hierarchy)}`);
- }
- if (
- fixtureState.synthetic !== true ||
- fixtureState.appPath !== expectedAppPath ||
- fixtureState.oop?.hostPID !== fixture.pid
- ) {
- throw new Error('fixture identity changed during model execution');
- }
- if (terminalEvent?.type !== 'complete' || terminalEvent.stopReason !== 'end_turn') {
- throw new Error(`model Runtime did not complete/end_turn: ${JSON.stringify(terminalEvent)}`);
- }
- if (scenario === 'ax-multi-step') {
- const setIndex = actions.findIndex((record) => record.action.type === 'set_value');
- const clickIndex = actions.findIndex((record) => record.action.type === 'click_element');
- const setAction = actions[setIndex];
- const clickAction = actions[clickIndex];
- if (
- setIndex < 0 ||
- clickIndex <= setIndex ||
- typeof setAction?.resultObservationId !== 'string' ||
- clickAction?.sourceObservationId !== setAction.resultObservationId
- ) {
- throw new Error(
- `multi-step action order or observation lineage mismatch: ${JSON.stringify(
- actions.map((record) => ({
- type: record.action.type,
- sourceObservationId: record.sourceObservationId,
- resultObservationId: record.resultObservationId,
- })),
- )}`,
- );
- }
- }
-
- const evidenceClass = scenario === 'intervention-recovery' ? 'fault-injection' : 'real-runtime';
- const canonicalFixtureState = {
- target: {
- setValue: fixtureState.controls.setValue,
- buttonClickCount: fixtureState.controls.buttonClickCount,
- hierarchyMode: fixtureState.hierarchy.mode,
- staleTargetClickCount: fixtureState.hierarchy.staleTargetClickCount,
- wrongTargetClickCount: fixtureState.hierarchy.wrongTargetClickCount,
- },
- };
- const fixtureIdentityInstances = [...fixtureInstances.entries()].map(([pid, windowIds]) => ({
- pid,
- windowIds: [...windowIds],
- }));
- const minimumActionsPassed =
- scenario !== 'ambiguity' || actions.some((record) => record.action.type === 'click_element');
- const actionsWithinBudget =
- totalActionAttempts <= actionBudgetFor(scenario).total &&
- [...actionAttemptCounts].every(
- ([action, count]) => count <= (actionBudgetFor(scenario).counts[action] ?? 0),
- );
- const actionResultsPassed = actions.every(
- (record) => record.success === true || record.expectedFailure === true,
- );
- const dispatchPathPassed =
- forbiddenDispatches.length === 0 &&
- setValueDispatches.length === expectedSetValueDispatches &&
- modelClickDispatches.length === expectedClickDispatches;
- const qualified =
- minimumActionsPassed && actionsWithinBudget && actionResultsPassed && dispatchPathPassed;
- const sanitized = sanitizeCuDirectReport({
- schemaVersion: 1,
- ...reportLineage,
- evidenceClass,
- scenarioId:
- scenario === 'observe-only'
- ? 'appkit-ax-observe-only'
- : scenario === 'intervention-recovery'
- ? 'appkit-ax-intervention-recovery'
- : scenario === 'restart-recovery'
- ? 'appkit-ax-restart-recovery'
- : scenario === 'ax-click'
- ? 'appkit-ax-click'
- : scenario === 'ax-multi-step'
- ? 'appkit-ax-multi-step'
- : scenario === 'ambiguity'
- ? 'appkit-ax-ambiguity'
- : 'appkit-ax-set-value',
- policyMode: 'enforced',
- qualificationEligible: evidenceClass === 'real-runtime',
- producer: 'computer-use/real-ax',
- transportClass: 'live-network',
- model: modelId,
- provider,
- baseUrl,
- status: qualified ? 'pass' : 'fail',
- terminal: terminalEvent,
- fixtureIdentity: {
- instances: fixtureIdentityInstances,
- },
- ...(evidenceClass === 'fault-injection'
- ? {
- faultInjection: {
- layer: 'runtime',
- kind: 'user_intervened',
- },
- }
- : {}),
- totalLatencyMs: Date.now() - startedAt,
- loopStatus: 'completed',
- state: {
- fixtureIdentityValid: true,
- freshAxValueMatched:
- scenario === 'observe-only' || scenario === 'ax-click' || scenario === 'ambiguity'
- ? undefined
- : freshAxValue === targetValue,
- injectedIntervention,
- staleRecoveryObserved,
- ambiguousRecoveryObserved,
- semanticStructureChangeRejected,
- ambiguityFixtureMutated,
- setValueDispatches: setValueDispatches.length,
- clickDispatches: modelClickDispatches.length,
- setupClickDispatches: setupClickDispatches.length,
- pixelDispatches: forbiddenDispatches.length,
- primaryButtonClicks: fixtureState.controls.buttonClickCount,
- ambiguousClicks: fixtureState.hierarchy.staleTargetClickCount,
- terminalEvent,
- toolCallsPersisted: messages.filter((message) => message.type === 'tool_call').length,
- toolResultsPersisted: messages.filter((message) => message.type === 'tool_result').length,
- telemetrySuccesses: telemetry.filter((record) => record.status === 'success').length,
- telemetryErrors: telemetry.filter((record) => record.status === 'error').length,
- telemetryErrorClasses: [
- ...new Set(
- telemetry
- .filter((record) => record.status === 'error')
- .map((record) => record.errorClass)
- .filter(Boolean),
- ),
- ],
- telemetryErrorActions: telemetry
- .filter((record) => record.status === 'error')
- .flatMap((record) => {
- try {
- const summary = JSON.parse(record.argsSummary ?? '{}');
- return typeof summary.action === 'string' ? [summary.action] : [];
- } catch {
- return [];
- }
- }),
- semanticFailures: semanticOutcomes.filter((outcome) => !outcome.ok).length,
- semanticFailureCode: semanticOutcomes.find((outcome) => !outcome.ok)?.error,
- ambiguousSemanticFailures: semanticOutcomes.filter(
- (outcome) => !outcome.ok && outcome.ambiguous,
- ).length,
- },
- actionAttempts: totalActionAttempts,
- actionCount: actions.length,
- actionCounts: Object.fromEntries(
- actions.reduce((counts, record) => {
- const type = record.action.type;
- counts.set(type, (counts.get(type) ?? 0) + 1);
- return counts;
- }, new Map()),
- ),
- minimumActionsPassed,
- actionsWithinBudget,
- dispatchPathPassed,
- actions,
- fixtureState: canonicalFixtureState,
- expectedState: [
- {
- windowId: 'target',
- path: 'buttonClickCount',
- pass:
- !['ax-click', 'ax-multi-step'].includes(scenario) ||
- fixtureState.controls.buttonClickCount === 1,
- actual: fixtureState.controls.buttonClickCount,
- },
- ],
- forbiddenEffects: {
- status:
- forbiddenDispatches.length === 0 && fixtureState.hierarchy.wrongTargetClickCount === 0
- ? 'pass'
- : 'fail',
- violations: [],
- },
- driverTraces: traces.filter((trace) => trace.toolCallId !== 'fixture-mutate-ambiguity'),
- });
- await writeFile(reportPath, `${JSON.stringify(sanitized, null, 2)}\n`, {
- flag: 'wx',
- mode: 0o600,
- });
- process.stdout.write(`${JSON.stringify(sanitized, null, 2)}\n`);
- if (!qualified) {
- throw new Error('real AX model qualification rejected canonical action evidence');
- }
-} finally {
- await runtime.dispose();
- backend.dispose();
-}
diff --git a/scripts/computer-use/real-ax-launcher.mjs b/scripts/computer-use/real-ax-launcher.mjs
deleted file mode 100644
index cadb65131f..0000000000
--- a/scripts/computer-use/real-ax-launcher.mjs
+++ /dev/null
@@ -1,381 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { spawn } from 'node:child_process';
-import { copyFile, mkdir, mkdtemp, readFile, rm } from 'node:fs/promises';
-import { tmpdir } from 'node:os';
-import { dirname, join } from 'node:path';
-import { fileURLToPath } from 'node:url';
-
-import { requireComputerUseLabRoot } from './lab-root.mjs';
-
-const here = dirname(fileURLToPath(import.meta.url));
-const repoRoot = join(here, '../..');
-const harnessPath = join(here, 'real-ax-harness.mjs');
-const monitorPath = join(here, 'real-e2e-monitor.swift');
-const inputAgeSource = join(here, 'physical-input-age.swift');
-const labRoot = requireComputerUseLabRoot();
-const statePath = join(labRoot, 'test-app/runtime/state.json');
-const fixtureBundleId = 'com.openai.codex.cualab';
-const expectedAppPath = join(labRoot, 'test-app/build/Codex CUA Lab.app');
-const scenario = process.env.MAKA_CU_AX_MODEL_SCENARIO ?? 'set-value';
-const reportPath =
- process.env.MAKA_CU_AX_MODEL_REPORT ??
- join(repoRoot, '.agents-workspace-data', 'cu-real-ax-model', `report-${Date.now()}.json`);
-const driverOverride = process.env.MAKA_CU_AX_MODEL_DRIVER_OVERRIDE;
-const overrideSha256 = process.env.MAKA_CU_AX_MODEL_EXPECTED_SHA256;
-const overrideVersion = process.env.MAKA_CU_AX_MODEL_EXPECTED_VERSION;
-const overrideConfigured = [driverOverride, overrideSha256, overrideVersion].filter(Boolean).length;
-if (overrideConfigured !== 0 && overrideConfigured !== 3) {
- throw new Error(
- 'candidate driver qualification requires override path, expected SHA-256, and expected version',
- );
-}
-if (overrideSha256 && !/^[a-f0-9]{64}$/.test(overrideSha256)) {
- throw new Error('candidate driver expected SHA-256 is invalid');
-}
-const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
-
-function runChild(file, args, options = {}) {
- return new Promise((resolve, reject) => {
- const child = spawn(file, args, {
- cwd: options.cwd,
- env: options.env,
- stdio: options.stdio ?? ['ignore', 'inherit', 'inherit'],
- });
- child.once('error', reject);
- child.once('exit', (code, signal) => {
- if (code === 0) resolve();
- else reject(new Error(`${file} failed (${signal ?? code})`));
- });
- });
-}
-
-async function terminateChild(child, label, timeoutMs = 3_000) {
- if (!child || child.exitCode !== null || child.signalCode !== null) return;
- const exited = new Promise((resolve) => child.once('exit', resolve));
- child.kill('SIGTERM');
- if (await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)])) return;
- child.kill('SIGKILL');
- if (!(await Promise.race([exited.then(() => true), delay(timeoutMs).then(() => false)]))) {
- throw new Error(`${label} did not exit after SIGKILL`);
- }
-}
-
-async function runFixtureScript(name, options = {}) {
- await runChild(join(labRoot, 'test-app', name), [], options);
-}
-
-async function waitForFixture(timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- const state = JSON.parse(await readFile(statePath, 'utf8'));
- if (
- state.synthetic === true &&
- state.bundleIdentifier === fixtureBundleId &&
- state.appPath === expectedAppPath &&
- Number.isInteger(state.oop?.hostPID) &&
- state.oop.hostPID > 0
- ) {
- process.kill(state.oop.hostPID, 0);
- return state;
- }
- } catch (error) {
- if (error?.code !== 'ENOENT' && error?.code !== 'ESRCH' && !(error instanceof SyntaxError))
- throw error;
- }
- await delay(50);
- }
- throw new Error('synthetic fixture did not publish a live identity');
-}
-
-async function waitForRestartedFixture(oldPID, timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- const state = await waitForFixture(1_000);
- if (state.oop.hostPID !== oldPID) return state;
- } catch (error) {
- if (!/did not publish/.test(String(error))) throw error;
- }
- await delay(50);
- }
- throw new Error(`synthetic fixture PID did not advance from ${oldPID}`);
-}
-
-async function waitForJson(path, label, timeoutMs = 15_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- try {
- return JSON.parse(await readFile(path, 'utf8'));
- } catch (error) {
- if (error?.code !== 'ENOENT' && !(error instanceof SyntaxError)) throw error;
- }
- await delay(50);
- }
- throw new Error(`${label} timeout`);
-}
-
-function startMonitor(fixturePID) {
- const child = spawn(
- 'swift',
- [
- monitorPath,
- '--concurrent-user',
- String(fixturePID),
- // The fixture must stay in the background for the whole run. `--concurrent-user`
- // only catches the OOP host PID; denying the bundle covers every process the
- // fixture owns, so a UI process stealing focus mid-run also fails the scenario.
- '--deny-frontmost-bundle',
- fixtureBundleId,
- ],
- {
- stdio: ['ignore', 'pipe', 'pipe'],
- },
- );
- let buffer = '';
- let stderr = '';
- let readyResolve;
- let readyReject;
- let failureResolve;
- let readySettled = false;
- let failureSettled = false;
- const ready = new Promise((resolve, reject) => {
- readyResolve = resolve;
- readyReject = reject;
- });
- const failure = new Promise((resolve) => {
- failureResolve = resolve;
- });
- const fail = (error) => {
- const normalized = error instanceof Error ? error : new Error(String(error));
- if (!readySettled) {
- readySettled = true;
- readyReject(normalized);
- }
- if (!failureSettled) {
- failureSettled = true;
- failureResolve(normalized);
- }
- };
- child.stdout.setEncoding('utf8');
- child.stdout.on('data', (chunk) => {
- buffer += chunk;
- const lines = buffer.split('\n');
- buffer = lines.pop() ?? '';
- for (const line of lines.map((value) => value.trim()).filter(Boolean)) {
- const [kind, ...fields] = line.split('\t');
- if (kind === 'READY') {
- readySettled = true;
- readyResolve({
- mode: fields[0],
- frontmostPID: Number(fields[1]),
- pointer: { x: Number(fields[2]), y: Number(fields[3]) },
- physicalInputAge: Number(fields[4]),
- bundleIdentifier: fields[5],
- canonicalAppPath: fields[6],
- });
- } else if (kind === 'CHANGE' || kind === 'ERROR') {
- fail(new Error(fields.join('\t') || line));
- }
- }
- });
- child.stderr.setEncoding('utf8');
- child.stderr.on('data', (chunk) => {
- stderr += chunk;
- });
- child.on('error', fail);
- child.on('exit', (code, signal) => {
- if (!failureSettled && code !== 0 && signal !== 'SIGTERM') {
- fail(
- new Error(
- `monitor exited (${signal ?? code})` + `${stderr.trim() ? `: ${stderr.trim()}` : ''}`,
- ),
- );
- }
- });
- return {
- child,
- ready,
- failure,
- stop: () => terminateChild(child, 'AX model safety monitor'),
- };
-}
-
-function validateMonitorBaseline(baseline, fixturePID, label) {
- if (baseline.mode !== 'concurrent_user') {
- throw new Error(`${label} reported unexpected mode '${String(baseline.mode)}'`);
- }
- if (!Number.isFinite(baseline.physicalInputAge) || baseline.physicalInputAge < 0) {
- throw new Error(`${label} reported invalid physical input age`);
- }
- // The fixture is launched with `open -n -g` precisely so that it never owns the
- // foreground. Asserting that it stays in the background is what makes this an
- // honest test of background execution: requiring the fixture to be frontmost
- // would both contradict the launch mode and make the run depend on whatever
- // else happens to be on the desktop.
- if (baseline.bundleIdentifier === fixtureBundleId) {
- throw new Error(`${label} fixture unexpectedly owns the frontmost application`);
- }
- if (baseline.canonicalAppPath === expectedAppPath) {
- throw new Error(`${label} fixture app path unexpectedly owns the frontmost application`);
- }
- if (baseline.frontmostPID === fixturePID) {
- throw new Error(`${label} synthetic fixture became frontmost`);
- }
-}
-
-async function run() {
- const temporaryDirectory = await mkdtemp(join(tmpdir(), 'maka-cu-real-ax-model-'));
- const inputAgePath = join(temporaryDirectory, 'cu-physical-input-age');
- let fixtureTouched = false;
- let caffeinate;
- let monitor;
- let harness;
- try {
- await mkdir(dirname(reportPath), { recursive: true });
- caffeinate = spawn('/usr/bin/caffeinate', ['-dimsu'], {
- stdio: ['ignore', 'ignore', 'inherit'],
- });
- for (const workspace of [
- '@maka/core',
- '@maka/storage',
- '@maka/runtime',
- '@maka/computer-use',
- ]) {
- await runChild('npm', ['--workspace', workspace, 'run', 'build'], {
- cwd: repoRoot,
- });
- }
- await runChild(process.execPath, ['scripts/computer-use.mjs', 'prepare'], { cwd: repoRoot });
- if (driverOverride) {
- await copyFile(driverOverride, join(repoRoot, 'apps/desktop/resources/bin/maka-cu'));
- }
- await runChild('swiftc', [inputAgeSource, '-o', inputAgePath], {
- stdio: ['ignore', 'ignore', 'inherit'],
- });
- fixtureTouched = true;
- await runFixtureScript('stop.sh');
- await runFixtureScript('reset.sh');
- await runFixtureScript('launch.sh', {
- env: { ...process.env, CUA_LAB_BACKGROUND: '1' },
- });
- const fixture = await waitForFixture();
- monitor = startMonitor(fixture.oop.hostPID);
- const baseline = await Promise.race([
- monitor.ready,
- delay(10_000).then(() => {
- throw new Error('AX model safety monitor startup timeout');
- }),
- ]);
- validateMonitorBaseline(baseline, fixture.oop.hostPID, 'AX model safety monitor');
- harness = spawn(process.execPath, [harnessPath], {
- cwd: repoRoot,
- env: {
- ...process.env,
- MAKA_CU_AX_MODEL_FIXTURE_PID: String(fixture.oop.hostPID),
- MAKA_CU_AX_MODEL_INPUT_AGE_PROBE: inputAgePath,
- MAKA_CU_AX_MODEL_LAB_ROOT: labRoot,
- MAKA_CU_AX_MODEL_TEMP_DIR: temporaryDirectory,
- MAKA_CU_AX_MODEL_SCENARIO: scenario,
- MAKA_CU_AX_MODEL_REPORT: reportPath,
- ...(overrideSha256 ? { MAKA_CU_AX_MODEL_EXPECTED_SHA256: overrideSha256 } : {}),
- ...(overrideVersion ? { MAKA_CU_AX_MODEL_EXPECTED_VERSION: overrideVersion } : {}),
- },
- stdio: ['ignore', 'inherit', 'inherit'],
- });
- const exit = new Promise((resolve, reject) => {
- harness.once('error', reject);
- harness.once('exit', (code, signal) => resolve({ code, signal }));
- });
- if (scenario === 'restart-recovery') {
- const request = await Promise.race([
- waitForJson(join(temporaryDirectory, 'restart-request.json'), 'AX model restart request'),
- exit.then((result) => {
- throw new Error(
- `AX model harness exited before restart request (${result.signal ?? result.code})`,
- );
- }),
- monitor.failure.then((error) => {
- throw error;
- }),
- ]);
- if (request.oldPID !== fixture.oop.hostPID) {
- throw new Error('AX model restart request PID mismatch');
- }
- await monitor.stop();
- monitor = undefined;
- await runFixtureScript('stop.sh');
- await runFixtureScript('launch.sh', {
- env: { ...process.env, CUA_LAB_BACKGROUND: '1' },
- });
- const restarted = await waitForRestartedFixture(request.oldPID);
- monitor = startMonitor(restarted.oop.hostPID);
- const restartedBaseline = await Promise.race([
- monitor.ready,
- delay(10_000).then(() => {
- throw new Error('restarted AX model safety monitor timeout');
- }),
- ]);
- validateMonitorBaseline(
- restartedBaseline,
- restarted.oop.hostPID,
- 'restarted AX model safety monitor',
- );
- await runChild(
- process.execPath,
- [
- '-e',
- "require('fs').writeFileSync(process.argv[1], process.argv[2], {flag:'wx',mode:0o600})",
- join(temporaryDirectory, 'restart-complete.json'),
- JSON.stringify({
- oldPID: request.oldPID,
- newPID: restarted.oop.hostPID,
- }),
- ],
- { stdio: ['ignore', 'ignore', 'inherit'] },
- );
- }
- const first = await Promise.race([
- exit.then((result) => ({ type: 'exit', result })),
- monitor.failure.then((error) => ({ type: 'safety', error })),
- ]);
- if (first.type === 'safety') {
- await terminateChild(harness, 'AX model harness');
- throw first.error;
- }
- if (first.result.code !== 0) {
- throw new Error(`AX model E2E failed (${first.result.signal ?? first.result.code})`);
- }
- await readFile(reportPath, 'utf8');
- process.stdout.write(`Real AX model Computer Use report: ${reportPath}\n`);
- } finally {
- await terminateChild(harness, 'AX model harness').catch(() => {});
- await monitor?.stop().catch(() => {});
- if (fixtureTouched) await runFixtureScript('stop.sh').catch(() => {});
- await terminateChild(caffeinate, 'AX model caffeinate').catch(() => {});
- await rm(temporaryDirectory, { recursive: true, force: true });
- }
-}
-
-run().catch((error) => {
- console.error('Computer Use real AX model E2E failed:', error);
- process.exitCode = 1;
-});
diff --git a/scripts/computer-use/real-e2e-monitor.swift b/scripts/computer-use/real-e2e-monitor.swift
deleted file mode 100644
index 3d78fbfba8..0000000000
--- a/scripts/computer-use/real-e2e-monitor.swift
+++ /dev/null
@@ -1,158 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import Cocoa
-import CoreGraphics
-import Darwin
-
-setbuf(stdout, nil)
-
-let concurrentUserMode = CommandLine.arguments.contains("--concurrent-user")
-let snapshotMode = CommandLine.arguments.contains("--snapshot")
-let deniedFrontmostBundle: String? = {
- guard let flagIndex = CommandLine.arguments.firstIndex(
- of: "--deny-frontmost-bundle"
- ),
- CommandLine.arguments.indices.contains(flagIndex + 1)
- else {
- return nil
- }
- return CommandLine.arguments[flagIndex + 1]
-}()
-let mode = concurrentUserMode ? "concurrent_user" : "isolated"
-let fixturePID: pid_t? = {
- guard concurrentUserMode,
- let flagIndex = CommandLine.arguments.firstIndex(of: "--concurrent-user"),
- CommandLine.arguments.indices.contains(flagIndex + 1),
- let value = Int32(CommandLine.arguments[flagIndex + 1])
- else {
- return nil
- }
- return value
-}()
-if concurrentUserMode && fixturePID == nil {
- print("ERROR\tconcurrent mode requires the fixture PID")
- exit(1)
-}
-
-func screenIsLocked() -> Bool {
- guard let session = CGSessionCopyCurrentDictionary() as? [String: Any] else {
- return true
- }
- return session["CGSSessionScreenIsLocked"] as? Bool ?? false
-}
-
-guard !screenIsLocked() else {
- print("ERROR\tscreen is locked")
- exit(1)
-}
-
-guard let initialApplication = NSWorkspace.shared.frontmostApplication else {
- print("ERROR\tno frontmost application")
- exit(1)
-}
-
-let frontmostPID = initialApplication.processIdentifier
-guard let bundleIdentifier = initialApplication.bundleIdentifier,
- let bundlePath = initialApplication.bundleURL?.resolvingSymlinksInPath().path
-else {
- print("ERROR\tfrontmost application identity is incomplete")
- exit(1)
-}
-let initialPointer = NSEvent.mouseLocation
-let startedAt = ProcessInfo.processInfo.systemUptime
-let physicalEventTypes: [CGEventType] = [
- .keyDown,
- .leftMouseDown,
- .rightMouseDown,
- .otherMouseDown,
- .mouseMoved,
- .leftMouseDragged,
- .rightMouseDragged,
- .otherMouseDragged,
- .scrollWheel
-]
-let initialPhysicalInputAge = physicalEventTypes.map { eventType in
- CGEventSource.secondsSinceLastEventType(
- .hidSystemState,
- eventType: eventType
- )
-}.min() ?? .infinity
-print(
- "READY\t\(mode)\t\(frontmostPID)\t\(initialPointer.x)\t\(initialPointer.y)"
- + "\t\(initialPhysicalInputAge)\t\(bundleIdentifier)\t\(bundlePath)"
-)
-if snapshotMode {
- exit(0)
-}
-
-while true {
- autoreleasepool {
- if screenIsLocked() {
- print("CHANGE\tscreen became locked")
- exit(2)
- }
- let currentPID =
- NSWorkspace.shared.frontmostApplication?.processIdentifier ?? -1
- let currentBundle =
- NSWorkspace.shared.frontmostApplication?.bundleIdentifier
- let currentPointer = NSEvent.mouseLocation
- if let deniedFrontmostBundle, currentBundle == deniedFrontmostBundle {
- print(
- "CHANGE\tdenied bundle became frontmost during E2E: "
- + deniedFrontmostBundle
- )
- exit(3)
- }
- if concurrentUserMode && currentPID == fixturePID {
- print("CHANGE\tsynthetic fixture became frontmost during concurrent E2E")
- exit(3)
- }
- if !concurrentUserMode && currentPID != frontmostPID {
- print("CHANGE\tfrontmost PID changed: \(frontmostPID) -> \(currentPID)")
- exit(3)
- }
- let displacement = hypot(
- currentPointer.x - initialPointer.x,
- currentPointer.y - initialPointer.y
- )
- if !concurrentUserMode && displacement > 4 {
- print(
- "CHANGE\tpointer moved during isolated E2E: "
- + "(\(initialPointer.x),\(initialPointer.y)) -> "
- + "(\(currentPointer.x),\(currentPointer.y)); "
- + "displacement \(displacement)"
- )
- exit(4)
- }
- let elapsed = ProcessInfo.processInfo.systemUptime - startedAt
- let receivedPhysicalInput = physicalEventTypes.contains { eventType in
- let age = CGEventSource.secondsSinceLastEventType(
- .hidSystemState,
- eventType: eventType
- )
- return age + 0.02 < elapsed
- }
- if !concurrentUserMode && receivedPhysicalInput {
- print("CHANGE\tphysical user input detected during isolated E2E")
- exit(5)
- }
- }
- usleep(5_000)
-}
diff --git a/scripts/computer-use/real-model-fixture.mjs b/scripts/computer-use/real-model-fixture.mjs
deleted file mode 100644
index 9d77f99b6f..0000000000
--- a/scripts/computer-use/real-model-fixture.mjs
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { app, BrowserWindow, screen } from 'electron';
-import { getCuE2eScenario } from './e2e-scenarios.mjs';
-import { createCuE2eFixture } from './e2e-fixture.mjs';
-
-const scenario = getCuE2eScenario(process.env.MAKA_CU_E2E_SCENARIO ?? 'l0-observe-only');
-
-app.setActivationPolicy('accessory');
-app.on('window-all-closed', () => {});
-
-let fixture;
-
-app.whenReady().then(async () => {
- fixture = await createCuE2eFixture({ BrowserWindow, screen, scenario });
- for (let index = 0; index < 4; index += 1) {
- for (const windowId of fixture.windowIds()) {
- const window = fixture.getWindow(windowId);
- window.showInactive();
- window.moveTop();
- }
- await new Promise((resolve) => setTimeout(resolve, 250));
- }
- process.stdout.write(`CU_FIXTURE_READY ${process.pid}\n`);
-});
-
-async function shutdown() {
- fixture?.destroy();
- app.exit(0);
-}
-
-process.on('SIGTERM', () => void shutdown());
-process.on('SIGINT', () => void shutdown());
diff --git a/scripts/computer-use/real-model.mjs b/scripts/computer-use/real-model.mjs
deleted file mode 100644
index 0d9828de93..0000000000
--- a/scripts/computer-use/real-model.mjs
+++ /dev/null
@@ -1,884 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-import { _electron as electron, chromium } from '@playwright/test';
-import { execFile, spawn } from 'node:child_process';
-import { randomUUID } from 'node:crypto';
-import { promisify } from 'node:util';
-import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';
-import { createServer } from 'node:net';
-import { homedir, tmpdir } from 'node:os';
-import { dirname, join, resolve } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import { evaluateCuE2eScenarioState, getCuE2eScenario } from './e2e-scenarios.mjs';
-import { validateRealReport } from './provider-matrix.mjs';
-import { sanitizeCuActionRecord, sanitizeCuReport, sanitizeCuTrace } from './report-sanitize.mjs';
-import { createSqliteAgentRunStore } from '../../packages/storage/dist/agent-run-store.js';
-import {
- resolveStorageRoot,
- tryAcquireInteractiveRootOwner,
-} from '../../packages/storage/dist/root-authority.js';
-import { openInteractiveRuntimePolicyStoresForWrite } from '../../packages/storage/dist/runtime-policy-stores.js';
-
-const here = dirname(fileURLToPath(import.meta.url));
-const repoRoot = join(here, '../..');
-const execFileAsync = promisify(execFile);
-const sourceWorkspace = join(
- homedir(),
- 'Library',
- 'Application Support',
- 'Maka',
- 'workspaces',
- 'default',
-);
-const scenario = getCuE2eScenario(process.env.MAKA_CU_E2E_SCENARIO ?? 'l0-observe-only');
-if (!scenario.realRunEnabled) {
- throw new Error(`scenario ${scenario.id} is not enabled for real-model runs`);
-}
-if (scenario.runner) {
- throw new Error(`scenario ${scenario.id} requires dedicated runner ${scenario.runner}`);
-}
-const availableCapabilities = new Set(
- String(process.env.MAKA_CU_EXECUTION_CAPABILITIES ?? '')
- .split(',')
- .map((value) => value.trim())
- .filter(Boolean),
-);
-for (const capability of scenario.requiresExecutionCapabilities) {
- if (!availableCapabilities.has(capability)) {
- throw new Error(`scenario ${scenario.id} requires unavailable capability ${capability}`);
- }
-}
-
-const timeoutMs = Number(process.env.MAKA_CU_REAL_MODEL_TIMEOUT_MS ?? 180_000);
-const keepProfile = process.env.MAKA_CU_KEEP_PROFILE === '1';
-const providerOverride = process.env.MAKA_CU_PROVIDER;
-const reportPath =
- process.env.MAKA_CU_REAL_MODEL_REPORT ??
- join(repoRoot, '.agents-workspace-data', 'cu-real-model', `report-${Date.now()}.json`);
-const runPrompt = 'Use the maka_computer tool to complete this task. ' + scenario.prompt;
-
-async function reportLineage() {
- const { stdout } = await execFileAsync('git', ['rev-parse', 'HEAD'], {
- cwd: repoRoot,
- });
- const generatedAt = new Date().toISOString();
- const gitRevision = stdout.trim();
- return {
- runId: randomUUID(),
- gitRevision,
- generatedAt,
- contentLineage: {
- generator: 'scripts/computer-use/real-model.mjs',
- gitRevision,
- generatedAt,
- },
- };
-}
-
-async function reservePort() {
- const server = createServer();
- await new Promise((resolve, reject) => {
- server.once('error', reject);
- server.listen(0, '127.0.0.1', resolve);
- });
- const address = server.address();
- const port = typeof address === 'object' && address ? address.port : 0;
- await new Promise((resolve) => server.close(resolve));
- if (!port) throw new Error('failed to reserve loopback port');
- return port;
-}
-
-async function copyProfileFile(name, workspace) {
- await cp(join(sourceWorkspace, name), join(workspace, name), {
- errorOnExist: true,
- });
-}
-
-async function copyProfileFileIfPresent(name, workspace) {
- try {
- await copyProfileFile(name, workspace);
- } catch (error) {
- if (error && error.code === 'ENOENT') return;
- throw error;
- }
-}
-
-async function prepareProviderProfile(workspace) {
- if (!providerOverride) {
- await copyProfileFile('connection-catalog.json', workspace);
- await Promise.all([
- copyProfileFileIfPresent('credential-vault.json', workspace),
- copyProfileFileIfPresent('credentials.json', workspace),
- copyProfileFile('settings.json', workspace),
- ]);
- return;
- }
- if (providerOverride !== 'openai') {
- throw new Error(`unsupported MAKA_CU_PROVIDER ${providerOverride}`);
- }
- await copyProfileFile('settings.json', workspace);
- const modelId = process.env.MAKA_CU_OPENAI_MODEL ?? 'gpt-5.4';
- const capability = await resolveStorageRoot({ path: workspace, kind: 'interactive' });
- const owner = await tryAcquireInteractiveRootOwner(capability);
- if (!owner) throw new Error('Unable to acquire the Computer Use catalog root');
- try {
- const stores = await openInteractiveRuntimePolicyStoresForWrite(owner.lease);
- const created = await stores.connectionCatalog.create({
- expectedCatalogRevision: 0,
- connection: {
- slug: 'cu-real-openai',
- name: 'Computer Use real-model OpenAI',
- providerType: 'openai',
- baseUrl: process.env.MAKA_CU_OPENAI_BASE_URL ?? 'http://127.0.0.1:8538/v1',
- enabled: true,
- enabledModelIds: [modelId],
- },
- });
- if (created.kind !== 'committed') {
- throw new Error(`Computer Use catalog create failed: ${created.kind}`);
- }
- const connection = created.snapshot.connections.find((item) => item.slug === 'cu-real-openai');
- if (!connection) throw new Error('Computer Use catalog create omitted cu-real-openai');
- const vault = await stores.credentialVault.set({
- locator: {
- scope: 'connection',
- connectionId: connection.connectionId,
- kind: 'api_key',
- },
- expected: null,
- secret: process.env.MAKA_CU_OPENAI_API_KEY ?? 'local-bridge',
- });
- if (vault.kind !== 'committed') {
- throw new Error(`Computer Use vault set failed: ${vault.kind}`);
- }
- const defaulted = await stores.connectionCatalog.setDefaultTarget({
- expectedCatalogRevision: created.snapshot.revision,
- target: { connectionId: connection.connectionId, modelId },
- });
- if (defaulted.kind !== 'committed') {
- throw new Error(`Computer Use default target failed: ${defaulted.kind}`);
- }
- } finally {
- await owner.close();
- }
-}
-
-async function waitForLine(child, marker, timeout) {
- return new Promise((resolve, reject) => {
- let stdout = '';
- const timer = setTimeout(() => reject(new Error(`timed out waiting for ${marker}`)), timeout);
- const onData = (chunk) => {
- stdout += String(chunk);
- process.stdout.write(chunk);
- const matched =
- marker === 'CU_FIXTURE_READY'
- ? /^CU_FIXTURE_READY\s+\d+\s*$/m.test(stdout)
- : stdout.includes(marker);
- if (matched) {
- clearTimeout(timer);
- child.stdout.off('data', onData);
- resolve(stdout);
- }
- };
- child.stdout.on('data', onData);
- child.stderr.on('data', (chunk) => process.stderr.write(chunk));
- child.once('error', reject);
- child.once('exit', (code, signal) => {
- clearTimeout(timer);
- reject(new Error(`fixture exited before ready: ${signal ?? `code ${code}`}`));
- });
- });
-}
-
-export function parseFixtureReady(text, childPid) {
- const matches =
- typeof text === 'string' ? [...text.matchAll(/^CU_FIXTURE_READY\s+(\d+)\s*$/gm)] : [];
- if (matches.length !== 1) {
- throw new Error('fixture READY identity is missing or ambiguous');
- }
- const readyPid = Number(matches[0][1]);
- if (!Number.isInteger(childPid) || childPid <= 0 || readyPid !== childPid) {
- throw new Error(`fixture READY pid ${readyPid} does not match launcher child pid ${childPid}`);
- }
- return readyPid;
-}
-
-export function parseTargetEvidence(text) {
- if (typeof text !== 'string') return undefined;
- const candidates = [text];
- for (const markerText of ['Fresh observation:\n', 'Fresh observation: ']) {
- const marker = text.lastIndexOf(markerText);
- if (marker >= 0) candidates.push(text.slice(marker + markerText.length));
- }
- for (const candidate of candidates.reverse()) {
- try {
- const value = JSON.parse(candidate);
- if (
- value &&
- typeof value === 'object' &&
- Number.isInteger(value.pid) &&
- Number.isInteger(value.window_id)
- ) {
- return {
- ...(typeof value.observation_id === 'string'
- ? { observationId: value.observation_id }
- : {}),
- pid: value.pid,
- windowId: value.window_id,
- };
- }
- } catch {
- // Continue to the next privacy-safe projection candidate.
- }
- }
- return undefined;
-}
-
-export function actionRecords(events) {
- const starts = new Map();
- const records = [];
- for (const event of events) {
- if (event.type === 'tool_start' && event.toolName === 'maka_computer') {
- starts.set(event.toolUseId, event);
- }
- if (event.type === 'tool_result' && starts.has(event.toolUseId)) {
- const start = starts.get(event.toolUseId);
- const text = event.content?.kind === 'text' ? event.content.text : undefined;
- const resultCode =
- typeof text === 'string'
- ? text.match(/\bfailed:\s*([a-z][a-z0-9_]{1,63})\b/i)?.[1]
- : undefined;
- const target = parseTargetEvidence(text);
- records.push(
- sanitizeCuActionRecord({
- action: start.args,
- toolCallId: event.toolUseId,
- sourceObservationId: start.args?.observation_id,
- resultObservationId: target?.observationId,
- targetPid: target?.pid,
- targetWindowId: target?.windowId,
- durationMs: event.durationMs,
- text,
- success: event.isError === false && !resultCode,
- }),
- );
- }
- }
- return records;
-}
-
-export async function waitForTraceFlush(path, expectedToolCallIds, timeoutMs = 2_000) {
- const deadline = Date.now() + timeoutMs;
- while (Date.now() < deadline) {
- const current = await readFile(path, 'utf8').catch((error) => {
- if (error?.code === 'ENOENT') return '';
- throw error;
- });
- const traces = [];
- let incompleteTrace = false;
- for (const line of current.split('\n').filter(Boolean)) {
- try {
- traces.push(JSON.parse(line));
- } catch (error) {
- if (!(error instanceof SyntaxError)) throw error;
- incompleteTrace = true;
- }
- }
- const observedToolCallIds = new Set(
- traces.filter((trace) => trace.type === 'dispatch').map((trace) => trace.toolCallId),
- );
- if (
- !incompleteTrace &&
- expectedToolCallIds.every((toolCallId) => observedToolCallIds.has(toolCallId))
- ) {
- return traces;
- }
- await new Promise((resolve) => setTimeout(resolve, 50));
- }
- throw new Error(
- `timed out waiting for Computer Use dispatch traces: ${expectedToolCallIds.join(',')}`,
- );
-}
-
-export async function discoverFixtureIdentity(
- fixturePid,
- windowSpecs,
- { listApps, timeoutMs = 2_000, pollIntervalMs = 50 } = {},
-) {
- if (!Number.isInteger(fixturePid) || fixturePid <= 0) {
- throw new Error('fixture discovery requires a valid launcher-owned pid');
- }
- if (typeof listApps !== 'function') {
- throw new Error('fixture discovery requires an independent window lister');
- }
- const expectedTitles = windowSpecs?.map((window) => window.title);
- if (
- !Array.isArray(expectedTitles) ||
- expectedTitles.length === 0 ||
- expectedTitles.some((title) => typeof title !== 'string' || title.length === 0) ||
- new Set(expectedTitles).size !== expectedTitles.length
- ) {
- throw new Error('fixture discovery requires unique expected window titles');
- }
-
- const deadline = Date.now() + timeoutMs;
- let lastFailure = 'fixture app was not found';
- while (Date.now() < deadline) {
- const apps = await listApps();
- const fixtureApps = Array.isArray(apps)
- ? apps.filter((app) => Number(app?.pid) === fixturePid)
- : [];
- if (fixtureApps.length === 1) {
- const windows = Array.isArray(fixtureApps[0].windows) ? fixtureApps[0].windows : [];
- const windowIds = [];
- let complete = true;
- for (const title of expectedTitles) {
- const matches = windows.filter(
- (window) =>
- window?.title === title && Number.isInteger(window?.windowId) && window.windowId > 0,
- );
- if (matches.length !== 1) {
- complete = false;
- lastFailure = `expected one fixture window "${title}", got ${matches.length}`;
- break;
- }
- windowIds.push(matches[0].windowId);
- }
- if (complete && new Set(windowIds).size === windowIds.length) {
- return {
- instances: [
- {
- pid: fixturePid,
- windowIds,
- },
- ],
- };
- }
- if (complete) lastFailure = 'fixture discovery returned duplicate window ids';
- } else {
- lastFailure = `expected one fixture app for pid ${fixturePid}, got ${fixtureApps.length}`;
- }
- await new Promise((resolvePromise) => setTimeout(resolvePromise, pollIntervalMs));
- }
- throw new Error(`fixture identity discovery failed: ${lastFailure}`);
-}
-
-async function discoverLauncherFixtureIdentity(fixturePid, windowSpecs) {
- const [{ createMakaCuBackend }, manifestText] = await Promise.all([
- import('../../packages/computer-use/dist/index.js'),
- readFile(join(repoRoot, 'apps', 'desktop', 'bundled-tools.json'), 'utf8'),
- ]);
- const manifest = JSON.parse(manifestText);
- const expectedBinarySha256 = manifest?.makaCu?.binarySha256;
- if (typeof expectedBinarySha256 !== 'string' || !/^[a-f0-9]{64}$/.test(expectedBinarySha256)) {
- throw new Error('fixture discovery cannot verify bundled maka-cu identity');
- }
- const backend = createMakaCuBackend({
- binaryPath: join(repoRoot, 'apps', 'desktop', 'resources', 'bin', 'maka-cu'),
- expectedBinarySha256,
- timeoutMs: 10_000,
- });
- try {
- return await discoverFixtureIdentity(fixturePid, windowSpecs, {
- listApps: () => backend.listApps(new AbortController().signal),
- timeoutMs: 5_000,
- });
- } finally {
- backend.dispose();
- }
-}
-
-function safeEvent(event) {
- if (event.type === 'tool_start') {
- const safeToolName =
- event.toolName === 'tool_search' || event.toolName === 'maka_computer'
- ? event.toolName
- : 'other';
- return {
- type: event.type,
- toolName: safeToolName,
- ...(event.toolName === 'maka_computer'
- ? { actionType: event.args?.action ?? 'unknown' }
- : {}),
- ts: event.ts,
- };
- }
- if (event.type === 'tool_result') {
- return {
- type: event.type,
- isError: event.isError,
- durationMs: event.durationMs,
- ts: event.ts,
- };
- }
- if (event.type === 'complete' || event.type === 'abort' || event.type === 'error') {
- const safeCode = [event.code, event.reason].find(
- (value) => typeof value === 'string' && /^[a-zA-Z][a-zA-Z0-9_.-]{0,63}$/.test(value),
- );
- return {
- type: event.type,
- ...(event.stopReason ? { stopReason: event.stopReason } : {}),
- ...(safeCode ? { code: safeCode } : {}),
- ts: event.ts,
- };
- }
- return null;
-}
-
-function safeFailureMetadata(message) {
- if (typeof message !== 'string') return undefined;
- const status = message.match(/\b([45]\d\d)\b/)?.[1];
- const errorName = message.match(/\b([A-Z][A-Za-z]+Error)\b/)?.[1];
- const providerType = message
- .match(
- /\b(api_error|authentication_error|billing_error|invalid_request_error|overloaded_error|permission_error|rate_limit_error)\b/i,
- )?.[1]
- ?.toLowerCase();
- const result = {
- ...(status ? { httpStatus: Number(status) } : {}),
- ...(errorName ? { errorName } : {}),
- ...(providerType ? { providerErrorType: providerType } : {}),
- };
- return Object.keys(result).length > 0 ? result : undefined;
-}
-
-async function run() {
- const lineage = await reportLineage();
- const userData = await mkdtemp(join(tmpdir(), 'maka-cu-real-model-'));
- const workspace = join(userData, 'workspaces', 'default');
- const tracePath = join(userData, 'computer-use-trace.jsonl');
- const fixturePort = await reservePort();
- const electronBinary = join(repoRoot, 'node_modules', '.bin', 'electron');
- let fixture;
- let desktop;
- let fixtureBrowser;
- try {
- await mkdir(workspace, { recursive: true });
- await mkdir(dirname(reportPath), { recursive: true });
- await prepareProviderProfile(workspace);
-
- fixture = spawn(
- electronBinary,
- [
- `--remote-debugging-port=${fixturePort}`,
- '--remote-allow-origins=*',
- join(here, 'real-model-fixture.mjs'),
- ],
- {
- cwd: repoRoot,
- env: {
- ...process.env,
- MAKA_CU_E2E_SCENARIO: scenario.id,
- },
- stdio: ['ignore', 'pipe', 'pipe'],
- },
- );
- const readyOutput = await waitForLine(fixture, 'CU_FIXTURE_READY', 30_000);
- const fixturePid = parseFixtureReady(readyOutput, fixture.pid);
- const fixtureIdentity = await discoverLauncherFixtureIdentity(
- fixturePid,
- activeWindowSpecs(scenario),
- );
-
- desktop = await electron.launch({
- args: ['apps/desktop'],
- cwd: repoRoot,
- env: {
- ...process.env,
- MAKA_CU_REAL_MODEL_E2E: '1',
- MAKA_E2E_USER_DATA_DIR: userData,
- MAKA_CU_REAL_MODEL_POLICY: JSON.stringify({
- allowedActions: scenario.allowedActions,
- maxTotalActions: scenario.maxTotalActions,
- maxActionCounts:
- scenario.maxActionCounts ??
- Object.fromEntries(
- scenario.allowedActions.map((action) => [action, scenario.maxTotalActions]),
- ),
- allowedApps: activeWindowSpecs(scenario).map((window) => window.title),
- }),
- MAKA_CU_REAL_MODEL_TRACE: tracePath,
- },
- timeout: 30_000,
- });
- const page = await desktop.firstWindow();
- await page.waitForFunction(() => Boolean(window.maka?.sessions));
-
- const runResult = await page.evaluate(
- async ({ prompt, timeout }) => {
- const connections = await window.maka.connections.list();
- const defaultSlug = await window.maka.connections.getDefault();
- const connection = connections.find((entry) => entry.slug === defaultSlug);
- if (!connection?.defaultModel) {
- throw new Error('isolated profile has no ready default model');
- }
- const session = await window.maka.sessions.create({
- backend: 'ai-sdk',
- llmConnectionSlug: connection.slug,
- model: connection.defaultModel,
- permissionMode: 'bypass',
- name: 'Computer Use real-model E2E',
- labels: ['computer-use', 'real-model-e2e'],
- });
- const events = [];
- const terminal = new Promise((resolve, reject) => {
- const timer = setTimeout(() => reject(new Error('real-model turn timed out')), timeout);
- const unsubscribe = window.maka.sessions.subscribeEvents(session.id, (event) => {
- events.push(event);
- if (event.type === 'complete' || event.type === 'abort' || event.type === 'error') {
- clearTimeout(timer);
- unsubscribe();
- resolve(event);
- }
- });
- });
- const turnId = crypto.randomUUID();
- await window.maka.sessions.send(session.id, {
- type: 'send',
- turnId,
- text: prompt,
- });
- const terminalEvent = await terminal;
- return {
- connection: {
- slug: connection.slug,
- providerType: connection.providerType,
- model: connection.defaultModel,
- },
- sessionId: session.id,
- turnId,
- events,
- terminalEvent,
- };
- },
- { prompt: runPrompt, timeout: timeoutMs },
- );
-
- fixtureBrowser = await chromium.connectOverCDP(`http://127.0.0.1:${fixturePort}`);
- const fixturePages = fixtureBrowser.contexts().flatMap((context) => context.pages());
- const fixtureState = {};
- for (const windowSpec of activeWindowSpecs(scenario)) {
- const pageForTitle = await Promise.all(
- fixturePages.map(async (candidate) => ({
- candidate,
- title: await candidate.title(),
- })),
- ).then((entries) => entries.find((entry) => entry.title === windowSpec.title)?.candidate);
- if (!pageForTitle) throw new Error(`missing fixture page ${windowSpec.title}`);
- fixtureState[windowSpec.id] = await pageForTitle.evaluate(
- () => globalThis.__makaCuFixtureState?.() ?? null,
- );
- }
- const evaluation = evaluateCuE2eScenarioState(scenario, fixtureState);
- const events = runResult.events.map(safeEvent).filter(Boolean);
- const rawActions = actionRecords(runResult.events);
- const expectedDispatchToolCallIds = rawActions
- .filter(
- (action) =>
- action.success === true &&
- !['list_apps', 'launch_app', 'observe', 'screenshot', 'wait'].includes(action.type),
- )
- .map((action) => action.toolCallId)
- .filter((toolCallId) => typeof toolCallId === 'string');
- const driverTraces = await waitForTraceFlush(tracePath, expectedDispatchToolCallIds);
- const actions = bindActionTargets(rawActions, driverTraces, fixtureIdentity);
- const runStore = createSqliteAgentRunStore(workspace);
- await runStore.ready?.();
- let runHeader;
- try {
- runHeader = await waitForRunHeader(runStore, runResult.sessionId, runResult.turnId);
- } finally {
- runStore.close?.();
- }
- const qualifyingActions = actions.filter(
- (action) =>
- action.success === true &&
- action.targetOwned === true &&
- scenario.allowedActions.includes(action.type),
- );
- const actionCounts = Object.fromEntries(
- actions.reduce((counts, action) => {
- counts.set(action.type, (counts.get(action.type) ?? 0) + 1);
- return counts;
- }, new Map()),
- );
- const qualifyingActionCounts = Object.fromEntries(
- scenario.allowedActions.map((action) => [
- action,
- qualifyingActions.filter((record) => record.type === action).length,
- ]),
- );
- const minimumActionsPassed = Object.entries(scenario.minimumActionCounts ?? {}).every(
- ([action, minimum]) => (qualifyingActionCounts[action] ?? 0) >= minimum,
- );
- const terminalPassed =
- runResult.terminalEvent.type === 'complete' &&
- runResult.terminalEvent.stopReason === 'end_turn';
- const actionsWithinBudget =
- actions.length <= scenario.maxTotalActions &&
- actions.every((action) => scenario.allowedActions.includes(action.type)) &&
- Object.entries(scenario.maxActionCounts ?? {}).every(
- ([action, maximum]) => actions.filter((record) => record.type === action).length <= maximum,
- );
- const dispatchPathPassed = requiredDispatchPathPassed(scenario, driverTraces);
- const ownershipPassed = allActionTargetsOwned(actions);
- const localChecksPassed =
- terminalPassed &&
- minimumActionsPassed &&
- actionsWithinBudget &&
- dispatchPathPassed &&
- ownershipPassed &&
- evaluation.pass;
- const reportInput = {
- schemaVersion: 1,
- ...lineage,
- evidenceClass: 'real-runtime',
- policyMode: 'bypassed',
- qualificationEligible: true,
- toolExposure: 'direct-e2e',
- scenarioId: scenario.id,
- producer: 'computer-use/real-model',
- transportClass: 'live-network',
- provider: runResult.connection.providerType,
- model: runResult.connection.model,
- fixtureIdentity,
- terminal: safeEvent(runResult.terminalEvent),
- run: runHeader
- ? {
- status: runHeader.status,
- failureClass: runHeader.failureClass,
- failure: safeFailureMetadata(runHeader.failureMessage),
- durationMs:
- runHeader.completedAt !== undefined
- ? Math.max(0, runHeader.completedAt - runHeader.createdAt)
- : undefined,
- }
- : undefined,
- actionAttempts: actions.length,
- actionCount: actions.length,
- actionCounts,
- minimumActionsPassed,
- actionsWithinBudget,
- dispatchPathPassed,
- actions,
- fixtureState,
- expectedState: evaluation.expected,
- forbiddenEffects: {
- status: evaluation.forbidden.every((entry) => entry.pass) ? 'pass' : 'fail',
- violations: evaluation.forbidden.filter((entry) => !entry.pass),
- },
- status: localChecksPassed
- ? 'pass'
- : terminalPassed && minimumActionsPassed
- ? 'fail'
- : 'inconclusive',
- traces: events.map(sanitizeCuTrace).filter(Boolean),
- driverTraces: driverTraces.map(sanitizeCuTrace).filter(Boolean),
- };
- const provider = {
- id: runResult.connection.providerType,
- producer: 'computer-use/real-model',
- model: runResult.connection.model,
- };
- const provisionalReport = sanitizeCuReport(reportInput);
- const qualificationErrors = validateRealReport(
- { ...provisionalReport, status: 'pass' },
- provider,
- scenario,
- );
- const report = sanitizeCuReport({
- ...reportInput,
- status: qualificationErrors.length === 0 ? 'pass' : 'fail',
- });
- const validationErrors = validateRealReport(report, provider, scenario);
- await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, {
- flag: 'wx',
- mode: 0o600,
- });
- process.stdout.write(`Real-model Computer Use report: ${reportPath}\n`);
- process.stdout.write(`${JSON.stringify(report, null, 2)}\n`);
- if (validationErrors.length > 0) {
- process.stderr.write(
- `Real-model Computer Use qualification failed: ${validationErrors.join('; ')}\n`,
- );
- process.exitCode = 1;
- }
- } finally {
- await fixtureBrowser?.close().catch(() => {});
- await desktop?.close().catch(() => {});
- if (fixture && fixture.exitCode === null) fixture.kill('SIGTERM');
- await new Promise((resolve) => setTimeout(resolve, 250));
- if (fixture && fixture.exitCode === null) fixture.kill('SIGKILL');
- if (keepProfile) {
- process.stderr.write(`Retained isolated debug profile: ${userData}\n`);
- } else {
- await rm(userData, { recursive: true, force: true });
- }
- }
-}
-
-async function handleRunFailure(error) {
- const lineage = await reportLineage().catch(() => undefined);
- const failure = sanitizeCuReport({
- schemaVersion: 1,
- ...lineage,
- evidenceClass: 'real-runtime',
- scenarioId: scenario.id,
- producer: 'computer-use/real-model',
- status: 'inconclusive',
- terminal: { type: 'error' },
- run: {
- status: 'failed',
- failureClass:
- typeof error?.code === 'string'
- ? error.code
- : error instanceof Error
- ? error.name
- : 'unknown',
- },
- });
- await mkdir(dirname(reportPath), { recursive: true }).catch(() => {});
- await writeFile(reportPath, `${JSON.stringify(failure, null, 2)}\n`, {
- flag: 'wx',
- mode: 0o600,
- }).catch(() => {});
- console.error('Real-model Computer Use E2E failed');
- process.exitCode = 1;
-}
-
-function requiredDispatchPathPassed(scenario, traces) {
- const mutationActions = scenario.allowedActions.filter(
- (action) => !['list_apps', 'launch_app', 'observe', 'screenshot', 'wait'].includes(action),
- );
- if (mutationActions.length === 0) return true;
- return traces.some(
- (trace) =>
- trace.type === 'dispatch' && (trace.address === 'ax' || trace.address === 'semantic'),
- );
-}
-
-function fixtureOwnsTarget(fixtureIdentity, pid, windowId) {
- return (
- fixtureIdentity?.instances?.some(
- (instance) => instance.pid === pid && instance.windowIds?.includes(windowId),
- ) === true
- );
-}
-
-export function bindActionTargets(actions, traces, fixtureIdentity) {
- const dispatches = traces.filter(
- (trace) =>
- trace.type === 'dispatch' && (trace.address === 'ax' || trace.address === 'semantic'),
- );
- const observationTargets = new Map(
- actions.flatMap((action) =>
- (action.type === 'observe' || action.type === 'screenshot') &&
- typeof action.resultObservationId === 'string' &&
- Number.isInteger(action.targetPid) &&
- Number.isInteger(action.targetWindowId)
- ? [
- [
- action.resultObservationId,
- {
- pid: action.targetPid,
- windowId: action.targetWindowId,
- },
- ],
- ]
- : [],
- ),
- );
- const consumed = new Set();
- return actions.map((action) => {
- if (['list_apps', 'launch_app', 'wait'].includes(action.type)) {
- return { ...action, targetOwned: false };
- }
- const index = dispatches.findIndex(
- (trace, traceIndex) =>
- !consumed.has(traceIndex) &&
- trace.toolCallId === action.toolCallId &&
- trace.actionType === action.type,
- );
- if (index >= 0) consumed.add(index);
- const dispatch = index >= 0 ? dispatches[index] : undefined;
- const sourceTarget =
- typeof action.sourceObservationId === 'string'
- ? observationTargets.get(action.sourceObservationId)
- : undefined;
- const directObservationTarget =
- (action.type === 'observe' || action.type === 'screenshot') &&
- Number.isInteger(action.targetPid) &&
- Number.isInteger(action.targetWindowId)
- ? { pid: action.targetPid, windowId: action.targetWindowId }
- : undefined;
- const target = dispatch
- ? { pid: dispatch.pid, windowId: dispatch.windowId }
- : sourceTarget
- ? sourceTarget
- : directObservationTarget;
- return {
- ...action,
- targetOwned: target ? fixtureOwnsTarget(fixtureIdentity, target.pid, target.windowId) : false,
- ...(target
- ? {
- targetPid: target.pid,
- targetWindowId: target.windowId,
- }
- : {}),
- };
- });
-}
-
-export function allActionTargetsOwned(actions) {
- return actions.every(
- (action) =>
- ['list_apps', 'launch_app', 'wait'].includes(action.type) || action.targetOwned === true,
- );
-}
-
-function activeWindowSpecs(scenario) {
- const specs = new Map(scenario.fixtureSetup.windows.map((window) => [window.id, window]));
- for (const transition of scenario.fixtureSetup.transitions ?? []) {
- specs.delete(transition.removeWindowId);
- specs.set(transition.addWindow.id, transition.addWindow);
- }
- return [...specs.values()];
-}
-
-async function waitForRunHeader(store, sessionId, turnId) {
- const deadline = Date.now() + 2_000;
- let latest;
- do {
- latest = (await store.listSessionRuns(sessionId)).find((entry) => entry.turnId === turnId);
- if (
- latest &&
- latest.status !== 'created' &&
- latest.status !== 'running' &&
- latest.status !== 'waiting_for_user'
- )
- return latest;
- await new Promise((resolve) => setTimeout(resolve, 50));
- } while (Date.now() < deadline);
- return latest;
-}
-
-if (process.argv[1] && resolve(process.argv[1]) === resolve(fileURLToPath(import.meta.url))) {
- run().catch(handleRunFailure);
-}
diff --git a/scripts/computer-use/report-sanitize.mjs b/scripts/computer-use/report-sanitize.mjs
deleted file mode 100644
index 9c1c35dd18..0000000000
--- a/scripts/computer-use/report-sanitize.mjs
+++ /dev/null
@@ -1,384 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-const SAFE_TRACE_KEYS = new Set([
- 'type',
- 'actionType',
- 'path',
- 'effect',
- 'verified',
- 'supported',
- 'ok',
- 'durationMs',
- 'at',
- 'toolCallId',
- 'pid',
- 'windowId',
- 'expectedPid',
- 'expectedWindowId',
- 'winnerPid',
- 'winnerWindowId',
- 'winnerZIndex',
- 'address',
- 'tool',
-]);
-const SAFE_TRACE_TYPES = new Set([
- 'dispatch',
- 'fallback',
- 'occlusion',
- 'outcome',
- 'semantic_result',
- 'snapshot',
- 'target',
- 'tool_start',
- 'tool_result',
- 'complete',
- 'abort',
- 'error',
-]);
-const SAFE_ACTION_TYPES = new Set([
- 'list_apps',
- 'launch_app',
- 'observe',
- 'click_element',
- 'set_value',
- 'select_text',
- 'secondary_action',
- 'scroll_element',
- 'element_sequence',
- 'window_action',
- 'press_key',
- 'screenshot',
- 'type',
- 'key',
- 'wait',
- 'unknown',
-]);
-const SAFE_TRACE_PATHS = new Set(['ax', 'cdp', 'cgevent', 'screenshot-detail']);
-const SAFE_TRACE_EFFECTS = new Set(['confirmed', 'unverifiable']);
-const SAFE_DISPATCH_ADDRESSES = new Set(['ax', 'px', 'semantic', 'none']);
-const SAFE_DISPATCH_TOOLS = new Set(['click', 'set_value', 'page', 'press_key', 'scroll']);
-
-function safeUrlOrigin(value) {
- try {
- return new URL(value).origin;
- } catch {
- return undefined;
- }
-}
-
-function actionType(action) {
- return typeof action?.type === 'string'
- ? action.type
- : typeof action?.action === 'string'
- ? action.action
- : 'unknown';
-}
-
-function safeToken(value, allowed) {
- return typeof value === 'string' && allowed.has(value) ? value : undefined;
-}
-
-function resultCode(text) {
- if (typeof text !== 'string') return undefined;
- const failed = text.match(/\bfailed:\s*([a-z][a-z0-9_]{1,63})\b/i);
- if (failed) return failed[1];
- const status = text.match(/\b(?:ok|error)=([a-z][a-z0-9_]{1,63})\b/i);
- return status?.[1];
-}
-
-export function sanitizeCuActionRecord(record) {
- const type = actionType(record?.action ?? record);
- return {
- type: SAFE_ACTION_TYPES.has(type) ? type : 'unknown',
- ...(safeId(record?.toolCallId) ? { toolCallId: safeId(record.toolCallId) } : {}),
- ...(safeId(record?.sourceObservationId)
- ? { sourceObservationId: safeId(record.sourceObservationId) }
- : {}),
- ...(type !== 'screenshot' && safeId(record?.resultObservationId)
- ? { resultObservationId: safeId(record.resultObservationId) }
- : {}),
- ...(Number.isInteger(record?.targetPid) && record.targetPid > 0
- ? { targetPid: record.targetPid }
- : {}),
- ...(Number.isInteger(record?.targetWindowId) && record.targetWindowId > 0
- ? { targetWindowId: record.targetWindowId }
- : {}),
- ...(typeof record?.success === 'boolean' ? { success: record.success } : {}),
- ...(typeof record?.targetOwned === 'boolean' ? { targetOwned: record.targetOwned } : {}),
- ...(record?.expectedFailure === true ? { expectedFailure: true } : {}),
- ...(Number.isFinite(record?.durationMs) ? { durationMs: record.durationMs } : {}),
- ...(Number.isFinite(record?.modelLatencyMs) ? { modelLatencyMs: record.modelLatencyMs } : {}),
- ...(Number.isFinite(record?.toolLatencyMs) ? { toolLatencyMs: record.toolLatencyMs } : {}),
- ...(Number.isFinite(record?.displayLagMs) ? { displayLagMs: record.displayLagMs } : {}),
- ...(resultCode(record?.text) ? { resultCode: resultCode(record.text) } : {}),
- };
-}
-
-export function sanitizeCuTrace(trace) {
- if (!trace || typeof trace !== 'object') return null;
- if (!SAFE_TRACE_TYPES.has(trace.type)) return null;
- const sanitized = {};
- for (const [key, value] of Object.entries(trace)) {
- if (!SAFE_TRACE_KEYS.has(key)) continue;
- if (key === 'type') {
- sanitized.type = trace.type;
- continue;
- }
- if (key === 'actionType') {
- const action = safeToken(value, SAFE_ACTION_TYPES);
- if (action) sanitized.actionType = action;
- continue;
- }
- if (key === 'path') {
- const path = safeToken(value, SAFE_TRACE_PATHS);
- if (path) sanitized.path = path;
- continue;
- }
- if (key === 'effect') {
- const effect = safeToken(value, SAFE_TRACE_EFFECTS);
- if (effect) sanitized.effect = effect;
- continue;
- }
- if (key === 'address') {
- const address = safeToken(value, SAFE_DISPATCH_ADDRESSES);
- if (address) sanitized.address = address;
- continue;
- }
- if (key === 'tool') {
- const tool = safeToken(value, SAFE_DISPATCH_TOOLS);
- if (tool) sanitized.tool = tool;
- continue;
- }
- if (typeof value === 'string' || typeof value === 'boolean' || Number.isFinite(value)) {
- sanitized[key] = value;
- }
- }
- return Object.keys(sanitized).length > 0 ? sanitized : null;
-}
-
-export function sanitizeCuDirectReport(report) {
- return sanitizeCuReport({
- ...report,
- endpointOrigin: safeUrlOrigin(report?.endpointOrigin ?? report?.baseUrl),
- });
-}
-
-export function sanitizeCuReport(report) {
- const terminal = report?.terminal;
- const fixtureState = report?.fixtureState;
- const expectedState = Array.isArray(report?.expectedState)
- ? report.expectedState.map(sanitizeAssertionResult).filter(Boolean)
- : [];
- const violations = Array.isArray(report?.forbiddenEffects?.violations)
- ? report.forbiddenEffects.violations.map(sanitizeAssertionResult).filter(Boolean)
- : [];
- return {
- schemaVersion: report?.schemaVersion === 1 ? 1 : undefined,
- runId: safeId(report?.runId),
- gitRevision: safeGitRevision(report?.gitRevision),
- generatedAt: safeTimestamp(report?.generatedAt),
- contentLineage: sanitizeContentLineage(report?.contentLineage),
- evidenceClass: safeToken(
- report?.evidenceClass,
- new Set(['real-runtime', 'fault-injection', 'hermetic-protocol', 'static-contract']),
- ),
- scenarioId: safeId(report?.scenarioId),
- producer: safeId(report?.producer),
- transportClass: safeToken(
- report?.transportClass,
- new Set(['live-network', 'hermetic', 'static']),
- ),
- policyMode: safeToken(report?.policyMode, new Set(['enforced', 'bypassed'])),
- qualificationEligible: report?.qualificationEligible === true,
- deprecated: report?.deprecated === true,
- toolExposure: safeToken(report?.toolExposure, new Set(['direct-e2e', 'deferred'])),
- provider: safeId(report?.provider),
- model: safeId(report?.model),
- endpointOrigin: safeUrlOrigin(report.baseUrl),
- status: safeToken(report?.status, new Set(['pass', 'fail', 'inconclusive'])),
- terminal:
- terminal && typeof terminal === 'object'
- ? {
- type: safeToken(terminal.type, new Set(['complete', 'abort', 'error'])),
- stopReason: safeToken(
- terminal.stopReason,
- new Set([
- 'end_turn',
- 'max_tokens',
- 'step_limit',
- 'error',
- 'user_stop',
- 'permission_handoff',
- ]),
- ),
- }
- : undefined,
- fixtureIdentity: sanitizeFixtureIdentity(report?.fixtureIdentity),
- faultInjection: sanitizeFaultInjection(report?.faultInjection),
- run:
- report?.run && typeof report.run === 'object'
- ? {
- status: safeToken(
- report.run.status,
- new Set([
- 'created',
- 'running',
- 'waiting_for_user',
- 'completed',
- 'failed',
- 'cancelled',
- ]),
- ),
- failureClass: safeId(report.run.failureClass),
- ...(Number.isFinite(report.run.durationMs)
- ? { durationMs: report.run.durationMs }
- : {}),
- }
- : undefined,
- ...(Number.isFinite(report?.totalLatencyMs) ? { totalLatencyMs: report.totalLatencyMs } : {}),
- actionAttempts: Number.isInteger(report?.actionAttempts) ? report.actionAttempts : undefined,
- actionCount: Number.isInteger(report?.actionCount) ? report.actionCount : undefined,
- actionCounts: sanitizeCountMap(report?.actionCounts),
- minimumActionsPassed: report?.minimumActionsPassed === true,
- actionsWithinBudget: report?.actionsWithinBudget === true,
- dispatchPathPassed: report?.dispatchPathPassed === true,
- actions: Array.isArray(report?.actions) ? report.actions.map(sanitizeCuActionRecord) : [],
- fixtureState: sanitizeFixtureState(fixtureState),
- expectedState,
- forbiddenEffects: {
- status: safeToken(report?.forbiddenEffects?.status, new Set(['pass', 'fail'])),
- violations,
- },
- traces: Array.isArray(report?.traces) ? report.traces.map(sanitizeCuTrace).filter(Boolean) : [],
- driverTraces: Array.isArray(report?.driverTraces)
- ? report.driverTraces.map(sanitizeCuTrace).filter(Boolean)
- : [],
- };
-}
-
-function sanitizeContentLineage(value) {
- if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
- const generator = safeScriptPath(value.generator);
- const gitRevision = safeGitRevision(value.gitRevision);
- const generatedAt = safeTimestamp(value.generatedAt);
- if (!generator || !gitRevision || !generatedAt) return undefined;
- return { generator, gitRevision, generatedAt };
-}
-
-function safeScriptPath(value) {
- return typeof value === 'string' && /^scripts\/[A-Za-z0-9._/-]{1,192}$/.test(value)
- ? value
- : undefined;
-}
-
-function safeGitRevision(value) {
- return typeof value === 'string' && /^[a-f0-9]{40}$/.test(value) ? value : undefined;
-}
-
-function safeTimestamp(value) {
- if (typeof value !== 'string') return undefined;
- const timestamp = Date.parse(value);
- return Number.isFinite(timestamp) && new Date(timestamp).toISOString() === value
- ? value
- : undefined;
-}
-
-function sanitizeFixtureIdentity(value) {
- if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
- const candidates = Array.isArray(value.instances) ? value.instances : [value];
- const instances = candidates.flatMap((candidate) => {
- if (!candidate || typeof candidate !== 'object' || Array.isArray(candidate)) return [];
- const pid = Number.isInteger(candidate.pid) && candidate.pid > 0 ? candidate.pid : undefined;
- const windowIds = Array.isArray(candidate.windowIds)
- ? [...new Set(candidate.windowIds.filter((entry) => Number.isInteger(entry) && entry > 0))]
- : [];
- return pid && windowIds.length > 0 ? [{ pid, windowIds }] : [];
- });
- if (instances.length === 0) return undefined;
- return {
- instances: instances.filter(
- (candidate, index) =>
- instances.findIndex(
- (entry) =>
- entry.pid === candidate.pid &&
- JSON.stringify(entry.windowIds) === JSON.stringify(candidate.windowIds),
- ) === index,
- ),
- };
-}
-
-function sanitizeFaultInjection(value) {
- if (!value || typeof value !== 'object' || Array.isArray(value)) return undefined;
- const layer = safeToken(value.layer, new Set(['runtime']));
- const kind = safeId(value.kind);
- return layer && kind ? { layer, kind } : undefined;
-}
-
-function safeId(value) {
- return typeof value === 'string' && /^[A-Za-z0-9._-]{1,128}$/.test(value) ? value : undefined;
-}
-
-function sanitizeCountMap(value) {
- if (!value || typeof value !== 'object' || Array.isArray(value)) return {};
- return Object.fromEntries(
- Object.entries(value).flatMap(([key, count]) =>
- SAFE_ACTION_TYPES.has(key) && Number.isInteger(count) && count >= 0 ? [[key, count]] : [],
- ),
- );
-}
-
-function sanitizeFixtureState(value) {
- if (!value || typeof value !== 'object' || Array.isArray(value)) return {};
- return Object.fromEntries(
- Object.entries(value).flatMap(([windowId, state]) => {
- if (!safeId(windowId) || !state || typeof state !== 'object' || Array.isArray(state)) {
- return [];
- }
- const safeState = Object.fromEntries(
- Object.entries(state).flatMap(([key, field]) =>
- safeId(key) &&
- (typeof field === 'boolean' ||
- typeof field === 'number' ||
- (typeof field === 'string' && /^[A-Za-z0-9 .:_-]{0,128}$/.test(field)))
- ? [[key, field]]
- : [],
- ),
- );
- return [[windowId, safeState]];
- }),
- );
-}
-
-function sanitizeAssertionResult(value) {
- if (!value || typeof value !== 'object') return null;
- const windowId = safeId(value.windowId);
- const path = safeId(value.path);
- if (!windowId || !path || typeof value.pass !== 'boolean') return null;
- return {
- windowId,
- path,
- pass: value.pass,
- ...(typeof value.actual === 'boolean' ||
- typeof value.actual === 'number' ||
- typeof value.actual === 'string'
- ? { actual: value.actual }
- : {}),
- };
-}
diff --git a/scripts/generate-cua-driver-notices.mjs b/scripts/generate-cua-driver-notices.mjs
new file mode 100644
index 0000000000..ebfed7687f
--- /dev/null
+++ b/scripts/generate-cua-driver-notices.mjs
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { createHash } from 'node:crypto';
+import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const root = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const licenseRoot = join(root, 'apps/desktop/resources/licenses/cua-driver');
+const outputPath = join(licenseRoot, 'THIRD_PARTY_NOTICES.txt');
+const inventory = readFileSync(join(root, 'docs/computer-use-cua-driver-dependencies.tsv'), 'utf8')
+ .trimEnd()
+ .split('\n')
+ .slice(1)
+ .map((line) => {
+ const [name, version, license, targets, source] = line.split('\t');
+ return { name, version, license, targets, source };
+ });
+
+const registry = readdirSync(join(homedir(), '.cargo/registry/src'))[0];
+if (!registry) throw new Error('Cargo registry source cache is empty');
+const registryRoot = join(homedir(), '.cargo/registry/src', registry);
+const upstream = {
+ 'madsmtm/objc2': {
+ '4fc083f1c6d6784577e38b0ee8dbd344481e2fd2': 'upstream/objc2-legacy-LICENSE.txt',
+ e282618be4c3a3b9542957e0c8540e9588472ce8: 'upstream/objc2-legacy-LICENSE.txt',
+ b4167b582b2f75f9a1be75495c41b765344fd03c: 'upstream/objc2-current-LICENSE.md',
+ '8852b424193ca41602281b3d7540d7c8ed51e49a': 'upstream/objc2-current-LICENSE.md',
+ '7b1abfd750a2cacaea71d6a56ecfb83cb7de560b': 'upstream/objc2-current-LICENSE.md',
+ '8d214f5477365ffcbcbb7de058c86ed9a518efb7': 'upstream/objc2-current-LICENSE.md',
+ },
+ 'mozilla/uniffi-rs': {
+ '309762f55db3f0548194a9ceba3027fa64b18a93': 'MPL-2.0.txt',
+ },
+ 'Nugine/simd': {
+ d74c030d9dc4f3cae02146d1f497ff62726ef09a: 'upstream/simd-LICENSE',
+ },
+ 'Stranger6667/jsonschema': {
+ ecaeceac2340908a8fbf71404442296bd1536520: 'upstream/jsonschema-LICENSE',
+ },
+};
+
+const hash = (value) => createHash('sha256').update(value).digest('hex');
+const texts = new Map();
+const entries = [];
+for (const item of inventory) {
+ if (item.source.includes('trycua/cua')) continue;
+ const directory = join(registryRoot, `${item.name}-${item.version}`);
+ if (!existsSync(directory)) throw new Error(`Cargo source missing: ${item.name}@${item.version}`);
+ let licenses = readdirSync(directory)
+ .filter((name) => /^(licen[cs]e|copying|notice)(?:[._-].*)?$/iu.test(name))
+ .sort()
+ .map((name) => ({ label: name, path: join(directory, name) }));
+ if (licenses.length === 0) {
+ const vcs = JSON.parse(readFileSync(join(directory, '.cargo_vcs_info.json'), 'utf8'));
+ const repository = Object.keys(upstream).find((name) => item.source.includes(name));
+ const source = upstream[repository]?.[vcs.git.sha1];
+ if (!source) throw new Error(`Unreviewed license source: ${item.name}@${item.version}`);
+ licenses = [
+ {
+ label: `${repository}@${vcs.git.sha1}/${source.split('/').at(-1)}`,
+ path: join(licenseRoot, source),
+ },
+ ];
+ }
+ const references = licenses.map(({ label, path }) => {
+ const body = readFileSync(path, 'utf8')
+ .replace(/\r\n?/gu, '\n')
+ .replace(/[\t ]+$/gmu, '')
+ .trimEnd();
+ const digest = hash(body);
+ texts.set(digest, body);
+ return `${label}: sha256:${digest}`;
+ });
+ entries.push(
+ `${item.name} ${item.version}\n License: ${item.license}\n Source: ${item.source}\n Text: ${references.join('; ')}`,
+ );
+}
+
+const output = `Cua Driver 0.28.2 third-party dependency notices
+================================================
+
+This conservative inventory includes normal and build dependencies for both
+macOS architectures. It can include crates absent from the published executable.
+The Cua Driver's own MIT license and the embedded Inter font's OFL license are
+separate files beside this notice. Source: docs/computer-use-cua-driver-dependencies.tsv.
+
+Packages
+--------
+${entries.join('\n\n')}
+
+License texts
+-------------
+${[...texts]
+ .sort(([a], [b]) => a.localeCompare(b))
+ .map(([digest, body]) => `sha256:${digest}\n${body}`)
+ .join('\n\n')}
+`;
+if (process.argv.includes('--check')) {
+ if (readFileSync(outputPath, 'utf8') !== output) throw new Error('Cua Driver notices are stale');
+} else {
+ writeFileSync(outputPath, output);
+ console.log(`Wrote ${outputPath}`);
+}
diff --git a/scripts/package-macos.mjs b/scripts/package-macos.mjs
index 6d5d914ef8..f7bca2d69b 100644
--- a/scripts/package-macos.mjs
+++ b/scripts/package-macos.mjs
@@ -126,6 +126,7 @@ export async function packageMacos({
await run('npm', ['run', 'build']);
await run('npm', ['run', 'build:runtime-host-peer']);
await run('npm', ['run', 'check:runtime-host-peer-notices']);
+ await run('node', ['scripts/computer-use.mjs', 'prepare']);
await run('npm', ['run', 'check:release']);
await remove(target.releaseDirectory, { recursive: true, force: true });
await run('npm', ['--workspace', '@maka/desktop', 'run', `package:macos-${targetArch}`]);
diff --git a/scripts/verify-macos-dmg.mjs b/scripts/verify-macos-dmg.mjs
index 21a5e6a05b..a44bb02d51 100644
--- a/scripts/verify-macos-dmg.mjs
+++ b/scripts/verify-macos-dmg.mjs
@@ -126,6 +126,35 @@ async function readPlistValue(run, infoPlist, key) {
return stdout.trim();
}
+export async function verifyBundledCuaDriver(
+ resources,
+ { run = runCommandFromRepo, checksum = sha256File } = {},
+) {
+ const manifest = JSON.parse(
+ await readFile(join(resources, 'bundled-tools.json'), 'utf8'),
+ ).cuaDriver;
+ if (manifest?.distributionReady !== true || !/^[a-f0-9]{64}$/.test(manifest.binarySha256 ?? '')) {
+ throw new Error('Packaged Cua Driver has no release-ready digest');
+ }
+ const binary = join(resources, 'bin', 'cua-driver');
+ if ((await checksum(binary)) !== manifest.binarySha256) {
+ throw new Error('Packaged Cua Driver digest differs from the pinned release');
+ }
+ await run('codesign', ['--verify', '--strict', '--verbose=2', binary]);
+ const signature = await run('codesign', ['-dv', '--verbose=4', binary]);
+ const signatureText = `${signature.stdout}${signature.stderr}`;
+ if (
+ !signatureText.includes('TeamIdentifier=YCK386LBJ7') ||
+ !signatureText.includes('Authority=Developer ID Application: Cua AI, Inc. (YCK386LBJ7)')
+ ) {
+ throw new Error('Packaged Cua Driver lost its pinned Developer ID signature');
+ }
+ const version = await run(binary, ['--version']);
+ if (!version.stdout.includes('cua-driver 0.28.2')) {
+ throw new Error('Packaged Cua Driver version differs from the pinned release');
+ }
+}
+
export async function verifyPackagedMacApp(
appPath,
{
@@ -167,6 +196,7 @@ export async function verifyPackagedMacApp(
await requirePath(executable);
await assertPackagedResources(resources, { requirePath, forbidPath });
+ await verifyBundledCuaDriver(resources, { run });
await assertPackagedUpdateConfiguration(resources, { channel });
await assertPackagedDependencyClosure(resources);
diff --git a/scripts/verify-packaged-app.mjs b/scripts/verify-packaged-app.mjs
index 781fa0743c..0c38890851 100644
--- a/scripts/verify-packaged-app.mjs
+++ b/scripts/verify-packaged-app.mjs
@@ -1029,11 +1029,23 @@ export async function assertPackagedResources(
// Current Desktop builds ship the direct-peer Client addon beside its Rust
// notices. Upgrade baselines may predate both resources.
requireDirectPeerArtifact = true,
+ requireCuaDriverArtifact = process.platform === 'darwin',
} = {},
) {
const required = [
'app.asar',
'bundled-tools.json',
+ ...(requireCuaDriverArtifact
+ ? [
+ join('bin', 'cua-driver'),
+ join('licenses', 'cua-driver', 'LICENSE'),
+ join('licenses', 'cua-driver', 'MPL-2.0.txt'),
+ join('licenses', 'cua-driver', 'Inter-OFL.txt'),
+ join('licenses', 'cua-driver', 'CDLA-Permissive-2.0.txt'),
+ join('licenses', 'cua-driver', 'NOTICE.txt'),
+ join('licenses', 'cua-driver', 'THIRD_PARTY_NOTICES.txt'),
+ ]
+ : []),
...(requireCanonicalIcon ? [join('assets', 'icon.png')] : []),
join('workers', 'filesystem-worker.js'),
...(requireDirectPeerArtifact
@@ -1087,15 +1099,7 @@ export async function assertPackagedResources(
join('licenses', 'git'),
join('tools', 'officecli'),
join('licenses', 'officecli'),
- // cua-driver is gone from this repository, and these two forbids stay for the
- // same reason the officecli ones next to them do: `apps/desktop/resources/bin`
- // is gitignored, so a binary a developer prepared before this change is still
- // sitting in their tree and would be packaged without anything noticing.
- join('bin', 'cua-driver'),
join('tools', 'cua-driver'),
- // maka-cu is built from source locally and is not signed, so it may not be in
- // a packaged build at all — an ad-hoc helper fails notarization for the whole
- // app, and `distributionReady` is false for exactly this reason.
join('bin', 'maka-cu'),
join('tools', 'maka-cu'),
];
diff --git a/scripts/verify-packaged-app.test.mjs b/scripts/verify-packaged-app.test.mjs
index 9abda0c910..67e9fdbd32 100644
--- a/scripts/verify-packaged-app.test.mjs
+++ b/scripts/verify-packaged-app.test.mjs
@@ -18,6 +18,7 @@
*/
import assert from 'node:assert/strict';
+import { createHash } from 'node:crypto';
import { mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import { dirname, join, relative } from 'node:path';
@@ -333,6 +334,19 @@ test('accepts the Intel Mach-O architecture for an x64 package', async () => {
join(resources, 'app-update.yml'),
'provider: github\nowner: apache\nrepo: maka\nchannel: dev\nupdaterCacheDirName: "@makadesktop-updater"\n',
);
+ const cuaBinary = join(resources, 'bin', 'cua-driver');
+ const cuaFixture = 'cua-driver fixture';
+ await mkdir(dirname(cuaBinary), { recursive: true });
+ await writeFile(cuaBinary, cuaFixture);
+ await writeFile(
+ join(resources, 'bundled-tools.json'),
+ JSON.stringify({
+ cuaDriver: {
+ distributionReady: true,
+ binarySha256: createHash('sha256').update(cuaFixture).digest('hex'),
+ },
+ }),
+ );
const version = '0.2.0-dev.14.20260902';
const app = join(dirname(resources), 'Maka.app');
await mkdir(join(app, 'Contents'), { recursive: true });
@@ -357,16 +371,62 @@ test('accepts the Intel Mach-O architecture for an x64 package', async () => {
return { stdout: `${values[args[1]]}\n` };
}
if (command === 'lipo') return { stdout: 'x86_64\n' };
- if (
- ['codesign', 'spctl', 'xcrun', join(app, 'Contents', 'MacOS', 'Maka')].includes(command)
- ) {
- return { stdout: '' };
- }
+ if (command === 'codesign' && args[0] === '-dv')
+ return {
+ stdout: '',
+ stderr:
+ 'TeamIdentifier=YCK386LBJ7\nAuthority=Developer ID Application: Cua AI, Inc. (YCK386LBJ7)',
+ };
+ if (command === join(app, 'Contents', 'Resources', 'bin', 'cua-driver'))
+ return { stdout: 'cua-driver 0.28.2\n' };
+ if (['codesign', 'spctl', 'xcrun', join(app, 'Contents', 'MacOS', 'Maka')].includes(command))
+ return { stdout: '', stderr: '' };
throw new Error(`Unexpected command: ${command}`);
},
});
});
+test('the Cua release gate rejects a replaced binary and the wrong signing identity', async (t) => {
+ const { verifyBundledCuaDriver } = await import('./verify-macos-dmg.mjs');
+ const root = await mkdtemp(join(tmpdir(), 'maka-cua-package-'));
+ t.after(() => rm(root, { recursive: true, force: true }));
+ const binary = join(root, 'bin', 'cua-driver');
+ const manifestPath = join(root, 'bundled-tools.json');
+ const pinnedDigest = createHash('sha256').update('pinned binary').digest('hex');
+ await mkdir(dirname(binary), { recursive: true });
+ await writeFile(binary, 'pinned binary');
+ await writeFile(
+ manifestPath,
+ JSON.stringify({
+ cuaDriver: {
+ distributionReady: true,
+ binarySha256: pinnedDigest,
+ },
+ }),
+ );
+ const run = async (command, args) => {
+ if (command === 'codesign' && args[0] === '-dv')
+ return {
+ stdout: '',
+ stderr: 'TeamIdentifier=OTHER\nAuthority=Developer ID Application: Other',
+ };
+ if (command === binary) return { stdout: 'cua-driver 0.28.2\n' };
+ return { stdout: '', stderr: '' };
+ };
+ await assert.rejects(() => verifyBundledCuaDriver(root, { run }), /signatur/);
+ await writeFile(
+ manifestPath,
+ JSON.stringify({ cuaDriver: { distributionReady: false, binarySha256: pinnedDigest } }),
+ );
+ await assert.rejects(() => verifyBundledCuaDriver(root, { run }), /release-ready/);
+ await writeFile(
+ manifestPath,
+ JSON.stringify({ cuaDriver: { distributionReady: true, binarySha256: pinnedDigest } }),
+ );
+ await writeFile(binary, 'replaced binary');
+ await assert.rejects(() => verifyBundledCuaDriver(root, { run }), /digest differs/);
+});
+
describe('assertPackagedDependencyClosure', () => {
test('accepts an artifact whose asar, bundle record, and shipped notices match', async () => {
const resources = await makeResources();