diff --git a/CHANGELOG.md b/CHANGELOG.md index 66203600..d968eeaa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ All notable changes to this project are documented here. The format follows [Kee - `install --global` (alias `-g`) is a real flag now that the daemon takes it; it needs `--skills`, and says so instead of ignoring it. - `set-color-scheme`, `set-reduced-motion`, `set-forced-colors`, `set-contrast`, `set-media` and their `clear-*` counterparts (new upstream commands): emulate CSS media features. The `set-*` commands carry the new `media-emulation` warning, because `matchMedia` then disagrees with the operating system setting that system colors and native controls follow. - `webmcp-list` / `webmcp-call` (new upstream commands): list and call the tools a page registers through WebMCP. They read the page from the utility world and need `--extra-arg=--enable-features=WebMCP`. +- `snapshot --interactive`: only what can be acted on (interactive roles and `[cursor=pointer]` elements, each with its subtree), plus headings and named landmarks for orientation; printed inline (or to `--filename`, or as a pruned `--json` tree), with the daemon's refs, so `click`/`fill` take them as usual. About half the size of a full snapshot on a link-heavy article, far less on forms and applications. - A negated flag (`--no-`) is accepted whenever the command declares `no-` (upstream). ### Changed diff --git a/skills/patchright-cli/SKILL.md b/skills/patchright-cli/SKILL.md index accf649b..0922eae1 100644 --- a/skills/patchright-cli/SKILL.md +++ b/skills/patchright-cli/SKILL.md @@ -394,6 +394,7 @@ patchright-cli snapshot "#main" # an element instead of the who patchright-cli snapshot --depth=4 # limit depth, then snapshot a subtree patchright-cli snapshot e34 patchright-cli snapshot --boxes # include [box=x,y,width,height] +patchright-cli snapshot --interactive # only controls, headings and named landmarks; prints inline, refs stay valid patchright-cli find "Add to cart" # search instead of capturing everything patchright-cli find --regex "\\$[0-9]+\\.[0-9]{2}" ``` diff --git a/src/commands/interactiveSnapshot.ts b/src/commands/interactiveSnapshot.ts new file mode 100644 index 00000000..d090c3cf --- /dev/null +++ b/src/commands/interactiveSnapshot.ts @@ -0,0 +1,206 @@ +/** + * Copyright (c) 2026 b1zya (https://github.com/b1zya/patchright-cli). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// `snapshot --interactive`: the controls of the page and just enough structure to tell them +// apart, a fraction of the full tree on a content page. The daemon takes the snapshot as usual +// (so its refs stay valid for click/fill/...); the client prunes it. A node is kept whole when +// it can be acted on (an interactive role, or `[cursor=pointer]`); headings are kept as one +// line to orient by; landmarks and other containers are kept only when they hold something +// kept and carry a name or a landmark role; every other wrapper is dropped and its kept +// descendants move up. + +import fs from 'fs'; +import path from 'path'; + +import type { MinimistArgs } from '../args'; +import type { GuardResult } from '../stealth'; +import type { Output } from '../output'; + +const interactiveRoles = new Set([ + 'button', 'checkbox', 'combobox', 'link', 'listbox', 'menuitem', 'menuitemcheckbox', 'menuitemradio', + 'option', 'radio', 'scrollbar', 'searchbox', 'slider', 'spinbutton', 'switch', 'tab', 'textbox', 'treeitem', +]); +const landmarkRoles = new Set([ + 'alertdialog', 'banner', 'complementary', 'contentinfo', 'dialog', 'form', 'grid', 'iframe', 'main', + 'menu', 'menubar', 'navigation', 'region', 'search', 'table', 'tablist', 'toolbar', 'tree', +]); + +type Kind = 'interactive' | 'heading' | 'container'; + +type Pruned = { node: T, kind: Kind, children: Pruned[] }; + +function prune(nodes: T[], kindOf: (node: T) => Kind, childrenOf: (node: T) => T[], keepContainer: (node: T) => boolean): Pruned[] { + const result: Pruned[] = []; + for (const node of nodes) { + const kind = kindOf(node); + if (kind === 'interactive') { + result.push({ node, kind, children: [] }); + continue; + } + if (kind === 'heading') { + result.push({ node, kind, children: [] }); + continue; + } + const children = prune(childrenOf(node), kindOf, childrenOf, keepContainer); + if (children.length && keepContainer(node)) + result.push({ node, kind, children }); + else + result.push(...children); + } + return result; +} + +// --- YAML (text and --raw output) --- + +type YamlNode = { line: string, depth: number, subtree: string[], children: YamlNode[] }; + +const nodeLine = /^(\s*)- (.*)$/; + +function parseYaml(yaml: string): YamlNode[] { + const roots: YamlNode[] = []; + const stack: YamlNode[] = []; + for (const line of yaml.split('\n')) { + const match = line.match(nodeLine); + if (!match) { + // A continuation line (multi-line text): it belongs to the innermost open node. + if (stack.length && line.trim()) + stack[stack.length - 1].subtree.push(line); + continue; + } + const depth = match[1].length / 2; + const node: YamlNode = { line: match[2], depth, subtree: [line], children: [] }; + while (stack.length && stack[stack.length - 1].depth >= depth) + stack.pop(); + for (const open of stack) + open.subtree.push(line); + if (stack.length) + stack[stack.length - 1].children.push(node); + else + roots.push(node); + stack.push(node); + } + return roots; +} + +function yamlRole(line: string): string { + return line.match(/^[\w-]+/)?.[0] ?? ''; +} + +function yamlKind(node: YamlNode): Kind { + const role = yamlRole(node.line); + if (interactiveRoles.has(role) || node.line.includes('[cursor=pointer]')) + return 'interactive'; + if (role === 'heading') + return 'heading'; + return 'container'; +} + +function yamlKeepContainer(node: YamlNode): boolean { + return landmarkRoles.has(yamlRole(node.line)) || /^[\w-]+ "/.test(node.line); +} + +function renderYaml(nodes: Pruned[], depth: number, out: string[]) { + for (const { node, kind, children } of nodes) { + const indent = ' '.repeat(depth); + if (kind === 'interactive') { + // The whole subtree (link targets, options, inner text), re-based to this depth. + const shift = node.depth * 2; + for (const line of node.subtree) + out.push(indent + line.slice(Math.min(shift, line.length - line.trimStart().length))); + } else if (kind === 'heading') { + out.push(`${indent}- ${node.line.replace(/:$/, '')}`); + } else { + out.push(`${indent}- ${node.line.endsWith(':') ? node.line : node.line + ':'}`); + renderYaml(children, depth + 1, out); + } + } +} + +export function interactiveYaml(yaml: string): string { + const out: string[] = []; + renderYaml(prune(parseYaml(yaml), yamlKind, node => node.children, yamlKeepContainer), 0, out); + return out.join('\n'); +} + +// --- JSON (--json output) --- + +type JsonNode = { role?: string, name?: string, cursor?: string, children?: JsonNode[], [key: string]: unknown }; + +function jsonKind(node: JsonNode): Kind { + const role = node.role ?? ''; + if (interactiveRoles.has(role) || node.cursor === 'pointer') + return 'interactive'; + if (role === 'heading') + return 'heading'; + return 'container'; +} + +function rebuildJson(nodes: Pruned[]): JsonNode[] { + return nodes.map(({ node, kind, children }) => { + if (kind === 'interactive') + return node; + const { children: _children, ...rest } = node; + return kind === 'heading' ? rest : { ...rest, children: rebuildJson(children) }; + }); +} + +export function interactiveJson(tree: JsonNode[]): JsonNode[] { + return rebuildJson(prune(tree, jsonKind, node => node.children ?? [], node => landmarkRoles.has(node.role ?? '') || !!node.name)); +} + +// --- the command --- + +// Takes the snapshot inline (raw) and prints, or writes to --filename, the pruned tree. +export function interactiveSnapshotRewrite(args: MinimistArgs, output: Output, cwd: string): GuardResult { + const { interactive: _interactive, filename, raw: userRaw, ...rest } = args; + const plain = !!userRaw; + return { + kind: 'rewrite', + args: rest, + raw: true, + silent: true, + onResult: text => { + if (output.json) { + let parsed: any; + try { + parsed = JSON.parse(text); + } catch { + output.toolResult(text); + return; + } + if (!Array.isArray(parsed?.snapshot)) { + output.toolResult(text); + return; + } + const snapshot = interactiveJson(parsed.snapshot); + if (typeof filename === 'string' && filename) { + fs.writeFileSync(path.resolve(cwd, filename), JSON.stringify(snapshot, null, 2) + '\n'); + output.toolResult(JSON.stringify({ snapshot: { file: filename, interactive: true } }, null, 2)); + } else { + output.toolResult(JSON.stringify({ ...parsed, snapshot }, null, 2)); + } + return; + } + const body = interactiveYaml(text); + if (typeof filename === 'string' && filename) { + fs.writeFileSync(path.resolve(cwd, filename), body + '\n'); + output.toolResult(`- [Snapshot (interactive)](${filename})`); + return; + } + output.toolResult(plain ? body : `### Snapshot (interactive)\n\`\`\`yaml\n${body}\n\`\`\``); + }, + }; +} diff --git a/src/help/help.generated.json b/src/help/help.generated.json index 32c1e0d7..7594ce5a 100644 --- a/src/help/help.generated.json +++ b/src/help/help.generated.json @@ -166,11 +166,12 @@ ] }, "snapshot": { - "help": "patchright-cli snapshot [target]\n\nCapture page snapshot to obtain element ref\n\nArguments:\n [target] element reference from the previous page snapshot, or a unique element selector for the root element to capture a partial snapshot instead of the whole page\nOptions:\n --filename save snapshot to markdown file instead of returning it in the response.\n --depth limit snapshot depth, unlimited by default.\n --boxes include each element's bounding box as [box=x,y,width,height] in the snapshot. coordinates are viewport-relative, in css pixels (element.getboundingclientrect).", + "help": "patchright-cli snapshot [target]\n\nCapture page snapshot to obtain element ref\n\nArguments:\n [target] element reference from the previous page snapshot, or a unique element selector for the root element to capture a partial snapshot instead of the whole page\nOptions:\n --filename save snapshot to markdown file instead of returning it in the response.\n --depth limit snapshot depth, unlimited by default.\n --boxes include each element's bounding box as [box=x,y,width,height] in the snapshot. coordinates are viewport-relative, in css pixels (element.getboundingclientrect).\n --interactive only what can be acted on (links, buttons, inputs, clickable elements) plus headings and named landmarks; refs stay valid", "flags": { "filename": "string", "depth": "string", - "boxes": "boolean" + "boxes": "boolean", + "interactive": "boolean" }, "args": [ "target" @@ -900,6 +901,7 @@ "geoip", "focus-emulation", "humanize", + "interactive", "main-world", "probe", "continue-on-error", diff --git a/src/stealth/flags.ts b/src/stealth/flags.ts index 1767b7d9..11410c34 100644 --- a/src/stealth/flags.ts +++ b/src/stealth/flags.ts @@ -53,6 +53,7 @@ export const clientFlags: ClientFlag[] = [ { command, name: 'idle-timeout', type: 'string' as FlagType, help: 'close the browser after this long without a command (30m, 2h, 90s; 0 keeps it open). default: 30m inside an agent harness, never otherwise' }, { command, name: 'owner-pid', type: 'string' as FlagType, help: 'close the browser when this process exits; defaults to PATCHRIGHT_CLI_OWNER_PID or CLAUDE_PID when set and alive, --no-owner-pid disables' }, ]), + { command: 'snapshot', name: 'interactive', type: 'boolean', help: 'only what can be acted on (links, buttons, inputs, clickable elements) plus headings and named landmarks; refs stay valid' }, { command: 'eval', name: 'main-world', type: 'boolean', help: 'evaluate in the page main world: sees page globals, but is observable by the page' }, { command: 'run-code', name: 'force', type: 'boolean', help: 'run code the safety scan would refuse (Runtime.enable is never allowed)' }, ...['click', 'dblclick', 'hover', 'drag', 'mousemove', 'mousewheel', 'type', 'fill'].map(command => ({ diff --git a/src/stealth/guards.ts b/src/stealth/guards.ts index 75082430..1871148a 100644 --- a/src/stealth/guards.ts +++ b/src/stealth/guards.ts @@ -21,6 +21,7 @@ import fs from 'fs'; +import { interactiveSnapshotRewrite } from '../commands/interactiveSnapshot'; import { humanizeRewrite } from '../humanize'; import { locatorExpression } from './locator'; import { leak, notice } from './warnings'; @@ -180,6 +181,8 @@ function guardCommandInner(name: string, args: MinimistArgs, ctx: CommandContext warnings.push(leak('route-shadows-init')); return { kind: 'allow', warnings }; } + case 'snapshot': + return args.interactive ? interactiveSnapshotRewrite(args, ctx.output, process.cwd()) : { kind: 'allow' }; case 'highlight': return args.hide ? { kind: 'allow' } : { kind: 'allow', warnings: [leak('highlight-dom')] }; case 'video-show-actions': diff --git a/tests/unit/interactiveSnapshot.spec.ts b/tests/unit/interactiveSnapshot.spec.ts new file mode 100644 index 00000000..6fb7b535 --- /dev/null +++ b/tests/unit/interactiveSnapshot.spec.ts @@ -0,0 +1,83 @@ +/** + * Copyright (c) 2026 b1zya (https://github.com/b1zya/patchright-cli). + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { test, expect } from 'patchright/test'; + +import { interactiveJson, interactiveSnapshotRewrite, interactiveYaml } from '../../src/commands/interactiveSnapshot'; + +import type { Output } from '../../src/output'; + +const page = [ + '- generic [active] [ref=e1]:', + ' - heading "Shop" [level=1] [ref=e2]', + ' - navigation [ref=e3]:', + ' - link "Home" [ref=e4] [cursor=pointer]:', + ' - /url: /a', + ' - list [ref=e5]:', + ' - listitem [ref=e6]:', + ' - link "Cart" [ref=e7] [cursor=pointer]:', + ' - /url: /b', + ' - paragraph [ref=e8]: Long text paragraph', + ' - generic [ref=e9]:', + ' - generic [ref=e10]:', + ' - text: Email', + ' - textbox "Email" [ref=e11]', + ' - generic [ref=e14] [cursor=pointer]: Card that opens a product', + ' - combobox [ref=e13]:', + ' - option "A" [selected]', + ' - region "Empty" [ref=e15]:', + ' - paragraph [ref=e16]: nothing to act on', +].join('\n'); + +test('keeps controls with their subtree, headings and landmarks that hold controls; drops the rest', () => { + expect(interactiveYaml(page)).toBe([ + '- heading "Shop" [level=1] [ref=e2]', + '- navigation [ref=e3]:', + ' - link "Home" [ref=e4] [cursor=pointer]:', + ' - /url: /a', + ' - link "Cart" [ref=e7] [cursor=pointer]:', + ' - /url: /b', + '- textbox "Email" [ref=e11]', + '- generic [ref=e14] [cursor=pointer]: Card that opens a product', + '- combobox [ref=e13]:', + ' - option "A" [selected]', + ].join('\n')); +}); + +test('the JSON tree is pruned the same way', () => { + const tree = [{ role: 'generic', ref: 'e1', children: [ + { role: 'heading', name: 'Shop', ref: 'e2', children: [{ role: 'text', name: 'x' }] }, + { role: 'navigation', ref: 'e3', children: [{ role: 'list', children: [{ role: 'link', name: 'Home', ref: 'e4', url: '/a' }] }] }, + { role: 'paragraph', ref: 'e8' }, + { role: 'generic', ref: 'e14', cursor: 'pointer', children: [{ role: 'text', name: 'Card' }] }, + ] }]; + expect(interactiveJson(tree)).toEqual([ + { role: 'heading', name: 'Shop', ref: 'e2' }, + { role: 'navigation', ref: 'e3', children: [{ role: 'link', name: 'Home', ref: 'e4', url: '/a' }] }, + { role: 'generic', ref: 'e14', cursor: 'pointer', children: [{ role: 'text', name: 'Card' }] }, + ]); +}); + +test('snapshot --interactive takes the snapshot inline and prints the pruned tree', () => { + const printed: string[] = []; + const output = { json: false, toolResult: (text: string) => printed.push(text) } as unknown as Output; + const rewrite = interactiveSnapshotRewrite({ _: ['snapshot'], interactive: true, depth: '4' }, output, process.cwd()); + expect(rewrite).toEqual(expect.objectContaining({ kind: 'rewrite', args: { _: ['snapshot'], depth: '4' }, raw: true, silent: true })); + if (rewrite.kind === 'rewrite') + rewrite.onResult!(page); + expect(printed[0]).toMatch(/^### Snapshot \(interactive\)\n```yaml\n- heading "Shop"/); + expect(printed[0]).not.toContain('paragraph'); +});