From 63a207a162ab30921920830b5bc47fa1c46008e5 Mon Sep 17 00:00:00 2001 From: miguel Date: Wed, 16 Sep 2026 23:36:45 -0700 Subject: [PATCH 1/2] feat(extension): TypeSafe Jev client and candidate-picking library --- packages/extension/services/jevAct/args.ts | 161 ++++ .../extension/services/jevAct/pageState.ts | 57 ++ packages/extension/services/jevAct/pick.ts | 417 ++++++++++ packages/extension/services/jevAct/tree.ts | 741 ++++++++++++++++++ .../services/jevAct/typesafeClient.ts | 190 +++++ .../extension/tests/jevActModules.test.ts | 370 +++++++++ packages/extension/tests/jevActTree.test.ts | 80 ++ 7 files changed, 2016 insertions(+) create mode 100644 packages/extension/services/jevAct/args.ts create mode 100644 packages/extension/services/jevAct/pageState.ts create mode 100644 packages/extension/services/jevAct/pick.ts create mode 100644 packages/extension/services/jevAct/tree.ts create mode 100644 packages/extension/services/jevAct/typesafeClient.ts create mode 100644 packages/extension/tests/jevActModules.test.ts create mode 100644 packages/extension/tests/jevActTree.test.ts diff --git a/packages/extension/services/jevAct/args.ts b/packages/extension/services/jevAct/args.ts new file mode 100644 index 0000000000..ced1be9146 --- /dev/null +++ b/packages/extension/services/jevAct/args.ts @@ -0,0 +1,161 @@ +import type { Variables } from "@browserbasehq/stagehand-protocol/types"; +import { resolveVariableValue } from "../../handlers/handlerUtils/variables.js"; + +/** + * Deterministic argument parsing for the Jev act pipeline. Jev cannot generate + * text, so every argument is either parsed from the instruction here or chosen + * by Jev from a closed set these helpers produce. + */ + +const VARIABLE = /^%[^%\s]+%$/; + +export function isVariable(value: string): boolean { + return VARIABLE.test(value); +} + +export function isLoneVariable(values: string[]): boolean { + return values.length === 1 && isVariable(values[0]!); +} + +/** Quoted strings and declared %variables%: the closed set a fill value can come from. */ +export function fillValueCandidates(instruction: string, variables?: Variables): string[] { + const values = new Set(); + for (const match of instruction.matchAll(/%([^%\s]+)%/g)) { + if (variables && Object.hasOwn(variables, match[1]!)) values.add(match[0]); + } + for (const quoted of quotedStrings(instruction)) values.add(quoted); + return [...values]; +} + +export function quotedStrings(instruction: string): string[] { + const values: string[] = []; + // Single quotes must sit at word boundaries so "user's" is not an opening quote. + for (const match of instruction.matchAll( + /(? = { + enter: "Enter", + return: "Enter", + tab: "Tab", + escape: "Escape", + esc: "Escape", + space: "Space", + backspace: "Backspace", + delete: "Delete", +}; + +export function parseKey(instruction: string): string | undefined { + const match = /\bpress(?:\s+the)?\s+["']?([a-z0-9]+)["']?/i.exec(instruction); + if (!match) return undefined; + const key = match[1]!; + return SPECIAL_KEYS[key.toLowerCase()] ?? (key.length === 1 ? key : undefined); +} + +/** + * Deterministic only when exactly one real option appears in the instruction. + * A quoted string may name the dropdown, and placeholders often repeat that + * name, so options equal to the control's own name never count. + */ +export function matchOption( + options: string[], + instruction: string, + controlName: string, +): string | undefined { + // Quoted only. Short options ("In", "All", "On") occur in instructions as + // ordinary words; an unquoted mention goes to Jev instead. + const quoted = quotedStrings(instruction).map((value) => value.trim().toLowerCase()); + const mentioned = options.filter( + (option) => + option.toLowerCase() !== controlName.toLowerCase() && + quoted.includes(option.trim().toLowerCase()), + ); + return mentioned.length === 1 ? mentioned[0] : undefined; +} + +export function substituteVariables(value: string, variables?: Variables): string { + if (!variables) return value; + let output = value; + for (const [key, variable] of Object.entries(variables)) { + output = output.split(`%${key}%`).join(resolveVariableValue(variable)); + } + return output; +} + +/** + * Guards text returned by the argument-only LLM call: it must be lifted from + * the instruction (or be a declared variable), never invented. + */ +export function isGroundedText(text: string, instruction: string, variables?: Variables): boolean { + return groundedSpan(text, instruction, variables) !== undefined; +} + +/** + * The instruction's OWN characters for the text the argument LLM returned. + * The model may re-case or re-space what it copies ("abc123" for "AbC123"); + * what gets typed must be the user's literal input, so the match is located + * loosely but the span is copied verbatim from the instruction. + */ +export function groundedSpan( + text: string, + instruction: string, + variables?: Variables, +): string | undefined { + const trimmed = text.trim(); + if (!trimmed) return undefined; + if (isVariable(trimmed)) + return variables && Object.hasOwn(variables, trimmed.slice(1, -1)) ? trimmed : undefined; + const exact = instruction.indexOf(trimmed); + if (exact >= 0) return trimmed; + const loose = new RegExp( + trimmed + .split(/\s+/) + .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")) + .join("\\s+"), + "i", + ).exec(instruction); + return loose ? loose[0] : undefined; +} + +/** + * Replaces resolved %variable% values with their placeholder in anything that + * leaves the process. After an earlier act typed a secret, later snapshots + * contain it (as the field's text), and candidate descriptions would carry it. + */ +export function redactor(variables: Variables | undefined): ((text: string) => string) | undefined { + // Values under three characters are not redacted on purpose: replacing every + // "a" or "no" in a page's text would destroy the candidate descriptions Jev + // reads, and such values are not secrets. Documented with the flag. + const secrets = Object.entries(variables ?? {}) + .map(([name, value]) => [name, resolveVariableValue(value)] as const) + .filter(([, value]) => value.length >= 3) + .sort((a, b) => b[1].length - a[1].length); + if (secrets.length === 0) return undefined; + return (text) => { + let output = text; + for (const [name, value] of secrets) { + for (const form of new Set([value, JSON.stringify(value).slice(1, -1)])) { + output = output.split(form).join(`%${name}%`); + } + } + return output; + }; +} diff --git a/packages/extension/services/jevAct/pageState.ts b/packages/extension/services/jevAct/pageState.ts new file mode 100644 index 0000000000..3390b3b77f --- /dev/null +++ b/packages/extension/services/jevAct/pageState.ts @@ -0,0 +1,57 @@ +import { ask, round, type AskContext } from "./pick.js"; +import { pageDigest, type OutlineNode } from "./tree.js"; +import { noulAnswer } from "./typesafeClient.js"; + +/** + * Page-state signals from one Jev request over a compact page digest. Used to + * explain why a target could not be found (the page is a bot wall, not a + * decision-tree miss) and to stop paying for an LLM fallback that cannot help. + */ +export const PAGE_STATE_SIGNALS = { + access_denied: + "The page is an access-denied, forbidden, or 'you have been blocked' page instead of the site's real content", + captcha: "The page is a CAPTCHA, 'verify you are human', or bot-check interstitial", + login_wall: + "The page's main content is a sign-in or registration wall that must be passed to continue", + cookie_banner: "A cookie-consent or privacy-choices banner or dialog is present on the page", + error_page: + "The page is an error page such as 404 not found, 500 server error, or 'something went wrong'", + empty_or_loading: "The page is essentially empty or still showing only a loading indicator", +} as const; + +export type PageSignal = keyof typeof PAGE_STATE_SIGNALS; +export type PageState = Record; + +/** A target can never be found on these, whoever looks for it. */ +const BLOCKING: PageSignal[] = ["access_denied", "captcha"]; +const BLOCKING_THRESHOLD = 0.9; + +export async function readPageState( + ctx: AskContext, + nodes: OutlineNode[], + url: string, +): Promise { + const response = await ask( + ctx, + "page_state", + { url, ...pageDigest(nodes) }, + Object.fromEntries( + Object.entries(PAGE_STATE_SIGNALS).map(([key, instructions]) => [ + key, + { type: "noul" as const, instructions }, + ]), + ), + ); + const state = Object.fromEntries( + (Object.keys(PAGE_STATE_SIGNALS) as PageSignal[]).map((key) => [ + key, + round(noulAnswer(response, key).noul), + ]), + ) as PageState; + ctx.trace[ctx.trace.length - 1]!.state = state; + return state; +} + +export function blockingSignal(state: PageState): PageSignal | undefined { + return BLOCKING.find((signal) => state[signal] >= BLOCKING_THRESHOLD); +} diff --git a/packages/extension/services/jevAct/pick.ts b/packages/extension/services/jevAct/pick.ts new file mode 100644 index 0000000000..92709daf6c --- /dev/null +++ b/packages/extension/services/jevAct/pick.ts @@ -0,0 +1,417 @@ +import type { StagehandLogger } from "../../logger.js"; +import { quotedStrings } from "./args.js"; +import { + buildView, + describeCandidate, + exactNameMatches, + nearbyTwins, + scoreCandidates, + type OutlineNode, + type ViewKind, +} from "./tree.js"; +import { + choiceAnswer, + noulAnswer, + systemOne, + type JevConfig, + type JevQuestion, + type JevResponse, + type JsonValue, +} from "./typesafeClient.js"; + +export type TraceEntry = Record; + +export type Snapshot = { + tree: string; + xpathMap: Record; + nodes: OutlineNode[]; +}; + +export type AskContext = { + config: JevConfig; + instruction: string; + trace: TraceEntry[]; + threshold: number; + logger: StagehandLogger; + ensureTimeRemaining: () => void; + /** DOM attribute hints for nameless candidates, keyed by node id. */ + domHints?: Map; + /** Called once per tier right before Jev is asked, e.g. to fetch DOM hints only when needed. */ + prepare?: (candidates: OutlineNode[]) => Promise; + /** + * Applied to everything sent to TypeSafe and to the trace. Values an earlier + * act typed from %variables% are on the page by now; they must not leave it. + */ + redact?: (text: string) => string; +}; + +export type Pick = { + picked?: OutlineNode; + accepted: boolean; + best: number; + none: number; + choice: string; + /** Candidates by descending probability; feeds retries and the focused LLM fallback. */ + ranked: Array<{ id: string; p: number }>; + via: "shortcut" | "jev"; +}; + +export type TargetResult = { + target?: OutlineNode; + reason: string; + ranked: Array<{ id: string; p: number }>; + /** The shortlist is worth showing to the LLM fallback instead of the whole page. */ + credible: boolean; + /** Strict's none-of-these probability on the last ask: high means "not on this page". */ + none: number; +}; + +export const NONE = "none_match"; +const NONE_DESCRIPTION = "None of these elements matches the instruction"; +/** Strict's none-of-these only vetoes the best pick above this probability. */ +const NONE_VETO = 0.9; +const SHARD_SIZE = 254; +const FINALISTS_PER_SHARD = 3; +/** ~4 chars/token against Jev's ~32K-token request limit, with headroom. */ +const SHARD_CHAR_BUDGET = 60_000; +/** Parallel shard requests per pick; pages needing more go to the LLM. */ +const MAX_SHARDS = 8; +/** Above this many candidates, a lexically pruned list is tried first. */ +const PRUNE_ABOVE = 40; +const PRUNE_KEEP = 30; +/** Accepted picks with strict's none above this are held while a later tier gets a chance. */ +const NONE_UNEASY = 0.5; +const CREDIBLE_TOP = 0.5; +const SURE_ENOUGH = 0.5; +const NONE_CERTAIN = 0.1; +/** A held pick is only acted on below this; above it strict is saying "not here". */ +const HELD_NONE_MAX = 0.7; +const CLEAR_LEADER = 0.6; +const LEADER_RATIO = 2.5; + +export async function ask( + ctx: AskContext, + node: string, + state: JsonValue, + questions: Record, +): Promise { + ctx.ensureTimeRemaining(); + if (ctx.redact) { + const redacted = JSON.parse(ctx.redact(JSON.stringify({ state, questions }))) as { + state: JsonValue; + questions: Record; + }; + state = redacted.state; + questions = redacted.questions; + } + const response = await systemOne(ctx.config, state, questions); + ctx.trace.push({ + node, + ms: response.durationMs, + questions: Object.keys(questions).length, + inputTokens: response.usage.inputTokens, + }); + return response; +} + +export function annotate(trace: TraceEntry[], data: TraceEntry): void { + const last = trace[trace.length - 1]; + if (last) Object.assign(last, data); +} + +/** + * Resolves a target through tiered views: the family's role-filtered view + * first, then a broad view of every named node (custom widgets built from + * plain divs only show up there). Within a tier, large lists are first cut to + * the candidates that share words with the instruction, so the common case is + * one Jev request; the full list is only asked when that pruned pick is rejected. + */ +export async function pickTarget( + ctx: AskContext, + node: string, + snap: Snapshot, + kinds: ViewKind[], + instructions: string, + options: { filter?: (candidate: OutlineNode) => boolean; quotedTargets?: string[] } = {}, +): Promise { + let reason = "no_candidates"; + let ranked: TargetResult["ranked"] = []; + let held: Pick | undefined; + let lastNone = 1; + const seen = new Set(); + const result = (pick: Pick | undefined): TargetResult => { + // A held pick is a fallback of last resort; when strict leaned hard toward + // "none of these", acting on it is how the wrong modal button got clicked. + const usable = held && held.none <= HELD_NONE_MAX ? held : undefined; + const chosen = pick ?? usable; + const shortlist = chosen?.ranked ?? ranked; + // A held pick that strict distrusts too much to act on is still a + // candidate that was evaluated; say so instead of "no candidates". + const why = held && !usable && !pick ? `held_unusable:none=${held.none}` : reason; + return { + ...(chosen?.picked ? { target: chosen.picked } : {}), + reason: chosen?.picked ? "" : why, + ranked: shortlist, + credible: (shortlist[0]?.p ?? 0) >= CREDIBLE_TOP, + none: chosen ? chosen.none : lastNone, + }; + }; + + for (const kind of kinds) { + const candidates = buildView(snap.nodes, kind).filter(options.filter ?? (() => true)); + const signature = candidates.map((candidate) => candidate.id).join(","); + if (candidates.length === 0 || seen.has(signature)) continue; + seen.add(signature); + + // Exactly one quoted target and exactly one candidate carrying that exact + // name: no ranking needed, but the quote may only be an anchor ("the link + // below 'Pricing'"), so Jev still confirms that one candidate (a ~400-token + // request instead of the full list). + const quoted = options.quotedTargets ?? quotedStrings(ctx.instruction); + if (quoted.length === 1) { + const exact = exactNameMatches(candidates, quoted[0]!); + if (exact.length === 1) { + const confirm = await pickCandidate( + ctx, + `${node}:${kind}:exact_name`, + snap, + exact, + instructions, + ); + if (confirm.picked && confirm.accepted && confirm.none <= NONE_UNEASY) + return result(confirm); + } + } + await ctx.prepare?.(candidates); + + const attempts: OutlineNode[][] = []; + if (candidates.length > PRUNE_ABOVE) { + const scores = scoreCandidates(snap.nodes, candidates, ctx.instruction); + const pruned = candidates + .filter((candidate) => (scores.get(candidate.id) ?? 0) > 0) + .sort((a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0) || a.index - b.index) + .slice(0, PRUNE_KEEP) + .sort((a, b) => a.index - b.index); + if (pruned.length > 0 && pruned.length < candidates.length) attempts.push(pruned); + } + attempts.push(candidates); + + for (const [index, attempt] of attempts.entries()) { + const label = `${node}:${kind}${attempts.length > 1 && index === 0 ? ":pruned" : ""}`; + // The broad tier is a last look, not worth a multi-request fan-out: one + // such act spent 169K Jev tokens to conclude nothing matched. + const pick = await pickCandidate(ctx, label, snap, attempt, instructions, kind !== "broad"); + lastNone = pick.none; + if (pick.ranked.length > 0) ranked = pick.ranked; + if (pick.picked && pick.accepted) { + if (pick.none <= NONE_UNEASY) return result(pick); + // Good enough to act on, but strict suspects the real target is not in + // this view (a modal's "Close" is plain text): let the next tier try. + if (!held || pick.best - pick.none > held.best - held.none) held = pick; + break; + } + reason = `${kind}:${pick.choice}@best=${pick.best},none=${pick.none}`; + // Jev saw plausible targets and could not choose between them. A wider + // list will not make that easier; stop here and hand over the shortlist. + if (pick.picked && pick.none < NONE_UNEASY) return result(undefined); + } + } + return result(undefined); +} + +/** + * Two questions fan out in one request: `best` ranks the candidates with no + * escape hatch, `strict` adds a none-of-these option. Loosely worded + * instructions ("the input field") make a lone strict question hedge toward + * none, so strict only vetoes when it is nearly certain nothing matches. + */ +export async function pickCandidate( + ctx: AskContext, + node: string, + snap: Snapshot, + candidates: OutlineNode[], + instructions: string, + allowShards = true, +): Promise { + const byId = new Map(candidates.map((candidate) => [candidate.id, candidate])); + const describe = (option: OutlineNode) => + describeCandidate(snap.nodes, option, ctx.domHints?.get(option.id)); + const state = { instruction: ctx.instruction }; + const rejected = (choice: string): Pick => ({ + accepted: false, + best: 0, + none: 1, + choice, + ranked: [], + via: "jev", + }); + + let finalists = candidates; + const shards = shardByBudget( + candidates, + (candidate) => JSON.stringify(describe(candidate)).length, + ); + if (shards.length > (allowShards ? MAX_SHARDS : 1)) { + ctx.trace.push({ + node: `${node}:too_many`, + ms: 0, + options: candidates.length, + shards: shards.length, + }); + return rejected("too_many_candidates"); + } + if (shards.length > 1) { + // Too many options (or tokens) for one question: every shard nominates its + // top candidates in parallel, then the finalists go through the real pick. + const responses = await Promise.all( + shards.map((shard, index) => + ask(ctx, `${node}:shard_${index}`, state, { + shard: { + type: "choice", + instructions, + criteria: { + ...Object.fromEntries(shard.map((option) => [option.id, describe(option)])), + [NONE]: NONE_DESCRIPTION, + }, + }, + }), + ), + ); + finalists = responses + .flatMap((response) => + Object.entries(choiceAnswer(response, "shard").probabilities) + .filter(([id, probability]) => id !== NONE && probability >= 0.02) + .sort((a, b) => b[1] - a[1]) + .slice(0, FINALISTS_PER_SHARD) + .map(([id]) => byId.get(id)) + .filter((candidate): candidate is OutlineNode => candidate !== undefined), + ) + .slice(0, SHARD_SIZE); + ctx.trace.push({ + node: `${node}:finalists`, + ms: 0, + options: candidates.length, + shards: shards.length, + finalists: finalists.length, + }); + if (finalists.length === 0) return rejected(NONE); + } + + const criteria = Object.fromEntries(finalists.map((option) => [option.id, describe(option)])); + const single = finalists.length === 1; + const response = await ask(ctx, node, state, { + strict: { + type: "choice", + instructions, + criteria: { ...criteria, [NONE]: NONE_DESCRIPTION }, + }, + best: single + ? { + type: "noul", + instructions: { + question: "Is this element a plausible target for the instruction?", + element: describe(finalists[0]!), + }, + } + : { + type: "choice", + instructions: { + question: instructions, + note: "Pick the best available element even if the wording does not match exactly.", + }, + criteria, + }, + }); + + const strict = choiceAnswer(response, "strict"); + const none = round(strict.probabilities[NONE] ?? 0); + let choice: string; + let best: number; + let ranked: Pick["ranked"]; + if (single) { + choice = finalists[0]!.id; + best = round(noulAnswer(response, "best").noul); + ranked = [{ id: choice, p: best }]; + } else { + const answer = choiceAnswer(response, "best"); + choice = answer.choice; + best = round(answer.confidence); + ranked = Object.entries(answer.probabilities) + .filter(([id]) => byId.has(id)) + .sort((a, b) => b[1] - a[1]) + .slice(0, 10) + .map(([id, p]) => ({ id, p: round(p) })); + } + + // Wrong picks in the eval runs came in at 0.9+ (a description problem, not a + // confidence one), while split-but-right picks sat at 0.5–0.7 with strict + // certain that something matches. Those are safe to take. + // The relaxed rule needs a clear leader: a 0.56 vs 0.38 split between a link + // and the toggle next to it was taken under a looser version, and was wrong. + const leader = ranked[0]?.p ?? 0; + const clearLeader = leader >= CLEAR_LEADER && leader >= LEADER_RATIO * (ranked[1]?.p ?? 0); + let accepted = + (best >= ctx.threshold && none <= NONE_VETO) || + (best >= SURE_ENOUGH && none <= NONE_CERTAIN && clearLeader); + // The vote split between indistinguishable copies of one control (desktop + // and mobile renderings of the same button): either is the answer. + if (!accepted && none <= NONE_UNEASY && ranked.length > 1) { + const [first, second] = ranked as [Pick["ranked"][number], Pick["ranked"][number]]; + const same = (id: string) => { + const { occurrence: _occurrence, ...rest } = describe(byId.get(id)!); + return JSON.stringify(rest); + }; + // Same item only: identical "Delete" buttons in different rows look the + // same to Jev too, and those are not interchangeable. + const sameItem = nearbyTwins(snap.nodes, byId.get(first.id)!).some( + (twin) => twin.id === second.id, + ); + if (sameItem && first.p + second.p >= ctx.threshold && same(first.id) === same(second.id)) { + accepted = true; + choice = first.id; + } + } + annotate(ctx.trace, { + picked: byId.has(choice) + ? (ctx.redact ?? ((text: string) => text))(JSON.stringify(describe(byId.get(choice)!))) + : null, + options: finalists.length, + choice, + best, + none, + strict: strict.choice, + accepted, + runner_up: ranked[1] ? `${ranked[1].id}@${ranked[1].p}` : null, + // What Jev saw, so misses can be attributed to descriptions vs. the model. + candidates: finalists + .slice(0, 12) + .map((option) => + (ctx.redact ?? ((text: string) => text))( + JSON.stringify({ id: option.id, ...describe(option) }), + ), + ), + }); + return { picked: byId.get(choice), accepted, best, none, choice, ranked, via: "jev" }; +} + +function shardByBudget(items: T[], size: (item: T) => number): T[][] { + const shards: T[][] = []; + let current: T[] = []; + let chars = 0; + for (const item of items) { + const cost = size(item) + 16; + if (current.length > 0 && (current.length >= SHARD_SIZE || chars + cost > SHARD_CHAR_BUDGET)) { + shards.push(current); + current = []; + chars = 0; + } + current.push(item); + chars += cost; + } + if (current.length > 0) shards.push(current); + return shards; +} + +export function round(value: number): number { + return Math.round(value * 100) / 100; +} diff --git a/packages/extension/services/jevAct/tree.ts b/packages/extension/services/jevAct/tree.ts new file mode 100644 index 0000000000..7a84fcc8c9 --- /dev/null +++ b/packages/extension/services/jevAct/tree.ts @@ -0,0 +1,741 @@ +import type { JsonValue } from "./typesafeClient.js"; + +/** + * Parses the hybrid snapshot outline back into nodes so the Jev act pipeline + * can build role-filtered candidate views. Lines follow `formatTreeLine`: + * `[frame-node] role: name [selected] [checked]`, two spaces per depth. + */ +export type OutlineNode = { + id: string; + role: string; + name: string; + flags: string[]; + depth: number; + parent?: number; + index: number; + /** Topmost node of an editable region per the browser (covers contenteditable). */ + editable?: boolean; +}; + +export type ViewKind = + | "pointer" + | "input" + | "select" + | "scroll" + | "option" + | "broad" + | "text" + | "link"; + +const LINE = /^( *)\[([^\]]+)\] (.*)$/; +const FLAGS = /((?: \[(?:selected|checked)\])+)$/; +const ENCODED_ID = /^\d+-\d+$/; + +const POINTER_ROLES = new Set([ + "button", + "link", + "checkbox", + "radio", + "tab", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "switch", + "option", + "treeitem", + "summary", + "combobox", + "listbox", + "slider", + "img", + "image", + "disclosuretriangle", +]); +const INPUT_ROLES = new Set(["textbox", "searchbox", "combobox", "spinbutton", "textarea"]); +const SELECT_ROLES = new Set(["combobox", "listbox", "select"]); +const OPTION_ROLES = new Set(["option", "menulistoption"]); +const TEXT_ROLES = new Set([ + "statictext", + "labeltext", + "listitem", + "cell", + "gridcell", + "generic", + "paragraph", + "label", + "menulistoption", +]); +const SCROLL_ROLES = new Set(["rootwebarea", "webarea", "iframe", "html", "document"]); + +/** + * Flags editable regions from the snapshot's side channel. A contenteditable + * paragraph has role `paragraph`; without this it never reaches the fill view + * and the nearest real textbox gets filled instead. + */ +export function markEditable( + nodes: OutlineNode[], + editableIds: Iterable | undefined, +): void { + const editable = new Set(editableIds ?? []); + if (editable.size === 0) return; + for (const node of nodes) { + if (!editable.has(node.id)) continue; + const parent = node.parent === undefined ? undefined : nodes[node.parent]; + if (!parent || !editable.has(parent.id)) node.editable = true; + } +} + +/** True when the snapshot gave the node nothing to be recognised by. */ +export function isNameless(nodes: OutlineNode[], node: OutlineNode): boolean { + return !node.name && !descendants(nodes, node, 6).some((child) => child.name); +} + +export function parseOutline(tree: string): OutlineNode[] { + const nodes: OutlineNode[] = []; + const stack: number[] = []; + + for (const line of tree.split("\n")) { + const match = LINE.exec(line); + if (!match) { + // Names are not newline-stripped by the producer; a line without an id + // is the continuation of the previous node's name. + const previous = nodes[nodes.length - 1]; + if (previous && line.trim()) previous.name = `${previous.name} ${line.trim()}`.trim(); + continue; + } + const [, indent = "", id = "", rest = ""] = match; + const depth = Math.floor(indent.length / 2); + + let body = rest; + const flagMatch = FLAGS.exec(body); + const flags = flagMatch + ? [...flagMatch[1]!.matchAll(/\[(\w+)\]/g)].map((flag) => flag[1]!) + : []; + if (flagMatch) body = body.slice(0, -flagMatch[1]!.length); + + const separator = body.indexOf(": "); + const role = (separator === -1 ? body : body.slice(0, separator)).trim(); + const name = separator === -1 ? "" : body.slice(separator + 2).trim(); + + while (stack.length > 0 && nodes[stack[stack.length - 1]!]!.depth >= depth) stack.pop(); + const node: OutlineNode = { + id, + role, + name, + flags, + depth, + parent: stack[stack.length - 1], + index: nodes.length, + }; + nodes.push(node); + stack.push(node.index); + } + + return nodes; +} + +function roleTokens(node: OutlineNode): string[] { + return node.role + .toLowerCase() + .split(/[\s,]+/) + .filter(Boolean); +} + +function hasRole(node: OutlineNode, roles: Set): boolean { + return roleTokens(node).some((token) => roles.has(token)); +} + +function isAddressable(node: OutlineNode): boolean { + return ENCODED_ID.test(node.id); +} + +/** Candidate nodes for a family. `option` is the post-expand view for custom dropdowns. */ +export function buildView(nodes: OutlineNode[], kind: ViewKind): OutlineNode[] { + const matches = (node: OutlineNode): boolean => { + switch (kind) { + case "pointer": + return hasRole(node, POINTER_ROLES); + case "input": + return hasRole(node, INPUT_ROLES) || node.editable === true; + case "select": + return hasRole(node, SELECT_ROLES) || hasRole(node, POINTER_ROLES); + case "scroll": + return roleTokens(node).includes("scrollable"); + case "link": + return roleTokens(node).includes("link"); + case "text": + // Anything whose text could BE a value: named nodes, plus nameless + // blocks (a paragraph with inline links) whose pieces read as one value. + return ( + (node.name.length > 0 && + !hasRole(node, SCROLL_ROLES) && + !repeatsParentName(nodes, node)) || + isTextBlock(nodes, node) + ); + case "broad": + return ( + node.name.length > 0 && !hasRole(node, SCROLL_ROLES) && !repeatsParentName(nodes, node) + ); + case "option": + return hasRole(node, POINTER_ROLES) || (hasRole(node, TEXT_ROLES) && node.name.length > 0); + } + }; + // A native select's options are arguments, not targets: offered alongside + // the select they split the vote with it (and balloon the candidate count). + const view = nodes.filter( + (node) => isAddressable(node) && matches(node) && !insideNativeSelect(nodes, node), + ); + // A document's RootWebArea and its `scrollable, html` are the same scroller; + // offering both splits the probability mass. Roots only stand in when the + // snapshot marked nothing scrollable. + if (kind === "scroll" && view.length === 0) { + return nodes.filter((node) => isAddressable(node) && hasRole(node, SCROLL_ROLES)); + } + return view; +} + +function insideNativeSelect(nodes: OutlineNode[], node: OutlineNode): boolean { + for (let parent = node.parent; parent !== undefined; parent = nodes[parent]!.parent) { + if (roleTokens(nodes[parent]!).includes("select")) return true; + } + return false; +} + +/** True for a scroll candidate that is the top-level document rather than an iframe or container. */ +export function isMainDocumentScroller(nodes: OutlineNode[], node: OutlineNode): boolean { + return roleTokens(node).includes("html") && !insideIframe(nodes, node); +} + +const TEXT_BLOCK_ROLES = new Set([ + "paragraph", + "cell", + "gridcell", + "listitem", + "heading", + "labeltext", + "time", + "term", + "definition", + "descriptionlistdetail", + "descriptionlistterm", + "p", + "td", + "th", + "li", + "dd", + "dt", + "span", + "div", + "blockquote", + "caption", + "figcaption", +]); +const TEXT_BLOCK_MAX_CHARS = 400; + +function isTextBlock(nodes: OutlineNode[], node: OutlineNode): boolean { + if (node.name || !hasRole(node, TEXT_BLOCK_ROLES)) return false; + const pieces = descendants(nodes, node, 25); + if (pieces.length < 2 || pieces.length >= 25) return false; + // A block of blocks is a container, not a value. + if ( + pieces.some( + (piece) => + !piece.name && + hasRole(piece, TEXT_BLOCK_ROLES) && + piece.depth === node.depth + 1 && + descendants(nodes, piece, 3).length > 1, + ) + ) { + return false; + } + const text = textOf(nodes, node); + return text.length > 0 && text.length <= TEXT_BLOCK_MAX_CHARS; +} + +/** The node's own name, or the text of its subtree read in order. */ +export function textOf(nodes: OutlineNode[], node: OutlineNode): string { + if (node.name) return node.name; + const parts: string[] = []; + for (const piece of descendants(nodes, node, 60)) { + if (!piece.name || repeatsParentName(nodes, piece)) continue; + parts.push(piece.name); + } + return parts.join(" ").replace(/\s+/g, " ").trim(); +} + +function repeatsParentName(nodes: OutlineNode[], node: OutlineNode): boolean { + if (node.parent === undefined || !hasRole(node, TEXT_ROLES)) return false; + const parent = nodes[node.parent]!; + return parent.name === node.name && !hasRole(parent, SCROLL_ROLES); +} + +/** + * Option labels of a native