feat(extension): decision-model client (TypeSafe Jev) and candidate-picking library

packages/extension/services/decisions/args.ts

@@ -0,0 +1,182 @@
+import type { Variables } from "@browserbasehq/stagehand-protocol/types";
+import { resolveVariableValue } from "../../handlers/handlerUtils/variables.js";
+
+/**
+ * Deterministic argument parsing for the Decisions act pipeline. The decision model cannot generate
+ * text, so every argument is either parsed from the instruction here or chosen
+ * by the decision model from a closed set these helpers produce.
+ */
+
+const VARIABLE = /^%[^%\s]+%$/;
+
+export function isVariable(value: string): boolean {
+  return VARIABLE.test(value);
+}
+
+export function isLoneVariable(values: string[]): boolean {
+  return values.length === 1 && isVariable(values[0]!);
+}
+
+/** Quoted strings and declared %variables%: the closed set a fill value can come from. */
+export function fillValueCandidates(instruction: string, variables?: Variables): string[] {
+  const values = new Set<string>();
+  for (const match of instruction.matchAll(/%([^%\s]+)%/g)) {
+    if (variables && Object.hasOwn(variables, match[1]!)) values.add(match[0]);
+  }
+  for (const quoted of quotedStrings(instruction)) values.add(quoted);
+  return [...values];
+}
+
+export function quotedStrings(instruction: string): string[] {
+  const values: string[] = [];
+  // Single quotes must sit at word boundaries so "user's" is not an opening quote.
+  for (const match of instruction.matchAll(
+    /(?<![\w])'([^'\n]+)'(?![\w])|"([^"\n]+)"|“([^”\n]+)”|‘([^’\n]+)’/g,
+  )) {
+    values.push((match[1] ?? match[2] ?? match[3] ?? match[4])!);
+  }
+  return values;
+}
+
+export function parsePercent(instruction: string): string | undefined {
+  const explicit = /(\d+(?:\.\d+)?)\s*%/.exec(instruction);
+  if (explicit) return `${explicit[1]}%`;
+  // The documented fractional form: "scroll to 0.75" means 75%. Only a real
+  // fraction with a decimal point counts: a bare "1" is a count or an ordinal
+  // ("scroll down 1 screen", "the 1st result"), never "the bottom".
+  const fraction = /(?<![\d.\w])(0?\.\d+|1\.0+)(?![\d.%\w])/.exec(instruction);
+  if (fraction && /\b(scroll|down|up|to)\b/i.test(instruction)) {
+    return `${Math.round(parseFloat(fraction[1]!) * 100)}%`;
+  }
+  if (/\bhalf ?way\b|\bmiddle\b/i.test(instruction)) return "50%";
+  if (/\bbottom\b/i.test(instruction)) return "100%";
+  if (/\btop\b/i.test(instruction)) return "0%";
+  return undefined;
+}
+
+const SPECIAL_KEYS: Record<string, string> = {
+  enter: "Enter",
+  return: "Enter",
+  tab: "Tab",
+  escape: "Escape",
+  esc: "Escape",
+  space: "Space",
+  backspace: "Backspace",
+  delete: "Delete",
+};
+
+export function parseKey(instruction: string): string | undefined {
+  const match = /\bpress(?:\s+the)?\s+["']?([a-z0-9]+)["']?/i.exec(instruction);
+  if (!match) return undefined;
+  const key = match[1]!;
+  return SPECIAL_KEYS[key.toLowerCase()] ?? (key.length === 1 ? key : undefined);
+}
+
+/**
+ * Deterministic only when exactly one real option appears in the instruction.
+ * A quoted string may name the dropdown, and placeholders often repeat that
+ * name, so options equal to the control's own name never count.
+ */
+export function matchOption(
+  options: string[],
+  instruction: string,
+  controlName: string,
+): string | undefined {
+  // Quoted only. Short options ("In", "All", "On") occur in instructions as
+  // ordinary words; an unquoted mention goes to the decision model instead.
+  const quoted = quotedStrings(instruction).map((value) => value.trim().toLowerCase());
+  const mentioned = options.filter(
+    (option) =>
+      option.toLowerCase() !== controlName.toLowerCase() &&
+      quoted.includes(option.trim().toLowerCase()),
+  );
+  return mentioned.length === 1 ? mentioned[0] : undefined;
+}
+
+export function substituteVariables(value: string, variables?: Variables): string {
+  if (!variables) return value;
+  let output = value;
+  for (const [key, variable] of Object.entries(variables)) {
+    output = output.split(`%${key}%`).join(resolveVariableValue(variable));
+  }
+  return output;
+}
+
+/**
+ * Guards text returned by the argument-only LLM call: it must be lifted from
+ * the instruction (or be a declared variable), never invented.
+ */
+export function isGroundedText(text: string, instruction: string, variables?: Variables): boolean {
+  return groundedSpan(text, instruction, variables) !== undefined;
+}
+
+/**
+ * The instruction's OWN characters for the text the argument LLM returned.
+ * The model may re-case or re-space what it copies ("abc123" for "AbC123");
+ * what gets typed must be the user's literal input, so the match is located
+ * loosely but the span is copied verbatim from the instruction.
+ */
+export function groundedSpan(
+  text: string,
+  instruction: string,
+  variables?: Variables,
+): string | undefined {
+  const trimmed = text.trim();
+  if (!trimmed) return undefined;
+  if (isVariable(trimmed))
+    return variables && Object.hasOwn(variables, trimmed.slice(1, -1)) ? trimmed : undefined;
+  const exact = instruction.indexOf(trimmed);
+  if (exact >= 0) return trimmed;
+  const loose = new RegExp(
+    trimmed
+      .split(/\s+/)
+      .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+      .join("\\s+"),
+    "i",
+  ).exec(instruction);
+  return loose ? loose[0] : undefined;
+}
+
+/**
+ * Replaces resolved %variable% values with their placeholder in anything that
+ * leaves the process. After an earlier act typed a secret, later snapshots
+ * contain it (as the field's text), and candidate descriptions would carry it.
+ */
+export function redactor(variables: Variables | undefined): ((text: string) => string) | undefined {
+  // Values under three characters are not redacted on purpose: replacing every
+  // "a" or "no" in a page's text would destroy the candidate descriptions the decision model
+  // reads, and such values are not secrets. Documented with the flag.
+  const secrets = Object.entries(variables ?? {})
+    .map(([name, value]) => [name, resolveVariableValue(value)] as const)
+    .filter(([, value]) => value.length >= 3)
+    .sort((a, b) => b[1].length - a[1].length);
+  if (secrets.length === 0) return undefined;
+  return (text) => {
+    let output = text;
+    for (const [name, value] of secrets) {
+      for (const form of new Set([value, JSON.stringify(value).slice(1, -1)])) {
+        output = output.split(form).join(`%${name}%`);
+      }
+    }
+    return output;
+  };
+}
+
+/**
+ * Redacts every string value inside a JSON-shaped value, leaving structure and
+ * keys alone. Redacting the serialised text instead would let a value that ends
+ * in a quote, or that equals a key name, corrupt the JSON.
+ */
+export function redactDeep<T>(value: T, redact: (text: string) => string): T {
+  if (typeof value === "string") return redact(value) as T;
+  if (Array.isArray(value)) return value.map((item) => redactDeep(item, redact)) as T;
+  if (value && typeof value === "object") {
+    return Object.fromEntries(
+      Object.entries(value as Record<string, unknown>).map(([key, item]) => [
+        key,
+        redactDeep(item, redact),
+      ]),
+    ) as T;
+  }
+  return value;
+}

packages/extension/services/decisions/pageState.ts

@@ -0,0 +1,57 @@
+import { ask, round, type AskContext } from "./pick.js";
+import { pageDigest, type OutlineNode } from "./tree.js";
+import { noulAnswer } from "./typesafeClient.js";
+
+/**
+ * Page-state signals from one decision-model request over a compact page digest. Used to
+ * explain why a target could not be found (the page is a bot wall, not a
+ * decision-tree miss) and to stop paying for an LLM fallback that cannot help.
+ */
+export const PAGE_STATE_SIGNALS = {
+  access_denied:
+    "The page is an access-denied, forbidden, or 'you have been blocked' page instead of the site's real content",
+  captcha: "The page is a CAPTCHA, 'verify you are human', or bot-check interstitial",
+  login_wall:
+    "The page's main content is a sign-in or registration wall that must be passed to continue",
+  cookie_banner: "A cookie-consent or privacy-choices banner or dialog is present on the page",
+  error_page:
+    "The page is an error page such as 404 not found, 500 server error, or 'something went wrong'",
+  empty_or_loading: "The page is essentially empty or still showing only a loading indicator",
+} as const;
+
+export type PageSignal = keyof typeof PAGE_STATE_SIGNALS;
+export type PageState = Record<PageSignal, number>;
+
+/** A target can never be found on these, whoever looks for it. */
+const BLOCKING: PageSignal[] = ["access_denied", "captcha"];
+const BLOCKING_THRESHOLD = 0.9;
+
+export async function readPageState(
+  ctx: AskContext,
+  nodes: OutlineNode[],
+  url: string,
+): Promise<PageState> {
+  const response = await ask(
+    ctx,
+    "page_state",
+    { url, ...pageDigest(nodes) },
+    Object.fromEntries(
+      Object.entries(PAGE_STATE_SIGNALS).map(([key, instructions]) => [
+        key,
+        { type: "noul" as const, instructions },
+      ]),
+    ),
+  );
+  const state = Object.fromEntries(
+    (Object.keys(PAGE_STATE_SIGNALS) as PageSignal[]).map((key) => [
+      key,
+      round(noulAnswer(response, key).noul),
+    ]),
+  ) as PageState;
+  ctx.trace[ctx.trace.length - 1]!.state = state;
+  return state;
+}
+
+export function blockingSignal(state: PageState): PageSignal | undefined {
+  return BLOCKING.find((signal) => state[signal] >= BLOCKING_THRESHOLD);
+}