feat(extension): supplement Jev shortlist with BM25 candidate

packages/extension/services/jevAct/args.ts

@@ -0,0 +1,161 @@
+import type { Variables } from "@browserbasehq/stagehand-protocol/types";
+import { resolveVariableValue } from "../../handlers/handlerUtils/variables.js";
+
+/**
+ * Deterministic argument parsing for the Jev act pipeline. Jev cannot generate
+ * text, so every argument is either parsed from the instruction here or chosen
+ * by Jev from a closed set these helpers produce.
+ */
+
+const VARIABLE = /^%[^%\s]+%$/;
+
+export function isVariable(value: string): boolean {
+  return VARIABLE.test(value);
+}
+
+export function isLoneVariable(values: string[]): boolean {
+  return values.length === 1 && isVariable(values[0]!);
+}
+
+/** Quoted strings and declared %variables%: the closed set a fill value can come from. */
+export function fillValueCandidates(instruction: string, variables?: Variables): string[] {
+  const values = new Set<string>();
+  for (const match of instruction.matchAll(/%([^%\s]+)%/g)) {
+    if (variables && Object.hasOwn(variables, match[1]!)) values.add(match[0]);
+  }
+  for (const quoted of quotedStrings(instruction)) values.add(quoted);
+  return [...values];
+}
+
+export function quotedStrings(instruction: string): string[] {
+  const values: string[] = [];
+  // Single quotes must sit at word boundaries so "user's" is not an opening quote.
+  for (const match of instruction.matchAll(
+    /(?<![\w])'([^'\n]+)'(?![\w])|"([^"\n]+)"|“([^”\n]+)”|‘([^’\n]+)’/g,
+  )) {
+    values.push((match[1] ?? match[2] ?? match[3] ?? match[4])!);
+  }
+  return values;
+}
+
+export function parsePercent(instruction: string): string | undefined {
+  const explicit = /(\d+(?:\.\d+)?)\s*%/.exec(instruction);
+  if (explicit) return `${explicit[1]}%`;
+  // The documented fractional form: "scroll to 0.75" means 75%.
+  const fraction = /(?<![\d.])(0?\.\d+|1\.0+|1)(?![\d.%])/.exec(instruction);
+  if (fraction && /\b(scroll|down|up|to)\b/i.test(instruction)) {
+    return `${Math.round(parseFloat(fraction[1]!) * 100)}%`;
+  }
+  if (/\bhalf ?way\b|\bmiddle\b/i.test(instruction)) return "50%";
+  if (/\bbottom\b/i.test(instruction)) return "100%";
+  if (/\btop\b/i.test(instruction)) return "0%";
+  return undefined;
+}
+
+const SPECIAL_KEYS: Record<string, string> = {
+  enter: "Enter",
+  return: "Enter",
+  tab: "Tab",
+  escape: "Escape",
+  esc: "Escape",
+  space: "Space",
+  backspace: "Backspace",
+  delete: "Delete",
+};
+
+export function parseKey(instruction: string): string | undefined {
+  const match = /\bpress(?:\s+the)?\s+["']?([a-z0-9]+)["']?/i.exec(instruction);
+  if (!match) return undefined;
+  const key = match[1]!;
+  return SPECIAL_KEYS[key.toLowerCase()] ?? (key.length === 1 ? key : undefined);
+}
+
+/**
+ * Deterministic only when exactly one real option appears in the instruction.
+ * A quoted string may name the dropdown, and placeholders often repeat that
+ * name, so options equal to the control's own name never count.
+ */
+export function matchOption(
+  options: string[],
+  instruction: string,
+  controlName: string,
+): string | undefined {
+  // Quoted only. Short options ("In", "All", "On") occur in instructions as
+  // ordinary words; an unquoted mention goes to Jev instead.
+  const quoted = quotedStrings(instruction).map((value) => value.trim().toLowerCase());
+  const mentioned = options.filter(
+    (option) =>
+      option.toLowerCase() !== controlName.toLowerCase() &&
+      quoted.includes(option.trim().toLowerCase()),
+  );
+  return mentioned.length === 1 ? mentioned[0] : undefined;
+}
+
+export function substituteVariables(value: string, variables?: Variables): string {
+  if (!variables) return value;
+  let output = value;
+  for (const [key, variable] of Object.entries(variables)) {
+    output = output.split(`%${key}%`).join(resolveVariableValue(variable));
+  }
+  return output;
+}
+
+/**
+ * Guards text returned by the argument-only LLM call: it must be lifted from
+ * the instruction (or be a declared variable), never invented.
+ */
+export function isGroundedText(text: string, instruction: string, variables?: Variables): boolean {
+  return groundedSpan(text, instruction, variables) !== undefined;
+}
+
+/**
+ * The instruction's OWN characters for the text the argument LLM returned.
+ * The model may re-case or re-space what it copies ("abc123" for "AbC123");
+ * what gets typed must be the user's literal input, so the match is located
+ * loosely but the span is copied verbatim from the instruction.
+ */
+export function groundedSpan(
+  text: string,
+  instruction: string,
+  variables?: Variables,
+): string | undefined {
+  const trimmed = text.trim();
+  if (!trimmed) return undefined;
+  if (isVariable(trimmed))
+    return variables && Object.hasOwn(variables, trimmed.slice(1, -1)) ? trimmed : undefined;
+  const exact = instruction.indexOf(trimmed);
+  if (exact >= 0) return trimmed;
+  const loose = new RegExp(
+    trimmed
+      .split(/\s+/)
+      .map((part) => part.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"))
+      .join("\\s+"),
+    "i",
+  ).exec(instruction);
+  return loose ? loose[0] : undefined;
+}
+
+/**
+ * Replaces resolved %variable% values with their placeholder in anything that
+ * leaves the process. After an earlier act typed a secret, later snapshots
+ * contain it (as the field's text), and candidate descriptions would carry it.
+ */
+export function redactor(variables: Variables | undefined): ((text: string) => string) | undefined {
+  // Values under three characters are not redacted on purpose: replacing every
+  // "a" or "no" in a page's text would destroy the candidate descriptions Jev
+  // reads, and such values are not secrets. Documented with the flag.
+  const secrets = Object.entries(variables ?? {})
+    .map(([name, value]) => [name, resolveVariableValue(value)] as const)
+    .filter(([, value]) => value.length >= 3)
+    .sort((a, b) => b[1].length - a[1].length);
+  if (secrets.length === 0) return undefined;
+  return (text) => {
+    let output = text;
+    for (const [name, value] of secrets) {
+      for (const form of new Set([value, JSON.stringify(value).slice(1, -1)])) {
+        output = output.split(form).join(`%${name}%`);
+      }
+    }
+    return output;
+  };
+}

packages/extension/services/jevAct/bm25.ts

@@ -0,0 +1,78 @@
+import {
+  describeCandidate,
+  flattenText,
+  scoreCandidates,
+  tokens,
+  type OutlineNode,
+} from "./tree.js";
+import type { JsonValue } from "./typesafeClient.js";
+
+/** BM25 with the current ranker's name, text, and table weights retained. */
+export function scoreCandidatesBm25(
+  nodes: OutlineNode[],
+  candidates: OutlineNode[],
+  instruction: string,
+): Map<string, number> {
+  const query = new Set(tokens(instruction));
+  const documents = candidates.map((candidate) => {
+    const description = describeCandidate(nodes, candidate);
+    const fields: Array<[JsonValue | undefined, number]> = [
+      [description.name, 3],
+      [description.text, 2],
+      [description.within, 1],
+      [description.under_heading, 1],
+      [description.near_text, 1],
+      [description.group_text, 1],
+      [description.table, 2],
+    ];
+    const frequencies = new Map<string, number>();
+    let length = 0;
+    for (const [value, weight] of fields) {
+      for (const word of tokens(value === undefined ? "" : flattenText(value))) {
+        length += weight;
+        if (query.has(word)) frequencies.set(word, (frequencies.get(word) ?? 0) + weight);
+      }
+    }
+    return { id: candidate.id, frequencies, length };
+  });
+  const averageLength =
+    documents.reduce((sum, document) => sum + document.length, 0) / (documents.length || 1) || 1;
+  const documentFrequency = new Map<string, number>();
+  for (const document of documents) {
+    for (const word of document.frequencies.keys())
+      documentFrequency.set(word, (documentFrequency.get(word) ?? 0) + 1);
+  }
+  const scores = new Map<string, number>();
+  const n = documents.length;
+  for (const document of documents) {
+    let score = 0;
+    for (const [word, frequency] of document.frequencies) {
+      const df = documentFrequency.get(word)!;
+      const idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+      const denominator = frequency + 1.2 * (0.65 + 0.35 * (document.length / averageLength));
+      score += idf * ((frequency * 2.2) / denominator);
+    }
+    scores.set(document.id, score);
+  }
+  return scores;
+}
+
+/** Keep every candidate from the existing top 30, then add BM25's best outsider. */
+export function shortlistWithBm25(
+  nodes: OutlineNode[],
+  candidates: OutlineNode[],
+  instruction: string,
+  keep = 30,
+): OutlineNode[] {
+  const select = (scores: Map<string, number>, limit: number) =>
+    candidates
+      .filter((candidate) => (scores.get(candidate.id) ?? 0) > 0)
+      .sort((a, b) => (scores.get(b.id) ?? 0) - (scores.get(a.id) ?? 0) || a.index - b.index)
+      .slice(0, limit);
+  const baseline = select(scoreCandidates(nodes, candidates, instruction), keep);
+  if (baseline.length < keep) return baseline.sort((a, b) => a.index - b.index);
+  const selected = new Set(baseline.map((candidate) => candidate.id));
+  const bestBm25 = select(scoreCandidatesBm25(nodes, candidates, instruction), 1)[0];
+  if (bestBm25 && !selected.has(bestBm25.id)) baseline.push(bestBm25);
+  return baseline.sort((a, b) => a.index - b.index);
+}

packages/extension/services/jevAct/pageState.ts

@@ -0,0 +1,57 @@
+import { ask, round, type AskContext } from "./pick.js";
+import { pageDigest, type OutlineNode } from "./tree.js";
+import { noulAnswer } from "./typesafeClient.js";
+
+/**
+ * Page-state signals from one Jev request over a compact page digest. Used to
+ * explain why a target could not be found (the page is a bot wall, not a
+ * decision-tree miss) and to stop paying for an LLM fallback that cannot help.
+ */
+export const PAGE_STATE_SIGNALS = {
+  access_denied:
+    "The page is an access-denied, forbidden, or 'you have been blocked' page instead of the site's real content",
+  captcha: "The page is a CAPTCHA, 'verify you are human', or bot-check interstitial",
+  login_wall:
+    "The page's main content is a sign-in or registration wall that must be passed to continue",
+  cookie_banner: "A cookie-consent or privacy-choices banner or dialog is present on the page",
+  error_page:
+    "The page is an error page such as 404 not found, 500 server error, or 'something went wrong'",
+  empty_or_loading: "The page is essentially empty or still showing only a loading indicator",
+} as const;
+
+export type PageSignal = keyof typeof PAGE_STATE_SIGNALS;
+export type PageState = Record<PageSignal, number>;
+
+/** A target can never be found on these, whoever looks for it. */
+const BLOCKING: PageSignal[] = ["access_denied", "captcha"];
+const BLOCKING_THRESHOLD = 0.9;
+
+export async function readPageState(
+  ctx: AskContext,
+  nodes: OutlineNode[],
+  url: string,
+): Promise<PageState> {
+  const response = await ask(
+    ctx,
+    "page_state",
+    { url, ...pageDigest(nodes) },
+    Object.fromEntries(
+      Object.entries(PAGE_STATE_SIGNALS).map(([key, instructions]) => [
+        key,
+        { type: "noul" as const, instructions },
+      ]),
+    ),
+  );
+  const state = Object.fromEntries(
+    (Object.keys(PAGE_STATE_SIGNALS) as PageSignal[]).map((key) => [
+      key,
+      round(noulAnswer(response, key).noul),
+    ]),
+  ) as PageState;
+  ctx.trace[ctx.trace.length - 1]!.state = state;
+  return state;
+}
+
+export function blockingSignal(state: PageState): PageSignal | undefined {
+  return BLOCKING.find((signal) => state[signal] >= BLOCKING_THRESHOLD);
+}