feat(extension): experimental Jev WebMCP tool selection in act()

.changeset/jev-webmcp-tools.md

@@ -0,0 +1,8 @@
+---
+"@browserbasehq/stagehand-extension": patch
+"@browserbasehq/stagehand-go": patch
+"@browserbasehq/stagehand": patch
+"@browserbasehq/stagehand-python": patch
+---
+
+experimental Jev path: opt-in `tools` lets `act()` invoke a WebMCP tool the page registered when Jev is confident the tool fulfils the instruction

packages/evals/initStagehand.ts

@@ -107,6 +107,7 @@ export async function initStagehand({
       ...(process.env.EVAL_JEV_ACT_LLM_FALLBACK === "0" ? { llmFallback: false } : {}),
       ...(process.env.EVAL_JEV_ARG_LLM === "0" ? { argumentLlm: false } : {}),
       ...(process.env.EVAL_JEV_OBSERVE === "1" ? { observe: true } : {}),
+      ...(process.env.EVAL_JEV_TOOLS === "1" ? { tools: true } : {}),
       ...(process.env.EVAL_JEV_EXTRACT === "pick" || process.env.EVAL_JEV_EXTRACT === "judge"
         ? { extract: process.env.EVAL_JEV_EXTRACT }
         : {}),

packages/extension/inference.ts

@@ -306,3 +306,83 @@ export async function actTextArgument(params: {
     inference_time_ms: result.durationMs,
   };
 }
+
+/**
+ * Argument-only inference for a WebMCP tool Jev already chose: the prompt is
+ * one tool, not the catalog, and the tool's own input schema shapes the answer.
+ */
+export async function toolArguments(params: {
+  instruction: string;
+  tool: { name: string; description: string; inputSchema?: Record<string, unknown> };
+  variableNames: string[];
+  generate: GenerateLlm;
+}): Promise<{
+  input: Record<string, unknown> | null;
+  prompt_tokens: number;
+  completion_tokens: number;
+  reasoning_tokens: number;
+  cached_input_tokens: number;
+  inference_time_ms: number;
+}> {
+  const startedAt = Date.now();
+  const variables =
+    params.variableNames.length > 0
+      ? ` Declared variables: ${params.variableNames.map((name) => `%${name}%`).join(", ")}; when one stands for a value, return it as written including the percent signs.`
+      : "";
+  const request = (schema: Record<string, unknown>, extra: string) =>
+    params.generate({
+      systemPrompt: `You fill in the input of one tool from a user's request. Use only values the request states or clearly implies; leave optional parameters out otherwise.${variables}${extra}`,
+      messages: [
+        {
+          role: "user",
+          content: {
+            type: "text",
+            text: `tool: ${JSON.stringify(params.tool)}\nrequest: ${params.instruction}`,
+          },
+        },
+      ],
+      responseFormat: { type: "json_schema", name: "ToolInput", schema: z.json().parse(schema) },
+    });
+  // A site's schema is whatever the site wrote; providers with strict
+  // structured output reject some of them (optional properties, keywords they
+  // do not know). Then the input travels as a JSON string instead.
+  let content: unknown;
+  let response: Awaited<ReturnType<GenerateLlm>>;
+  const properties = Object.keys((params.tool.inputSchema?.properties as object | undefined) ?? {});
+  const required = params.tool.inputSchema?.required;
+  // Optional properties are the common rejection; do not pay a failed call to find out.
+  const strictFriendly =
+    Array.isArray(required) && properties.every((name) => required.includes(name));
+  try {
+    if (!strictFriendly) throw new Error("schema has optional properties");
+    response = await request({ type: "object", ...params.tool.inputSchema }, "");
+    content = response.outputFormat === "json_schema" ? response.structuredContent : null;
+  } catch {
+    response = await request(
+      {
+        type: "object",
+        properties: { input_json: { type: "string" } },
+        required: ["input_json"],
+        additionalProperties: false,
+      },
+      " Return the tool's input object serialised as JSON in input_json.",
+    );
+    const wrapped = response.outputFormat === "json_schema" ? response.structuredContent : null;
+    try {
+      content = JSON.parse((wrapped as { input_json?: string } | null)?.input_json ?? "null");
+    } catch {
+      content = null;
+    }
+  }
+  return {
+    input:
+      content && typeof content === "object" && !Array.isArray(content)
+        ? (content as Record<string, unknown>)
+        : null,
+    prompt_tokens: response.usage?.inputTokens ?? 0,
+    completion_tokens: response.usage?.outputTokens ?? 0,
+    reasoning_tokens: response.usage?.reasoningTokens ?? 0,
+    cached_input_tokens: response.usage?.cachedInputTokens ?? 0,
+    inference_time_ms: Date.now() - startedAt,
+  };
+}

packages/extension/services/actService.ts

@@ -29,14 +29,18 @@ import * as cacheService from "./cacheService.js";
 import { checkCachedAction } from "./jevAct/cacheCheck.js";
 import { runJevActPipeline, type JevActConfig, type JevActOutcome } from "./jevAct/pipeline.js";
 import { redactor } from "./jevAct/args.js";
+import type { JevToolDeps } from "./jevAct/toolAct.js";
 import { focusOutline, parseOutline } from "./jevAct/tree.js";
+import type { JsonValue } from "./jevAct/typesafeClient.js";
 import * as llmService from "./llmService.js";
 import { disabledCacheMetadata, zeroStagehandResultUsage } from "./resultUsage.js";
 
 // Set high on purpose: on ordinary pages the shortlist cost accuracy (the LLM
 // lost the context that disambiguates), so it is reserved for huge trees where
 // the full-page call is slow and expensive.
 const FOCUS_MIN_TREE_CHARS = 120_000;
+/** Tools registered at load are reported within the listing's quiet window. */
+const LIST_TOOLS_TIMEOUT_MS = 300;
 
 type ActInferenceResponse = Awaited<ReturnType<typeof inference.act>>;
 type ActInferenceElement = NonNullable<ActInferenceResponse["element"]>;
@@ -121,9 +125,41 @@ export async function act({
     focusLocator: options?.locator,
     ignoreLocators: options?.ignoreLocators,
   };
+  // Listed while the DOM settles, so knowing the page's tools costs the act
+  // nothing. A scoped act is about that element; tools are page-level.
+  const webmcp: JevToolDeps | undefined =
+    jevAct?.tools && jevAct.enabled !== false && !options?.locator
+      ? {
+          page,
+          // Browsers without the WebMCP domain reject the enable call.
+          tools: page.listWebMCPTools({ timeout: LIST_TOOLS_TIMEOUT_MS }).catch(() => []),
+          fillArguments: async (tool) => {
+            const response = await inference.toolArguments({
+              instruction,
+              tool,
+              variableNames: Object.keys(variables ?? {}),
+              generate: (input) =>
+                llmService.generate(
+                  context.model,
+                  input,
+                  context.clientLLMGenerate,
+                  context.gateway,
+                ),
+            });
+            recordUsage({ ...response, element: null, twoStep: false });
+            const required = Array.isArray(tool.inputSchema?.required)
+              ? tool.inputSchema.required
+              : [];
+            const input = response.input;
+            return input && required.every((name) => typeof name === "string" && name in input)
+              ? (input as Record<string, JsonValue>)
+              : null;
+          },
+        }
+      : undefined;
   await waitForDomNetworkQuiet(page.mainFrame(), logger, domSettleTimeoutMs);
   ensureTimeRemaining();
-  let actPath: "llm" | "jev" | "jev+arg-llm" | "jev+llm" = "llm";
+  let actPath: "llm" | "jev" | "jev+arg-llm" | "jev+llm" | "jev-tool" | "jev-tool+arg-llm" = "llm";
   let usedArgumentLlm = false;
   // Jev's shortlist when it narrowed the choice but could not commit.
   let jevFocusIds: string[] = [];
@@ -193,6 +229,7 @@ export async function act({
         snapshotOptions,
         ensureTimeRemaining,
         openPageCount,
+        ...(webmcp ? { webmcp } : {}),
         extractText: async (text) => {
           const response = await inference.actTextArgument({
             instruction: text,
@@ -220,6 +257,8 @@ export async function act({
       if (outcome.kind === "done") {
         jevNoCache = outcome.noCache === true;
         if (usedArgumentLlm) actPath = "jev+arg-llm";
+        if (outcome.viaTool)
+          actPath = outcome.viaTool.argumentLlm ? "jev-tool+arg-llm" : "jev-tool";
         return actResult(outcome.result, operationUsage);
       }
       actPath = "jev+llm";

packages/extension/services/jevAct/README.md

@@ -49,9 +49,45 @@ replayed, so a selector that now resolves to a different control is re-inferred
 | `cacheCheck`    | `false`    | Before each cached action is replayed, one Jev yes/no checks that its selector still points at a matching element; stale ones are re-inferred. Adds a snapshot per cached action, and a request when the selector still resolves in it.                                                                                                                                                             |
 | `extract`       | `"off"`    | `"judge"`: Jev's yes/no replaces extract()'s completion LLM call. `"pick"`: Jev picks the elements holding each scalar or list field's value and code copies their text; booleans and enums are judged directly; schemas the planner cannot map, unresolved required fields, or a failed completion gate send the whole extraction to the LLM. **Both send page or extracted content to TypeSafe.** |
 | `observe`       | `false`    | Resolve `observe()` through Jev first. "Find all" is answered exhaustively or handed to the LLM (over 600 candidates; over 400 elements with no instruction), never truncated.                                                                                                                                                                                                                      |
+| `tools`         | `false`    | Let `act()` invoke a WebMCP tool the page registered when Jev is sure the tool is the request (see below). Sends tool names, descriptions and parameter names to TypeSafe.                                                                                                                                                                                                                          |
 | `retryNoEffect` | `false`    | Click the runner-up when an ambiguous click provably changed nothing. Off: effects the outline cannot show (aria-pressed, copy, play) look like "nothing". Never cached.                                                                                                                                                                                                                            |
 | `focusFallback` | `false`    | On trees over 120K chars, show the LLM Jev's shortlist first. Off: it found the target in a minority of firings and cost accuracy on ordinary pages.                                                                                                                                                                                                                                                |
 
+## WebMCP tools (`tools: true`)
+
+The page's tools are listed while `act()` waits for the DOM to settle, and the tool questions ride
+in the intent request that every act already makes, so a page without tools adds no question and a
+page with tools adds no round trip. Alongside "which tool" (with a "none" option) a guard asks
+whether the instruction names a control: "click the Add to cart button" always takes the element
+path, even when `add_to_cart` exists. A tool is used only at ≥ 0.8 with "none" ≤ 0.2.
+
+Arguments are picked by Jev as spans of the instruction (enums and booleans as choices) and accepted
+only when every parameter, stated or not, is ≥ 0.8. The argument questions of the two tools whose
+names and descriptions share most words with the instruction ride in the same request, so a
+confident tool call is usually one request (~300 ms); another winner costs one more. Values that
+are not spans (dates to normalise, lists, nested objects) go to an argument-only LLM call that sees
+just that tool, with its input schema as the response format; when the schema alone shows the
+likely tool will need it, that call starts alongside the Jev request. Any doubt means the ordinary
+act path continues from the intent it already has. Once a tool has been invoked the act is over,
+success or error: it never also clicks through the UI. Tool acts are not cached.
+
+```ts
+// TypeScript SDK; Chrome needs its WebMCP features on, which localBrowser.launch() does.
+process.env.STAGEHAND_EXPERIMENTAL_JEV_ACT = JSON.stringify({ apiKey, tools: true });
+const stagehand = await Stagehand.create({ browser, model });
+await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/webmcp-test/");
+
+await stagehand.act("add 19 and 23 together");
+// → { method: "webmcp", selector: "webmcp:calculateSum", arguments: ['{"a":19,"b":23}'] }
+//   message: 'Invoked WebMCP tool calculateSum: {"a":19,"b":23,"sum":42}'   (one Jev request, no LLM call)
+
+await stagehand.act("click the Calculate button");
+// → names a control, so the ordinary element path runs
+```
+
+On 380 LLM-written requests over 166 tools harvested from six live sites: tool choice answered by
+Jev for 79% of requests at 99% precision; arguments filled by Jev for 69% of calls at 98% precision.
+
 ## What leaves the process
 
 Sent to TypeSafe: the instruction, candidate descriptions built from the accessibility outline

packages/extension/services/jevAct/pipeline.ts

@@ -18,6 +18,8 @@ import {
   redactor,
   substituteVariables,
 } from "./args.js";
+import { invokeTool, type JevToolDeps } from "./toolAct.js";
+import { readToolDecision, toolQuestions } from "./tools.js";
 import { blockingSignal, readPageState } from "./pageState.js";
 import {
   NONE,
@@ -95,6 +97,11 @@ export type JevActConfig = JevConfig & {
   extract?: "off" | "judge" | "pick";
   /** Resolve observe() through Jev first. Default false. */
   observe?: boolean;
+  /**
+   * Let act() invoke a WebMCP tool the page registered when Jev is sure the
+   * tool is the request. Sends tool names and descriptions to TypeSafe. Default false.
+   */
+  tools?: boolean;
 };
 
 export type JevActDeps = {
@@ -112,13 +119,17 @@ export type JevActDeps = {
    */
   extractText?: (instruction: string) => Promise<string | null>;
   takeAction: (action: Action) => Promise<ActResultData>;
+  /** Present when `tools` is on and the act is not scoped to a locator. */
+  webmcp?: JevToolDeps;
 };
 
 export type JevActOutcome =
   | {
       kind: "done";
       result: ActResultData;
       /** Must not be written to the act cache. */ noCache?: boolean;
+      /** The act was a WebMCP tool call; `argumentLlm` when its input came from the LLM. */
+      viaTool?: { argumentLlm: boolean };
     }
   | {
       kind: "fallback";
@@ -271,6 +282,16 @@ async function decideAndAct(
   // Intent fan-out: every question that only needs the instruction rides in
   // one request, so press / not-an-action / whole-page scroll finish here.
   const fillValues = fillValueCandidates(deps.instruction, deps.variables);
+  // Tool choice only needs the instruction too, so it costs no round trip of
+  // its own, and a page without tools adds no question at all.
+  const tools = deps.webmcp ? await deps.webmcp.tools.catch(() => []) : [];
+  const asked = tools.length > 0 ? toolQuestions(deps.instruction, tools) : undefined;
+  // Known from the schema alone: if this tool wins, only the LLM can fill it.
+  const speculative = asked?.needsArgumentLlm;
+  const speculativeInput =
+    speculative && deps.webmcp?.fillArguments && config.argumentLlm !== false
+      ? deps.webmcp.fillArguments(speculative).catch(() => null)
+      : undefined;
   const intent = await ask(
     ctx,
     "intent",
@@ -348,10 +369,52 @@ async function decideAndAct(
           other: "Some other key, or the instruction is not a key press",
         },
       },
+      ...asked?.questions,
     },
   );
+  const intentEntry = trace[trace.length - 1]!;
+  if (asked && deps.webmcp) {
+    const decision = await readToolDecision(ctx, intent, asked, intentEntry);
+    if (decision.kind === "tool") {
+      let input = decision.input;
+      let argumentLlm = false;
+      if (!input && deps.webmcp.fillArguments && config.argumentLlm !== false) {
+        argumentLlm = true;
+        const started = performance.now();
+        input =
+          (await (decision.tool === speculative && speculativeInput
+            ? speculativeInput
+            : deps.webmcp.fillArguments(decision.tool).catch(() => null))) ?? undefined;
+        trace.push({
+          node: "tool_arguments_llm",
+          ms: Math.round(performance.now() - started),
+          speculative: decision.tool === speculative,
+        });
+      }
+      if (input) {
+        deps.ensureTimeRemaining();
+        const result = await invokeTool(
+          deps.webmcp,
+          deps.instruction,
+          deps.variables,
+          decision.tool,
+          input,
+          trace,
+        );
+        // Replay only knows element actions.
+        return { kind: "done", result, noCache: true, viaTool: { argumentLlm } };
+      }
+      intentEntry.tool_skip = "arguments_not_filled";
+    } else {
+      intentEntry.tool_skip = decision.reason;
+    }
+  }
   const family = resolveFamily(choiceAnswer(intent, "family"), ctx.threshold);
-  annotate(trace, { choice: family.choice, confidence: family.confidence, top: family.top });
+  Object.assign(intentEntry, {
+    choice: family.choice,
+    confidence: family.confidence,
+    top: family.top,
+  });
   if (family.confidence < ctx.threshold) return fallback(`intent_low_confidence:${family.top}`);
 
   if (family.choice === "not_an_action") {