diff --git a/.changeset/jev-webmcp-tools.md b/.changeset/jev-webmcp-tools.md new file mode 100644 index 000000000..c55155589 --- /dev/null +++ b/.changeset/jev-webmcp-tools.md @@ -0,0 +1,8 @@ +--- +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +"@browserbasehq/stagehand": patch +"@browserbasehq/stagehand-python": patch +--- + +experimental Jev path: opt-in `tools` lets `act()` invoke a WebMCP tool the page registered when Jev is confident the tool fulfils the instruction diff --git a/packages/evals/initStagehand.ts b/packages/evals/initStagehand.ts index 10708661d..62f6ac5c1 100644 --- a/packages/evals/initStagehand.ts +++ b/packages/evals/initStagehand.ts @@ -107,6 +107,7 @@ export async function initStagehand({ ...(process.env.EVAL_JEV_ACT_LLM_FALLBACK === "0" ? { llmFallback: false } : {}), ...(process.env.EVAL_JEV_ARG_LLM === "0" ? { argumentLlm: false } : {}), ...(process.env.EVAL_JEV_OBSERVE === "1" ? { observe: true } : {}), + ...(process.env.EVAL_JEV_TOOLS === "1" ? { tools: true } : {}), ...(process.env.EVAL_JEV_EXTRACT === "pick" || process.env.EVAL_JEV_EXTRACT === "judge" ? { extract: process.env.EVAL_JEV_EXTRACT } : {}), diff --git a/packages/extension/inference.ts b/packages/extension/inference.ts index f07a729e5..92a44a080 100644 --- a/packages/extension/inference.ts +++ b/packages/extension/inference.ts @@ -306,3 +306,83 @@ export async function actTextArgument(params: { inference_time_ms: result.durationMs, }; } + +/** + * Argument-only inference for a WebMCP tool Jev already chose: the prompt is + * one tool, not the catalog, and the tool's own input schema shapes the answer. + */ +export async function toolArguments(params: { + instruction: string; + tool: { name: string; description: string; inputSchema?: Record }; + variableNames: string[]; + generate: GenerateLlm; +}): Promise<{ + input: Record | null; + prompt_tokens: number; + completion_tokens: number; + reasoning_tokens: number; + cached_input_tokens: number; + inference_time_ms: number; +}> { + const startedAt = Date.now(); + const variables = + params.variableNames.length > 0 + ? ` Declared variables: ${params.variableNames.map((name) => `%${name}%`).join(", ")}; when one stands for a value, return it as written including the percent signs.` + : ""; + const request = (schema: Record, extra: string) => + params.generate({ + systemPrompt: `You fill in the input of one tool from a user's request. Use only values the request states or clearly implies; leave optional parameters out otherwise.${variables}${extra}`, + messages: [ + { + role: "user", + content: { + type: "text", + text: `tool: ${JSON.stringify(params.tool)}\nrequest: ${params.instruction}`, + }, + }, + ], + responseFormat: { type: "json_schema", name: "ToolInput", schema: z.json().parse(schema) }, + }); + // A site's schema is whatever the site wrote; providers with strict + // structured output reject some of them (optional properties, keywords they + // do not know). Then the input travels as a JSON string instead. + let content: unknown; + let response: Awaited>; + const properties = Object.keys((params.tool.inputSchema?.properties as object | undefined) ?? {}); + const required = params.tool.inputSchema?.required; + // Optional properties are the common rejection; do not pay a failed call to find out. + const strictFriendly = + Array.isArray(required) && properties.every((name) => required.includes(name)); + try { + if (!strictFriendly) throw new Error("schema has optional properties"); + response = await request({ type: "object", ...params.tool.inputSchema }, ""); + content = response.outputFormat === "json_schema" ? response.structuredContent : null; + } catch { + response = await request( + { + type: "object", + properties: { input_json: { type: "string" } }, + required: ["input_json"], + additionalProperties: false, + }, + " Return the tool's input object serialised as JSON in input_json.", + ); + const wrapped = response.outputFormat === "json_schema" ? response.structuredContent : null; + try { + content = JSON.parse((wrapped as { input_json?: string } | null)?.input_json ?? "null"); + } catch { + content = null; + } + } + return { + input: + content && typeof content === "object" && !Array.isArray(content) + ? (content as Record) + : null, + prompt_tokens: response.usage?.inputTokens ?? 0, + completion_tokens: response.usage?.outputTokens ?? 0, + reasoning_tokens: response.usage?.reasoningTokens ?? 0, + cached_input_tokens: response.usage?.cachedInputTokens ?? 0, + inference_time_ms: Date.now() - startedAt, + }; +} diff --git a/packages/extension/services/actService.ts b/packages/extension/services/actService.ts index 149b5eeee..fdbf22cd8 100644 --- a/packages/extension/services/actService.ts +++ b/packages/extension/services/actService.ts @@ -29,7 +29,9 @@ import * as cacheService from "./cacheService.js"; import { checkCachedAction } from "./jevAct/cacheCheck.js"; import { runJevActPipeline, type JevActConfig, type JevActOutcome } from "./jevAct/pipeline.js"; import { redactor } from "./jevAct/args.js"; +import type { JevToolDeps } from "./jevAct/toolAct.js"; import { focusOutline, parseOutline } from "./jevAct/tree.js"; +import type { JsonValue } from "./jevAct/typesafeClient.js"; import * as llmService from "./llmService.js"; import { disabledCacheMetadata, zeroStagehandResultUsage } from "./resultUsage.js"; @@ -37,6 +39,8 @@ import { disabledCacheMetadata, zeroStagehandResultUsage } from "./resultUsage.j // lost the context that disambiguates), so it is reserved for huge trees where // the full-page call is slow and expensive. const FOCUS_MIN_TREE_CHARS = 120_000; +/** Tools registered at load are reported within the listing's quiet window. */ +const LIST_TOOLS_TIMEOUT_MS = 300; type ActInferenceResponse = Awaited>; type ActInferenceElement = NonNullable; @@ -121,9 +125,41 @@ export async function act({ focusLocator: options?.locator, ignoreLocators: options?.ignoreLocators, }; + // Listed while the DOM settles, so knowing the page's tools costs the act + // nothing. A scoped act is about that element; tools are page-level. + const webmcp: JevToolDeps | undefined = + jevAct?.tools && jevAct.enabled !== false && !options?.locator + ? { + page, + // Browsers without the WebMCP domain reject the enable call. + tools: page.listWebMCPTools({ timeout: LIST_TOOLS_TIMEOUT_MS }).catch(() => []), + fillArguments: async (tool) => { + const response = await inference.toolArguments({ + instruction, + tool, + variableNames: Object.keys(variables ?? {}), + generate: (input) => + llmService.generate( + context.model, + input, + context.clientLLMGenerate, + context.gateway, + ), + }); + recordUsage({ ...response, element: null, twoStep: false }); + const required = Array.isArray(tool.inputSchema?.required) + ? tool.inputSchema.required + : []; + const input = response.input; + return input && required.every((name) => typeof name === "string" && name in input) + ? (input as Record) + : null; + }, + } + : undefined; await waitForDomNetworkQuiet(page.mainFrame(), logger, domSettleTimeoutMs); ensureTimeRemaining(); - let actPath: "llm" | "jev" | "jev+arg-llm" | "jev+llm" = "llm"; + let actPath: "llm" | "jev" | "jev+arg-llm" | "jev+llm" | "jev-tool" | "jev-tool+arg-llm" = "llm"; let usedArgumentLlm = false; // Jev's shortlist when it narrowed the choice but could not commit. let jevFocusIds: string[] = []; @@ -193,6 +229,7 @@ export async function act({ snapshotOptions, ensureTimeRemaining, openPageCount, + ...(webmcp ? { webmcp } : {}), extractText: async (text) => { const response = await inference.actTextArgument({ instruction: text, @@ -220,6 +257,8 @@ export async function act({ if (outcome.kind === "done") { jevNoCache = outcome.noCache === true; if (usedArgumentLlm) actPath = "jev+arg-llm"; + if (outcome.viaTool) + actPath = outcome.viaTool.argumentLlm ? "jev-tool+arg-llm" : "jev-tool"; return actResult(outcome.result, operationUsage); } actPath = "jev+llm"; diff --git a/packages/extension/services/jevAct/README.md b/packages/extension/services/jevAct/README.md index bab5332f3..08f504a27 100644 --- a/packages/extension/services/jevAct/README.md +++ b/packages/extension/services/jevAct/README.md @@ -49,9 +49,45 @@ replayed, so a selector that now resolves to a different control is re-inferred | `cacheCheck` | `false` | Before each cached action is replayed, one Jev yes/no checks that its selector still points at a matching element; stale ones are re-inferred. Adds a snapshot per cached action, and a request when the selector still resolves in it. | | `extract` | `"off"` | `"judge"`: Jev's yes/no replaces extract()'s completion LLM call. `"pick"`: Jev picks the elements holding each scalar or list field's value and code copies their text; booleans and enums are judged directly; schemas the planner cannot map, unresolved required fields, or a failed completion gate send the whole extraction to the LLM. **Both send page or extracted content to TypeSafe.** | | `observe` | `false` | Resolve `observe()` through Jev first. "Find all" is answered exhaustively or handed to the LLM (over 600 candidates; over 400 elements with no instruction), never truncated. | +| `tools` | `false` | Let `act()` invoke a WebMCP tool the page registered when Jev is sure the tool is the request (see below). Sends tool names and descriptions to TypeSafe, and for the two tools sharing most words with the instruction also their parameter names, descriptions, types and enum values. | | `retryNoEffect` | `false` | Click the runner-up when an ambiguous click provably changed nothing. Off: effects the outline cannot show (aria-pressed, copy, play) look like "nothing". Never cached. | | `focusFallback` | `false` | On trees over 120K chars, show the LLM Jev's shortlist first. Off: it found the target in a minority of firings and cost accuracy on ordinary pages. | +## WebMCP tools (`tools: true`) + +The page's tools are listed while `act()` waits for the DOM to settle, and the tool questions ride +in the intent request that every act already makes, so a page without tools adds no question and a +page with tools adds no round trip. Alongside "which tool" (with a "none" option) a guard asks +whether the instruction names a control: "click the Add to cart button" always takes the element +path, even when `add_to_cart` exists. A tool is used only at ≥ 0.8 with "none" ≤ 0.2. + +Arguments are picked by Jev as spans of the instruction (enums and booleans as choices) and accepted +only when every parameter, stated or not, is ≥ 0.8. The argument questions of the two tools whose +names and descriptions share most words with the instruction ride in the same request, so a +confident tool call is usually one request (~300 ms); another winner costs one more. Values that +are not spans (dates to normalise, lists, nested objects) go to an argument-only LLM call that sees +just that tool, with its input schema as the response format; when the schema alone shows the +likely tool will need it, that call starts alongside the Jev request. Any doubt means the ordinary +act path continues from the intent it already has. Once a tool has been invoked the act is over, +success or error: it never also clicks through the UI. Tool acts are not cached. + +```ts +// TypeScript SDK; Chrome needs its WebMCP features on, which localBrowser.launch() does. +process.env.STAGEHAND_EXPERIMENTAL_JEV_ACT = JSON.stringify({ apiKey, tools: true }); +const stagehand = await Stagehand.create({ browser, model }); +await page.goto("https://browserbase.github.io/stagehand-eval-sites/sites/webmcp-test/"); + +await stagehand.act("add 19 and 23 together"); +// → { method: "webmcp", selector: "webmcp:calculateSum", arguments: ['{"a":19,"b":23}'] } +// message: 'Invoked WebMCP tool calculateSum: {"a":19,"b":23,"sum":42}' (one Jev request, no LLM call) + +await stagehand.act("click the Calculate button"); +// → names a control, so the ordinary element path runs +``` + +On 380 LLM-written requests over 166 tools harvested from six live sites: tool choice answered by +Jev for 79% of requests at 99% precision; arguments filled by Jev for 69% of calls at 98% precision. + ## What leaves the process Sent to TypeSafe: the instruction, candidate descriptions built from the accessibility outline @@ -65,7 +101,9 @@ logs its instruction and a trace of candidate descriptions at info level. ## Known limits -- Jev usage is logged but not part of `result.metadata.usage`. +- Jev usage is logged but not part of `result.metadata.usage`. With `tools`, an argument LLM call + started speculatively and not used finishes after the act has returned, and its tokens are not + counted anywhere. - Thresholds (0.7 accept, 0.9 none veto, 0.7 held-pick cap) were set on the act and breadth suites. The cache-check threshold (0.35) comes from direct API probes; no eval exercises the cache path. - No eval exercises page-state fail-fast or `retryNoEffect` end to end; both have unit tests only. diff --git a/packages/extension/services/jevAct/pipeline.ts b/packages/extension/services/jevAct/pipeline.ts index 69c97f492..cc117a37b 100644 --- a/packages/extension/services/jevAct/pipeline.ts +++ b/packages/extension/services/jevAct/pipeline.ts @@ -18,6 +18,8 @@ import { redactor, substituteVariables, } from "./args.js"; +import { invokeTool, type JevToolDeps } from "./toolAct.js"; +import { readToolDecision, toolQuestions } from "./tools.js"; import { blockingSignal, readPageState } from "./pageState.js"; import { NONE, @@ -95,6 +97,13 @@ export type JevActConfig = JevConfig & { extract?: "off" | "judge" | "pick"; /** Resolve observe() through Jev first. Default false. */ observe?: boolean; + /** + * Let act() invoke a WebMCP tool the page registered when Jev is sure the + * tool is the request. Sends tool names and descriptions, and for the likely + * tools their parameter names, descriptions, types and enum values, to + * TypeSafe. Default false. + */ + tools?: boolean; }; export type JevActDeps = { @@ -112,6 +121,8 @@ export type JevActDeps = { */ extractText?: (instruction: string) => Promise; takeAction: (action: Action) => Promise; + /** Present when `tools` is on and the act is not scoped to a locator. */ + webmcp?: JevToolDeps; }; export type JevActOutcome = @@ -119,6 +130,8 @@ export type JevActOutcome = kind: "done"; result: ActResultData; /** Must not be written to the act cache. */ noCache?: boolean; + /** The act was a WebMCP tool call; `argumentLlm` when its input came from the LLM. */ + viaTool?: { argumentLlm: boolean }; } | { kind: "fallback"; @@ -273,6 +286,16 @@ async function decideAndAct( // Intent fan-out: every question that only needs the instruction rides in // one request, so press / not-an-action / whole-page scroll finish here. const fillValues = fillValueCandidates(deps.instruction, deps.variables); + // Tool choice only needs the instruction too, so it costs no round trip of + // its own, and a page without tools adds no question at all. + const tools = deps.webmcp ? await deps.webmcp.tools.catch(() => []) : []; + const asked = tools.length > 0 ? toolQuestions(deps.instruction, tools) : undefined; + // Known from the schema alone: if this tool wins, only the LLM can fill it. + const speculative = asked?.needsArgumentLlm; + const speculativeInput = + speculative && deps.webmcp?.fillArguments && config.argumentLlm !== false + ? deps.webmcp.fillArguments(speculative).catch(() => null) + : undefined; const intent = await ask( ctx, "intent", @@ -350,10 +373,52 @@ async function decideAndAct( other: "Some other key, or the instruction is not a key press", }, }, + ...asked?.questions, }, ); + const intentEntry = trace[trace.length - 1]!; + if (asked && deps.webmcp) { + const decision = await readToolDecision(ctx, intent, asked, intentEntry); + if (decision.kind === "tool") { + let input = decision.input; + let argumentLlm = false; + if (!input && deps.webmcp.fillArguments && config.argumentLlm !== false) { + argumentLlm = true; + const started = performance.now(); + input = + (await (decision.tool === speculative && speculativeInput + ? speculativeInput + : deps.webmcp.fillArguments(decision.tool).catch(() => null))) ?? undefined; + trace.push({ + node: "tool_arguments_llm", + ms: Math.round(performance.now() - started), + speculative: decision.tool === speculative, + }); + } + if (input) { + deps.ensureTimeRemaining(); + const result = await invokeTool( + deps.webmcp, + deps.instruction, + deps.variables, + decision.tool, + input, + trace, + ); + // Replay only knows element actions. + return { kind: "done", result, noCache: true, viaTool: { argumentLlm } }; + } + intentEntry.tool_skip = "arguments_not_filled"; + } else { + intentEntry.tool_skip = decision.reason; + } + } const family = resolveFamily(choiceAnswer(intent, "family"), ctx.threshold); - annotate(trace, { choice: family.choice, confidence: family.confidence, top: family.top }); + Object.assign(intentEntry, { + choice: family.choice, + confidence: family.confidence, + top: family.top, + }); if (family.confidence < ctx.threshold) return fallback(`intent_low_confidence:${family.top}`); if (family.choice === "not_an_action") { diff --git a/packages/extension/services/jevAct/toolAct.ts b/packages/extension/services/jevAct/toolAct.ts new file mode 100644 index 000000000..63498cf74 --- /dev/null +++ b/packages/extension/services/jevAct/toolAct.ts @@ -0,0 +1,97 @@ +import type { + ActResultData, + Variables, + WebMCPToolDescriptor, +} from "@browserbasehq/stagehand-protocol/types"; +import type { Page } from "../../understudy/page.js"; +import { redactor, substituteVariables } from "./args.js"; +import type { TraceEntry } from "./pick.js"; +import type { JsonValue } from "./typesafeClient.js"; + +/** + * act() through a WebMCP tool: when the page registers tools and Jev is sure + * one of them IS the request, the tool is invoked instead of finding something + * to click. The choice itself rides in the act pipeline's intent request. + */ + +const RESULT_TIMEOUT_MS = 30_000; +const RESULT_MESSAGE_CHARS = 2_000; + +export type ToolInput = Record; + +export type JevToolDeps = { + page: Pick; + /** + * The page's tools. A promise, so the listing overlaps whatever the caller + * was already waiting for; empty on browsers without the WebMCP domain. + */ + tools: Promise; + /** Argument-only LLM call for one tool; null when it cannot fill them. */ + fillArguments?: (tool: WebMCPToolDescriptor) => Promise; +}; + +export async function invokeTool( + deps: JevToolDeps, + instruction: string, + variables: Variables | undefined, + tool: WebMCPToolDescriptor, + input: ToolInput, + trace: TraceEntry[], +): Promise { + const started = performance.now(); + const invocation = await deps.page.invokeWebMCPTool(tool.frameId, tool.name, { + input: resolveVariables(input, variables), + }); + // Past this point the tool is running: a failure to hear back is a failed + // act, never a reason to go and click through the UI as well. + const response = await deps.page + .waitForWebMCPInvocationResult(invocation.invocationId, { timeout: RESULT_TIMEOUT_MS }) + .catch((error: unknown) => ({ + status: "Error" as const, + output: undefined, + errorText: error instanceof Error ? error.message : String(error), + })); + trace.push({ + node: "tool_invoke", + ms: Math.round(performance.now() - started), + tool: tool.name, + status: response.status, + }); + + const success = response.status === "Completed"; + // The tool may echo what it was given; the message goes to the caller and + // the logs, so resolved %variable% values are put back behind their names. + const redact = redactor(variables) ?? ((text: string) => text); + const detail = success + ? response.output === undefined + ? "" + : `: ${redact(JSON.stringify(response.output)).slice(0, RESULT_MESSAGE_CHARS)}` + : `: ${redact(response.errorText ?? response.status)}`; + return { + success, + message: `${success ? "Invoked" : "Failed to invoke"} WebMCP tool ${tool.name}${detail}`, + actionDescription: instruction, + actions: [ + { + selector: `webmcp:${tool.name}`, + description: tool.description, + method: "webmcp", + // The placeholders, not the resolved values: results are logged and returned. + arguments: [JSON.stringify(input)], + }, + ], + }; +} + +/** Placeholders can sit inside arrays and objects when the argument LLM shaped the input. */ +function resolveVariables(input: ToolInput, variables: Variables | undefined): ToolInput { + const resolve = (value: JsonValue): JsonValue => { + if (typeof value === "string") return substituteVariables(value, variables); + if (Array.isArray(value)) return value.map(resolve); + if (value && typeof value === "object") { + return Object.fromEntries(Object.entries(value).map(([key, child]) => [key, resolve(child)])); + } + return value; + }; + return Object.fromEntries(Object.entries(input).map(([name, value]) => [name, resolve(value)])); +} diff --git a/packages/extension/services/jevAct/tools.ts b/packages/extension/services/jevAct/tools.ts new file mode 100644 index 000000000..55e759e2f --- /dev/null +++ b/packages/extension/services/jevAct/tools.ts @@ -0,0 +1,371 @@ +import type { WebMCPToolDescriptor } from "@browserbasehq/stagehand-protocol/types"; +import { ask, round, type AskContext, type TraceEntry } from "./pick.js"; +import { + choiceAnswer, + noulAnswer, + type JevQuestion, + type JevResponse, + type JsonValue, +} from "./typesafeClient.js"; + +/** + * WebMCP tool selection on Jev. A page that registers tools has already said + * what it can do, so "which tool fulfils this request" is a closed-set choice, + * and most arguments are words of the instruction (a span Jev can point at) or + * an enum/boolean. Anything else is left to the caller: an argument-only LLM + * call for the chosen tool, or the ordinary element path. + */ + +const NONE = "none_of_these"; +const UNSET = "unset"; +/** Set on 380 labelled requests over 166 live tools: 79% answered at 99% precision. */ +const TOOL_MIN = 0.8; +const TOOL_NONE_MAX = 0.2; +/** Every parameter, including the ones judged "not stated": 69% filled at 98% precision. */ +const ARGUMENT_MIN = 0.8; +/** One option is reserved for "unset". */ +const MAX_SPANS = 250; +const MAX_TOOLS = 254; +const DESCRIPTION_CHARS = 600; +/** + * Argument questions for the tools most likely to win ride in the same request + * as the choice, so a confident tool call is one round trip, not two. + */ +const SPECULATIVE_TOOLS = 2; +const STOPWORDS = new Set([ + "the", + "a", + "an", + "to", + "of", + "in", + "on", + "for", + "and", + "my", + "me", + "it", +]); + +type ToolInput = Record; + +export type ToolOutcome = + /** `input` is absent when Jev chose the tool but could not fill its arguments with confidence. */ + | { kind: "tool"; tool: WebMCPToolDescriptor; input?: ToolInput } + /** The request is not a tool call (or Jev is unsure): carry on with the element path. */ + | { kind: "skip"; reason: string }; + +type Property = { type?: JsonValue; enum?: JsonValue[]; description?: JsonValue }; + +function propertiesOf(tool: WebMCPToolDescriptor): Record { + const properties = tool.inputSchema?.properties; + return properties && typeof properties === "object" && !Array.isArray(properties) + ? (properties as Record) + : {}; +} + +function requiredOf(tool: WebMCPToolDescriptor): string[] { + const required = tool.inputSchema?.required; + return Array.isArray(required) + ? required.filter((name): name is string => typeof name === "string") + : []; +} + +export type ToolQuestions = { + offered: WebMCPToolDescriptor[]; + questions: Record; + /** Tools whose argument questions are already in `questions`. */ + withArguments: WebMCPToolDescriptor[]; + /** Best lexical match when it needs an argument LLM whatever Jev says about spans. */ + needsArgumentLlm?: WebMCPToolDescriptor; + spanIds: Map; +}; + +/** + * Questions to merge into a request that only needs the instruction (act's + * intent fan-out). Undefined when there is nothing to ask. + */ +export function toolQuestions( + instruction: string, + tools: WebMCPToolDescriptor[], +): ToolQuestions | undefined { + // Two frames may register the same name; the choice is keyed by name, so + // only unambiguous names are offered. + const counts = new Map(); + for (const tool of tools) counts.set(tool.name, (counts.get(tool.name) ?? 0) + 1); + const offered = tools.filter((tool) => counts.get(tool.name) === 1 && tool.name !== NONE); + if (offered.length === 0 || offered.length > MAX_TOOLS) return undefined; + + const criteria = Object.fromEntries( + offered.map((tool) => { + const takes = Object.keys(propertiesOf(tool)); + return [ + tool.name, + { + does: tool.description.slice(0, DESCRIPTION_CHARS), + ...(takes.length > 0 ? { takes } : {}), + }, + ]; + }), + ); + const instructions = { task: "Which tool fulfils the user's request?", request: instruction }; + const questions: Record = { + tool_best: { type: "choice", instructions, criteria }, + tool_strict: { + type: "choice", + instructions, + criteria: { + ...criteria, + [NONE]: + "No listed tool can do what the request asks; a different capability would be needed", + }, + }, + // "click the Add to cart button" names a control. The user asked for a + // click, so they get a click, even though add_to_cart would match. + tool_names_control: { + type: "noul", + instructions: { + question: + "Does the request tell the agent to operate a specific on-page control (click, type into, select from, press, scroll, hover a named button, link, field, icon or menu) rather than state a goal?", + request: instruction, + }, + }, + }; + + const spanIds = new Map(instructionSpans(instruction).map((span, index) => [`s${index}`, span])); + const ranked = rankByWords(instruction, offered); + const withArguments: WebMCPToolDescriptor[] = []; + for (const { tool } of ranked.slice(0, SPECULATIVE_TOOLS)) { + const own = argumentQuestions(tool, instruction, spanIds); + if (!own || Object.keys(own).length === 0) continue; + for (const [name, question] of Object.entries(own)) { + questions[argumentKey(tool, name)] = question; + } + withArguments.push(tool); + } + // A clear lexical leader with a list, object or otherwise non-scalar + // parameter will need the argument LLM if it wins; the caller may start it now. + const [leader, runnerUp] = ranked; + const needsArgumentLlm = + leader && + leader.score > (runnerUp?.score ?? 0) && + Object.keys(propertiesOf(leader.tool)).length > 0 && + !argumentQuestions(leader.tool, "", spanIds) + ? leader.tool + : undefined; + return { + offered, + questions, + withArguments, + spanIds, + ...(needsArgumentLlm ? { needsArgumentLlm } : {}), + }; +} + +/** Reads the tool decision out of the response the questions rode in. */ +export async function readToolDecision( + ctx: AskContext, + response: JevResponse, + asked: ToolQuestions, + entry: TraceEntry, +): Promise { + const best = choiceAnswer(response, "tool_best"); + const none = choiceAnswer(response, "tool_strict").probabilities[NONE] ?? 0; + const namesControl = noulAnswer(response, "tool_names_control").noul; + Object.assign(entry, { + tool: best.choice, + tool_best: round(best.confidence), + tool_none: round(none), + names_control: round(namesControl), + tools: asked.offered.length, + }); + + if (namesControl >= 0.5) return { kind: "skip", reason: "names_a_control" }; + if (none > TOOL_NONE_MAX) return { kind: "skip", reason: "no_tool_fits" }; + if (best.confidence < TOOL_MIN) return { kind: "skip", reason: "tool_ambiguous" }; + const tool = asked.offered.find((candidate) => candidate.name === best.choice); + if (!tool) return { kind: "skip", reason: "tool_ambiguous" }; + + const names = Object.keys(propertiesOf(tool)); + let input: ToolInput | undefined; + if (names.length === 0) { + input = {}; + } else if (asked.withArguments.includes(tool)) { + input = readArguments(response, tool, asked.spanIds, (name) => argumentKey(tool, name), entry); + } else { + // The winner was not among the lexical favourites: one more request. + const questions = argumentQuestions(tool, ctx.instruction, asked.spanIds); + if (questions) { + const second = await ask(ctx, "tool_arguments", { request: ctx.instruction }, questions); + input = readArguments( + second, + tool, + asked.spanIds, + (name) => name, + ctx.trace[ctx.trace.length - 1]!, + ); + } + } + return { kind: "tool", tool, ...(input ? { input } : {}) }; +} + +function argumentKey(tool: WebMCPToolDescriptor, name: string): string { + return `tool_arg:${tool.name}:${name}`; +} + +function words(text: string): string[] { + const found: string[] = text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? []; + return found.filter((word) => word.length > 1 && !STOPWORDS.has(word)); +} + +/** Cheap guess at the likely winners; only decides whose argument questions ride along. */ +function rankByWords( + instruction: string, + tools: WebMCPToolDescriptor[], +): Array<{ tool: WebMCPToolDescriptor; score: number }> { + const wanted = new Set(words(instruction)); + const score = (tool: WebMCPToolDescriptor): number => + 3 * + new Set(words(tool.name.replace(/([a-z])([A-Z])/g, "$1 $2")).filter((w) => wanted.has(w))) + .size + + new Set(words(tool.description).filter((w) => wanted.has(w))).size; + return tools + .map((tool) => ({ tool, score: score(tool) })) + .filter((entry) => entry.score > 0) + .sort((a, b) => b.score - a.score); +} + +/** Quoted strings first, then every run of one to four words, punctuation and possessives stripped. */ +export function instructionSpans(instruction: string): string[] { + const spans: string[] = []; + const add = (span: string): void => { + if (span && !spans.includes(span)) spans.push(span); + }; + for (const match of instruction.matchAll(/"([^"\n]+)"|“([^”\n]+)”|(? + word.replace(/^[("'“‘]+|[)"'”’.,;:!?]+$/g, "").replace(/['’]s$/, ""), + ); + for (let size = 1; size <= 4; size++) { + for (let start = 0; start + size <= words.length; start++) { + add( + words + .slice(start, start + size) + .join(" ") + .trim(), + ); + } + } + return spans.slice(0, MAX_SPANS); +} + +/** + * Undefined when a parameter is not a scalar. Values the instruction only + * implies ("July 15th" for an ISO date, lists, nested objects) are not spans, + * and guessing them is how a tool gets called with the wrong input. + */ +function argumentQuestions( + tool: WebMCPToolDescriptor, + instruction: string, + spanIds: Map, +): Record | undefined { + const properties = propertiesOf(tool); + const unset = "The request does not state a value for this parameter"; + const questions: Record = {}; + for (const [name, property] of Object.entries(properties)) { + const instructions = { + task: `What value does the request give for the parameter '${name}' of the tool '${tool.name}'?`, + parameter: { + name, + ...(property.description === undefined ? {} : { description: property.description }), + ...(property.type === undefined ? {} : { type: property.type }), + }, + request: instruction, + }; + if (Array.isArray(property.enum) && property.enum.length > 0) { + questions[name] = { + type: "choice", + instructions, + criteria: { + ...Object.fromEntries( + property.enum.map((value, index) => [ + `e${index}`, + `the request means ${JSON.stringify(value)}`, + ]), + ), + [UNSET]: unset, + }, + }; + } else if (property.type === "boolean") { + questions[name] = { + type: "choice", + instructions, + criteria: { + true: "the request wants this on / yes", + false: "the request wants this off / no", + [UNSET]: unset, + }, + }; + } else if ( + property.type === "string" || + property.type === "number" || + property.type === "integer" + ) { + questions[name] = { + type: "choice", + instructions, + criteria: { + ...Object.fromEntries([...spanIds].map(([id, span]) => [id, `the exact words: ${span}`])), + [UNSET]: unset, + }, + }; + } else { + return undefined; + } + } + return questions; +} + +/** Undefined unless Jev is sure about every parameter, stated or not, and none required is missing. */ +function readArguments( + response: JevResponse, + tool: WebMCPToolDescriptor, + spanIds: Map, + keyOf: (name: string) => string, + entry: TraceEntry, +): ToolInput | undefined { + const properties = propertiesOf(tool); + const input: ToolInput = {}; + let weakest = 1; + for (const [name, property] of Object.entries(properties)) { + const answer = choiceAnswer(response, keyOf(name)); + weakest = Math.min(weakest, answer.confidence); + if (answer.choice === UNSET) continue; + if (Array.isArray(property.enum) && property.enum.length > 0) { + input[name] = property.enum[Number(answer.choice.slice(1))]!; + } else if (property.type === "boolean") { + input[name] = answer.choice === "true"; + } else { + const span = spanIds.get(answer.choice); + if (span === undefined) return undefined; + if (property.type === "string") { + input[name] = span; + } else { + const number = /-?\d+(?:\.\d+)?/.exec(span.replaceAll(",", "")); + if (!number || (property.type === "integer" && number[0].includes("."))) return undefined; + input[name] = Number(number[0]); + } + } + } + Object.assign(entry, { arguments_weakest: round(weakest), arguments_filled: Object.keys(input) }); + if (weakest < ARGUMENT_MIN) return undefined; + // "search the store for mugs" put "mugs" into both query and category, each + // with high confidence. One span is one value; which parameter is a judgement call. + const spans = Object.entries(input) + .filter(([name]) => !properties[name]?.enum && properties[name]?.type !== "boolean") + .map(([, value]) => (typeof value === "string" ? value : JSON.stringify(value))); + if (new Set(spans).size < spans.length) return undefined; + if (requiredOf(tool).some((name) => !(name in input))) return undefined; + return input; +} diff --git a/packages/extension/tests/jevTools.test.ts b/packages/extension/tests/jevTools.test.ts new file mode 100644 index 000000000..829b77dda --- /dev/null +++ b/packages/extension/tests/jevTools.test.ts @@ -0,0 +1,392 @@ +import { trace } from "@opentelemetry/api"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { + LLMGenerateParams, + LLMGenerateResult, + WebMCPToolDescriptor, +} from "@browserbasehq/stagehand-protocol/types"; +import { toolArguments } from "../inference.js"; +import { StagehandLogger } from "../logger.js"; +import type { Variables } from "@browserbasehq/stagehand-protocol/types"; +import { runJevActPipeline, type JevActDeps } from "../services/jevAct/pipeline.js"; +import type { JevToolDeps } from "../services/jevAct/toolAct.js"; +import { instructionSpans } from "../services/jevAct/tools.js"; + +const tools: WebMCPToolDescriptor[] = [ + { + name: "add_to_cart", + description: "Add a product to the cart", + frameId: "main", + inputSchema: { + type: "object", + required: ["product_id"], + properties: { + product_id: { type: "string" }, + quantity: { type: "integer" }, + size: { type: "string", enum: ["S", "M", "L"] }, + }, + }, + }, + { name: "clear_cart", description: "Empty the cart", frameId: "main" }, + { + name: "search_flights", + description: "Search flights", + frameId: "main", + inputSchema: { type: "object", properties: { legs: { type: "array" } } }, + }, +]; + +type Probabilities = Record; +type Scripted = { + tool?: string; + toolP?: number; + none?: number; + namesControl?: number; + /** Per parameter: the wanted option (a span's text, an enum value, "unset") and its confidence. */ + args?: Record; +}; + +/** Answers tool questions from a script; span and enum options are looked up by their text. */ +function stubJev(script: Scripted) { + const requests: Array }>> = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + const body = JSON.parse(init.body as string) as { + questions: Record }>; + }; + requests.push(body.questions); + const answers: Record = {}; + for (const [key, question] of Object.entries(body.questions)) { + if (question.type === "noul") { + answers[key] = { type: "noul", noul: script.namesControl ?? 0 }; + continue; + } + let choice = script.tool ?? "clear_cart"; + let confidence = script.toolP ?? 0.95; + const probabilities: Probabilities = {}; + if (key === "tool_strict") probabilities.none_of_these = script.none ?? 0; + if (key === "family") { + answers[key] = { + type: "choice", + choice: "not_an_action", + confidence: 0.99, + probabilities: { not_an_action: 0.99 }, + }; + continue; + } + if (key !== "tool_best" && key !== "tool_strict") { + choice = "unset"; + confidence = 0.95; + } + const parameter = key.startsWith("tool_arg:") ? key.split(":")[2]! : key; + if ( + script.args && + key.startsWith("tool_arg:") && + !key.startsWith(`tool_arg:${script.tool}:`) + ) { + // A lexical favourite that is not the scripted winner. + } else if (script.args && parameter in script.args) { + const [wanted, p] = script.args[parameter]!; + choice = + Object.entries(question.criteria ?? {}).find( + ([id, text]) => + id === wanted || + text.endsWith(`: ${wanted}`) || + text.endsWith(` ${JSON.stringify(wanted)}`), + )?.[0] ?? "unset"; + confidence = p; + } + probabilities[choice] = confidence; + answers[key] = { type: "choice", choice, confidence, probabilities }; + } + return new Response(JSON.stringify({ answers, usage: { input_tokens: 10 } }), { + status: 200, + }); + }), + ); + return requests; +} + +type Harness = { + deps: JevActDeps; + invoked: Array<{ name: string; input: unknown }>; + /** `tool_skip` from the intent trace entry of the last run. */ + skipReason: () => string | undefined; + webmcp: JevToolDeps; +}; + +function harness(instruction: string, overrides: Partial = {}, variables?: Variables) { + const invoked: Harness["invoked"] = []; + const logged: string[] = []; + const webmcp: JevToolDeps = { + tools: Promise.resolve(tools), + page: { + invokeWebMCPTool: async (frameId, toolName, options) => { + invoked.push({ name: toolName, input: options?.input }); + return { invocationId: "inv-1", toolName, frameId, input: options?.input ?? {} }; + }, + waitForWebMCPInvocationResult: async () => ({ + invocationId: "inv-1", + status: "Completed", + output: { ok: true }, + }), + }, + ...overrides, + }; + const deps = { + instruction, + ...(variables ? { variables } : {}), + logger: new StagehandLogger({ tracer: trace.getTracer("jev-tools-test") }, (line) => { + logged.push(JSON.stringify(line)); + }), + ensureTimeRemaining: () => {}, + snapshotOptions: {}, + // The scripted intent is "not an action", so a skipped tool ends the act + // before any page work. + page: {} as JevActDeps["page"], + takeAction: async () => { + throw new Error("no element action expected"); + }, + webmcp, + } satisfies JevActDeps; + const skipReason = (): string | undefined => + /tool_skip\\*":\\*"([a-z_]+)/.exec(logged.join("\n"))?.[1]; + return { deps, invoked, skipReason, webmcp } satisfies Harness; +} + +const config = { apiKey: "test", tools: true }; + +describe("Jev WebMCP tool act", () => { + afterEach(() => vi.unstubAllGlobals()); + + it("invokes a tool that takes no input inside the intent request", async () => { + const requests = stubJev({ tool: "clear_cart" }); + const h = harness("empty my cart"); + const outcome = await runJevActPipeline(config, h.deps); + expect(h.invoked).toEqual([{ name: "clear_cart", input: {} }]); + expect(requests).toHaveLength(1); + expect(Object.keys(requests[0]!)).toEqual(expect.arrayContaining(["family", "tool_best"])); + expect(outcome).toMatchObject({ + kind: "done", + noCache: true, + viaTool: { argumentLlm: false }, + result: { + success: true, + actions: [{ selector: "webmcp:clear_cart", method: "webmcp", arguments: ["{}"] }], + }, + }); + }); + + it("fills scalar arguments from the instruction's own words in the same request", async () => { + const requests = stubJev({ + tool: "add_to_cart", + args: { product_id: ["p_102", 0.97], quantity: ["2", 0.95], size: ["M", 0.93] }, + }); + const h = harness("add 2 of product p_102, size M, to my cart"); + await runJevActPipeline(config, h.deps); + expect(requests).toHaveLength(1); + expect(h.invoked[0]).toEqual({ + name: "add_to_cart", + input: { product_id: "p_102", quantity: 2, size: "M" }, + }); + }); + + it("asks once more when the winner was not a lexical favourite", async () => { + const requests = stubJev({ tool: "add_to_cart", args: { product_id: ["p_102", 0.97] } }); + const h = harness("I want p_102"); + await runJevActPipeline(config, h.deps); + expect(requests).toHaveLength(2); + expect(h.invoked[0]?.input).toEqual({ product_id: "p_102" }); + }); + + it("leaves requests that name a control to the element path", async () => { + stubJev({ tool: "add_to_cart", namesControl: 0.97 }); + const h = harness("click the Add to cart button"); + await runJevActPipeline(config, h.deps); + expect(h.invoked).toEqual([]); + expect(h.skipReason()).toBe("names_a_control"); + }); + + it("skips when no tool fits or the choice is split", async () => { + stubJev({ tool: "clear_cart", none: 0.6 }); + const none = harness("open the footer newsletter link"); + await runJevActPipeline(config, none.deps); + expect(none.skipReason()).toBe("no_tool_fits"); + + stubJev({ tool: "clear_cart", toolP: 0.55 }); + const split = harness("sort out my cart"); + await runJevActPipeline(config, split.deps); + expect(split.skipReason()).toBe("tool_ambiguous"); + expect([...none.invoked, ...split.invoked]).toEqual([]); + }); + + it("hands unsure arguments to the argument LLM, and skips without one", async () => { + stubJev({ tool: "add_to_cart", args: { product_id: ["p_102", 0.55] } }); + const fillArguments = vi.fn(async () => ({ product_id: "p_102" })); + const h = harness("add that p_102 product to the cart", { fillArguments }); + const outcome = await runJevActPipeline(config, h.deps); + expect(outcome).toMatchObject({ kind: "done", viaTool: { argumentLlm: true } }); + expect(h.invoked[0]?.input).toEqual({ product_id: "p_102" }); + + stubJev({ tool: "add_to_cart", args: { product_id: ["p_102", 0.55] } }); + const without = harness("add that p_102 product to the cart"); + await runJevActPipeline({ ...config, argumentLlm: false }, without.deps); + expect(without.invoked).toEqual([]); + expect(without.skipReason()).toBe("arguments_not_filled"); + }); + + it("starts the argument LLM alongside Jev when the likely tool takes a list or object", async () => { + let jevAnswered = false; + stubJev({ tool: "search_flights" }); + const fillArguments = vi.fn(async () => { + expect(jevAnswered).toBe(false); + return { legs: ["SFO-JFK"] }; + }); + const h = harness("search flights from SFO to JFK", { fillArguments }); + const ensure = h.deps.ensureTimeRemaining; + // ensureTimeRemaining runs again right before the invocation, after Jev answered. + let calls = 0; + h.deps.ensureTimeRemaining = () => { + if (++calls > 1) jevAnswered = true; + ensure(); + }; + await runJevActPipeline(config, h.deps); + expect(fillArguments).toHaveBeenCalledTimes(1); + expect(h.invoked[0]?.input).toEqual({ legs: ["SFO-JFK"] }); + }); + + it("does not trust one span filling two parameters", async () => { + stubJev({ tool: "add_to_cart", args: { product_id: ["2", 0.95], quantity: ["2", 0.95] } }); + const fillArguments = vi.fn(async () => ({ product_id: "2" })); + const h = harness("add product 2 to my cart", { fillArguments }); + await runJevActPipeline(config, h.deps); + expect(fillArguments).toHaveBeenCalledTimes(1); + expect(h.invoked[0]?.input).toEqual({ product_id: "2" }); + }); + + it("skips when a required argument is not stated", async () => { + stubJev({ tool: "add_to_cart", args: { product_id: ["unset", 0.95] } }); + const h = harness("add a nice product to my cart"); + await runJevActPipeline(config, h.deps); + expect(h.invoked).toEqual([]); + expect(h.skipReason()).toBe("arguments_not_filled"); + }); + + it("never sends a variable's value to TypeSafe and resolves it only for the page", async () => { + const requests = stubJev({ tool: "add_to_cart", args: { product_id: ["%sku%", 0.96] } }); + const h = harness("add product %sku% to my cart", {}, { sku: "secret-sku-9" }); + const outcome = await runJevActPipeline(config, h.deps); + expect(JSON.stringify(requests)).not.toContain("secret-sku-9"); + expect(h.invoked[0]?.input).toEqual({ product_id: "secret-sku-9" }); + expect(outcome.kind === "done" && outcome.result.actions[0]?.arguments?.[0]).toBe( + '{"product_id":"%sku%"}', + ); + }); + + it("resolves %variables% nested inside LLM-shaped arguments and redacts what the tool echoes", async () => { + stubJev({ tool: "search_flights" }); + const fillArguments = vi.fn(async () => ({ legs: ["%from%-%to%"], note: { who: "%from%" } })); + const h = harness("fly %from% to %to%", { fillArguments }, { from: "SFO", to: "JFK" }); + h.webmcp.page.waitForWebMCPInvocationResult = async () => ({ + invocationId: "inv-1", + status: "Completed", + output: { booked: "SFO-JFK for SFO" }, + }); + const outcome = await runJevActPipeline(config, h.deps); + expect(h.invoked[0]?.input).toEqual({ legs: ["SFO-JFK"], note: { who: "SFO" } }); + expect(outcome.kind === "done" && outcome.result.message).toContain("%from%-%to%"); + expect(outcome.kind === "done" && outcome.result.message).not.toContain("SFO"); + }); + + it("reports a tool error as a failed act instead of falling through to the UI", async () => { + stubJev({ tool: "clear_cart" }); + const h = harness("empty my cart"); + h.webmcp.page.waitForWebMCPInvocationResult = async () => { + throw new Error("Timed out waiting for WebMCP tool"); + }; + const outcome = await runJevActPipeline(config, h.deps); + expect(outcome).toMatchObject({ kind: "done", result: { success: false } }); + }); + + it("adds no question when the page has no tools or no WebMCP support", async () => { + const requests = stubJev({}); + const unsupported = Promise.reject(new Error("no WebMCP domain")); + const h = harness("empty my cart", { tools: unsupported }); + await runJevActPipeline(config, h.deps); + expect(Object.keys(requests[0]!)).not.toContain("tool_best"); + expect(h.invoked).toEqual([]); + }); + + it("builds spans from quotes and word runs without punctuation or possessives", () => { + const spans = instructionSpans(`export doc_45's sequence as "plain FASTA", please.`); + expect(spans[0]).toBe("plain FASTA"); + expect(spans).toContain("doc_45"); + expect(spans).toContain("please"); + expect(spans).not.toContain("doc_45's"); + }); +}); + +describe("toolArguments inference", () => { + const usage = { inputTokens: 5, outputTokens: 2, totalTokens: 7 }; + + it("shapes the answer with the tool's own schema when every property is required", async () => { + const generate = vi.fn(async (request: LLMGenerateParams): Promise => { + expect(request.responseFormat).toMatchObject({ + schema: { required: ["a", "b"], properties: { a: { type: "number" } } }, + }); + return { + role: "assistant", + content: { type: "text", text: "" }, + outputFormat: "json_schema", + structuredContent: { a: 1, b: 2 }, + usage, + }; + }); + const result = await toolArguments({ + instruction: "add 1 and 2", + variableNames: [], + generate, + tool: { + name: "sum", + description: "Add", + inputSchema: { + type: "object", + required: ["a", "b"], + properties: { a: { type: "number" }, b: { type: "number" } }, + }, + }, + }); + expect(result.input).toEqual({ a: 1, b: 2 }); + expect(generate).toHaveBeenCalledTimes(1); + }); + + it("carries the input as a JSON string when the schema has optional properties", async () => { + const generate = vi.fn(async (request: LLMGenerateParams): Promise => { + expect(request.responseFormat).toMatchObject({ schema: { required: ["input_json"] } }); + return { + role: "assistant", + content: { type: "text", text: "" }, + outputFormat: "json_schema", + structuredContent: { input_json: '{"query":"mugs"}' }, + usage, + }; + }); + const result = await toolArguments({ + instruction: "search for mugs", + variableNames: [], + generate, + tool: { + name: "search", + description: "Search", + inputSchema: { + type: "object", + required: ["query"], + properties: { query: { type: "string" }, category: { type: "string" } }, + }, + }, + }); + expect(result.input).toEqual({ query: "mugs" }); + expect(result.prompt_tokens).toBe(5); + }); +}); diff --git a/packages/protocol/schemas.ts b/packages/protocol/schemas.ts index 6974848f4..f7b779465 100644 --- a/packages/protocol/schemas.ts +++ b/packages/protocol/schemas.ts @@ -1668,6 +1668,7 @@ export const StagehandInitParamsSchema = z argumentLlm: z.boolean().optional(), pageState: z.boolean().optional(), observe: z.boolean().optional(), + tools: z.boolean().optional(), }) .optional() .meta({ diff --git a/packages/protocol/stagehand.v4.json b/packages/protocol/stagehand.v4.json index 0cc3fbf01..7d44c82ca 100644 --- a/packages/protocol/stagehand.v4.json +++ b/packages/protocol/stagehand.v4.json @@ -1267,6 +1267,9 @@ }, "observe": { "type": "boolean" + }, + "tools": { + "type": "boolean" } }, "required": ["api_key"], diff --git a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip index 39c9827cd..e9a365db1 100644 Binary files a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip and b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip differ diff --git a/packages/sdk-go/models.gen.go b/packages/sdk-go/models.gen.go index 50cbb4b2a..fafcfb62a 100644 --- a/packages/sdk-go/models.gen.go +++ b/packages/sdk-go/models.gen.go @@ -1944,6 +1944,9 @@ type StagehandInitParamsExperimentalJevAct struct { // RetryNoEffect corresponds to the JSON schema field "retry_no_effect". RetryNoEffect *bool `json:"retry_no_effect,omitempty,omitzero"` + // Tools corresponds to the JSON schema field "tools". + Tools *bool `json:"tools,omitempty,omitzero"` + // Verify corresponds to the JSON schema field "verify". Verify *StagehandInitParamsExperimentalJevActVerify `json:"verify,omitempty,omitzero"` } diff --git a/packages/sdk-python/src/stagehand/_generated/input_types.py b/packages/sdk-python/src/stagehand/_generated/input_types.py index d11682d6e..7fe128ba7 100644 --- a/packages/sdk-python/src/stagehand/_generated/input_types.py +++ b/packages/sdk-python/src/stagehand/_generated/input_types.py @@ -239,6 +239,7 @@ class ExperimentalJevAct(TypedDict): argument_llm: NotRequired[bool] page_state: NotRequired[bool] observe: NotRequired[bool] + tools: NotRequired[bool] class ExternalProxyConfig(TypedDict): diff --git a/packages/sdk-python/src/stagehand/_generated/models.py b/packages/sdk-python/src/stagehand/_generated/models.py index 2c6acea29..e8183d1b8 100644 --- a/packages/sdk-python/src/stagehand/_generated/models.py +++ b/packages/sdk-python/src/stagehand/_generated/models.py @@ -583,6 +583,7 @@ class ExperimentalJevAct(WireModel): argument_llm: Optional[StrictBool] = None page_state: Optional[StrictBool] = None observe: Optional[StrictBool] = None + tools: Optional[StrictBool] = None class ExternalProxyConfig(WireModel):