diff --git a/.changeset/decisions-providers.md b/.changeset/decisions-providers.md new file mode 100644 index 000000000..d5266256d --- /dev/null +++ b/.changeset/decisions-providers.md @@ -0,0 +1,8 @@ +--- +"@browserbasehq/stagehand-extension": patch +"@browserbasehq/stagehand-go": patch +"@browserbasehq/stagehand": patch +"@browserbasehq/stagehand-python": patch +--- + +The experimental `experimentalDecisions` path can use other decision models: `provider: "cloudflare"` (Clef on Workers AI, needs `accountId`), `"perplexity"` (Decider) or `"openai"` (Decisions API preview), next to the default `"typesafe"` (Jev). Answers from every provider are validated before they are acted on. diff --git a/packages/evals/initStagehand.ts b/packages/evals/initStagehand.ts index 8cecc5d1e..b85c6d476 100644 --- a/packages/evals/initStagehand.ts +++ b/packages/evals/initStagehand.ts @@ -88,13 +88,29 @@ export async function initStagehand({ experimentalDecisions = { apiKey: "instrument-only", enabled: false }; } if (process.env.EVAL_DECISIONS === "1") { - const typesafeApiKey = resolveKey("TYPESAFE_API_KEY").value; - if (!typesafeApiKey) { - throw new Error("Stagehand init: EVAL_DECISIONS=1 requires TYPESAFE_API_KEY"); + // EVAL_DECISIONS_PROVIDER: typesafe (default) | cloudflare | perplexity | openai + const provider = process.env.EVAL_DECISIONS_PROVIDER ?? "typesafe"; + const keyName = { + typesafe: "TYPESAFE_API_KEY", + cloudflare: "CLOUDFLARE_API_TOKEN", + perplexity: "PERPLEXITY_API_KEY", + openai: "OPENAI_API_KEY", + }[provider]; + if (!keyName) throw new Error(`Stagehand init: unknown EVAL_DECISIONS_PROVIDER "${provider}"`); + const decisionsApiKey = resolveKey(keyName).value; + if (!decisionsApiKey) { + throw new Error( + `Stagehand init: EVAL_DECISIONS=1 with provider "${provider}" requires ${keyName}`, + ); + } + if (provider === "cloudflare" && !process.env.CLOUDFLARE_ACCOUNT_ID) { + throw new Error('Stagehand init: provider "cloudflare" requires CLOUDFLARE_ACCOUNT_ID'); } const actConfidence = Number(process.env.EVAL_DECISIONS_CONFIDENCE); experimentalDecisions = { - apiKey: typesafeApiKey, + ...(provider === "typesafe" ? {} : { provider }), + apiKey: decisionsApiKey, + ...(provider === "cloudflare" ? { accountId: process.env.CLOUDFLARE_ACCOUNT_ID } : {}), ...(process.env.EVAL_DECISIONS_MODEL ? { model: process.env.EVAL_DECISIONS_MODEL } : {}), ...(Number.isFinite(actConfidence) && process.env.EVAL_DECISIONS_CONFIDENCE ? { actConfidence } diff --git a/packages/extension/controllers/stagehandController.ts b/packages/extension/controllers/stagehandController.ts index 80b650af3..3af523002 100644 --- a/packages/extension/controllers/stagehandController.ts +++ b/packages/extension/controllers/stagehandController.ts @@ -147,7 +147,7 @@ export function createStagehandController( systemPrompt: state.initParams.systemPrompt, cache: cacheService.buildCacheContext(state.initParams), gateway, - // Separate opt-in: this sends the extracted data to TypeSafe. + // Separate opt-in: this sends the extracted data to the decision provider. decisions: state.initParams.experimentalDecisions?.extract && state.initParams.experimentalDecisions.extract !== "off" && diff --git a/packages/extension/services/actService.ts b/packages/extension/services/actService.ts index a7afec975..5c7481183 100644 --- a/packages/extension/services/actService.ts +++ b/packages/extension/services/actService.ts @@ -35,7 +35,7 @@ import { import { redactor } from "./decisions/args.js"; import type { DecisionsToolDeps } from "./decisions/toolAct.js"; import { focusOutline, parseOutline } from "./decisions/tree.js"; -import type { JsonValue } from "./decisions/typesafeClient.js"; +import type { JsonValue } from "./decisions/client.js"; import * as llmService from "./llmService.js"; import { disabledCacheMetadata, zeroStagehandResultUsage } from "./resultUsage.js"; diff --git a/packages/extension/services/decisions/README.md b/packages/extension/services/decisions/README.md index 47a7c78b5..4774958fc 100644 --- a/packages/extension/services/decisions/README.md +++ b/packages/extension/services/decisions/README.md @@ -1,16 +1,52 @@ # Decisions (experimental) -Resolves `act("…")` through a decision tree of [TypeSafe Jev](https://docs.typesafe.ai) questions -instead of one LLM call, and hands the act to the existing LLM pipeline whenever the decision model is not -confident. The decision model is a System One model: it answers typed questions (Choice / Noul) with -probabilities, cannot generate text, and costs roughly 100–300 ms and a few thousand input tokens -per request. +Resolves `act()`, `observe()` and `extract()` through small typed questions to a **decision +model** instead of one LLM call, and hands over to the existing LLM pipeline whenever the model is +not confident. A decision model answers "which of these options" (Choice) and "yes or no" (Noul) +with probabilities; it cannot generate text, and answers in roughly 100–300 ms for a few thousand +input tokens per request. Off unless `experimentalDecisions` is present in the init params. The TypeScript SDK sets it from the `STAGEHAND_EXPERIMENTAL_DECISIONS` environment variable (the config below, as JSON); it is deliberately not a field of the public create config. Evals build that variable from `EVAL_DECISIONS=1` and the other `EVAL_DECISIONS_*` switches in `packages/evals/initStagehand.ts`. +## Providers + +`provider` selects the service; everything above the client (`client.ts`) is provider-agnostic. + +| `provider` | Model (default) | Needs | Wire format | Verified | +| ---------------------- | ------------------------------- | --------------------- | ---------------------------------------------------------------------------- | --------------------------------------------------------------- | +| `"typesafe"` (default) | Jev (`jev-latest`) | `apiKey` | System One: `{ state, questions }` → `{ answers }` | Live, and by every eval in this stack | +| `"cloudflare"` | Clef (`clef`, or `clef-flash`) | `apiKey`, `accountId` | System One body on Workers AI `/ai/run/@cf/cloudflare/`, `{ result }` | Against the documented format | +| `"perplexity"` | Decider (`pplx-decider-v1-27b`) | `apiKey` | System One body on `/v1/decisions` | Against the documented format | +| `"openai"` | Decisions API (`gpt-6-luna`) | `apiKey` with preview | `{ input, questions: [...] }` → `{ answers: [...] }` | Against a response recorded by a preview user; no public schema | + +What `client.ts` does for all of them, so the pipeline sees one behaviour: + +- **Validation.** An answer is used only if its choice is one of the offered options, every + probability is in range, the distribution is consistent (does not exceed 1; sums to 1 when all + options are listed) and the choice is its most likely option. Anything else is a typed error and + the act falls back. +- **Ids and text.** TypeSafe takes JSON instructions and any ids. The others take text and a + restricted id alphabet: question keys and option ids outside it are sent under an alias (the + description keeps the original), and JSON is serialised. Answers always come back under the + caller's own ids. +- **Limits.** A request with more questions than the provider takes (64 Cloudflare and OpenAI, 128 + Perplexity) is split over the same state and merged. A choice among one option is answered + locally as certain and never sent. +- **Failure handling.** Per-provider timeout (8 s TypeSafe, 15 s others), retry on 429/529 honouring + `Retry-After`, and a circuit breaker per provider + endpoint + key (60 s after an auth failure, + 30 s after three consecutive failures, malformed payloads included). + +Thresholds (0.7 accept, 0.9 "none" veto, …) were tuned on Jev. Another model's probabilities are +not calibrated the same way: treat them as starting points and re-run the evals per provider. + +`tests/decisionsProviders.test.ts` runs one request through all four wire formats; +`tests/decisionsCrossProvider.test.ts` runs act, observe, extract and tool scenarios through each; +`DECISIONS_LIVE=1` with any provider key runs `tests/decisionsProvidersLive.test.ts` against the +real services. + ## Flow | Step | Who | Notes | @@ -21,7 +57,7 @@ deliberately not a field of the public create config. Evals build that variable | Candidates | code | Role view per family, then every named element. Described with name, ancestors, heading, card/row text for twins, `n of m`, iframe flag, DOM attributes for nameless controls. Lists over 40 are first cut to the 30 sharing words with the instruction; one exact quoted-name match skips the decision model. | | Pick | the decision model, 1 request (parallel shards on huge lists) | One exact quoted-name match is confirmed with a single small request instead of ranking the list. `best` (no none option) + `strict` (none vetoes above 0.9). A pick strict is uneasy about is held while the next tier tries. Indistinguishable twins share their vote. | | Act | code | Same `Action` shape as the LLM path, so caching and replay are unchanged. Detached fill targets are re-picked once. | -| Checks | code | Fill read-back, native `` `[selected]` flag, no-effect retry on a credible runner-up. `verify: "full"` adds a logged-only decision-model yes/no. | | No target | the decision model, 1 request | Page-state signals; access-denied / captcha fail fast. Otherwise the LLM gets the decision model's shortlist (with each item's card/row) before the whole tree. | Steps that do not depend on each other overlap: the intent request needs no page, so it is asked @@ -39,26 +75,27 @@ replayed, so a selector that now resolves to a different control is re-inferred - `pick.ts` — tiers, pruning, shards, best/strict acceptance. - `tree.ts` — outline parsing, views, candidate descriptions, focus outline, page digest. - `args.ts` — deterministic argument parsing and grounding. -- `pageState.ts`, `cacheCheck.ts`, `typesafeClient.ts`. +- `pageState.ts`, `cacheCheck.ts`, `client.ts`, `providers.ts`. ## Configuration (`experimentalDecisions`) -| Field | Default | Meaning | -| ----------------- | ---------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `apiKey` | required | TypeSafe key. `apiUrl` (https only) and `model` are optional. | -| `enabled` | `true` | `false` keeps only the per-act timing log (eval baselines). | -| `actConfidence` | `0.7` | Minimum confidence to act on a node's answer. | -| `verify` | `"checks"` | `"checks"`: fill read-back + native-select flag. `"full"` adds a logged-only the decision model yes/no. `"off"`: none. | -| `llmFallback` | `true` | `false` fails the act when the decision model abstains: the fastest way to see what the decision model alone gets wrong. | -| `argumentLlm` | `true` | The argument-only LLM call for unquoted text. Independent of `llmFallback`; turn both off for an LLM-free run. The typed text is always the instruction's own characters, never the model's re-cased copy. | -| `pageState` | `true` | Page-state request when the decision model leans toward "not on this page". | -| `cacheCheck` | `false` | Before each cached action is replayed, one decision-model yes/no checks that its selector still points at a matching element; stale ones are re-inferred. Adds a snapshot per cached action, and a request when the selector still resolves in it. | -| `extract` | `"off"` | `"judge"`: the decision model's yes/no replaces extract()'s completion LLM call. `"pick"`: the decision model picks the elements holding each scalar or list field's value and code copies their text; booleans and enums are judged directly; schemas the planner cannot map, unresolved required fields, or a failed completion gate send the whole extraction to the LLM. **Both send page or extracted content to TypeSafe.** | -| `observe` | `false` | Resolve `observe()` through the decision model first. "Find all" is answered exhaustively or handed to the LLM (over 600 candidates; over 400 elements with no instruction), never truncated. | -| `targetReadiness` | `false` | Act as soon as the target is found and staying put, instead of waiting out the DOM-settle heuristic (see below). The settle wait remains the upper bound. | -| `tools` | `false` | Let `act()` invoke a WebMCP tool the page registered when the decision model is sure the tool is the request (see below). Sends tool names and descriptions to the decision provider, and for the two tools sharing most words with the instruction also their parameter names, descriptions, types and enum values. | -| `retryNoEffect` | `false` | Click the runner-up when an ambiguous click provably changed nothing. Off: effects the outline cannot show (aria-pressed, copy, play) look like "nothing". Never cached. | -| `focusFallback` | `false` | On trees over 120K chars, show the LLM the decision model's shortlist first. Off: it found the target in a minority of firings and cost accuracy on ordinary pages. | +| Field | Default | Meaning | +| ----------------- | ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `provider` | `"typesafe"` | `"typesafe"`, `"cloudflare"`, `"perplexity"` or `"openai"` (see Providers). | +| `apiKey` | required | The provider's key. `model` and `apiUrl` (https only) are optional; `accountId` is required for Cloudflare. | +| `enabled` | `true` | `false` keeps only the per-act timing log (eval baselines). | +| `actConfidence` | `0.7` | Minimum confidence to act on a node's answer. | +| `verify` | `"checks"` | `"checks"`: fill read-back + native-select flag. `"full"` adds a logged-only decision-model yes/no. `"off"`: none. | +| `llmFallback` | `true` | `false` fails the act when the decision model abstains: the fastest way to see what the decision model alone gets wrong. | +| `argumentLlm` | `true` | The argument-only LLM call for unquoted text. Independent of `llmFallback`; turn both off for an LLM-free run. The typed text is always the instruction's own characters, never the model's re-cased copy. | +| `pageState` | `true` | Page-state request when the decision model leans toward "not on this page". | +| `cacheCheck` | `false` | Before each cached action is replayed, one decision-model yes/no checks that its selector still points at a matching element; stale ones are re-inferred. Adds a snapshot per cached action, and a request when the selector still resolves in it. | +| `extract` | `"off"` | `"judge"`: the decision model's yes/no replaces extract()'s completion LLM call. `"pick"`: the decision model picks the elements holding each scalar or list field's value and code copies their text; booleans and enums are judged directly; schemas the planner cannot map, unresolved required fields, or a failed completion gate send the whole extraction to the LLM. **Both send page or extracted content to the decision provider.** | +| `observe` | `false` | Resolve `observe()` through the decision model first. "Find all" is answered exhaustively or handed to the LLM (over 600 candidates; over 400 elements with no instruction), never truncated. | +| `targetReadiness` | `false` | Act as soon as the target is found and staying put, instead of waiting out the DOM-settle heuristic (see below). The settle wait remains the upper bound. | +| `tools` | `false` | Let `act()` invoke a WebMCP tool the page registered when the decision model is sure the tool is the request (see below). Sends tool names and descriptions to the decision provider, and for the two tools sharing most words with the instruction also their parameter names, descriptions, types and enum values. | +| `retryNoEffect` | `false` | Click the runner-up when an ambiguous click provably changed nothing. Off: effects the outline cannot show (aria-pressed, copy, play) look like "nothing". Never cached. | +| `focusFallback` | `false` | On trees over 120K chars, show the LLM the decision model's shortlist first. Off: it found the target in a minority of firings and cost accuracy on ordinary pages. | ## Target readiness (`targetReadiness: true`) @@ -132,7 +169,7 @@ the decision model for 79% of requests at 99% precision; arguments filled by the ## What leaves the process -Sent to TypeSafe: the instruction, candidate descriptions built from the accessibility outline +Sent to the decision provider: the instruction, candidate descriptions built from the accessibility outline (names, nearby text, card/row text, DOM attributes of nameless controls), the page URL without query string or fragment, a digest of the first visible content (page state), and — only with their own opt-ins — extracted data (`extract`) and cached action descriptions (`cacheCheck`). diff --git a/packages/extension/services/decisions/cacheCheck.ts b/packages/extension/services/decisions/cacheCheck.ts index 51b6b8fa2..dbae23102 100644 --- a/packages/extension/services/decisions/cacheCheck.ts +++ b/packages/extension/services/decisions/cacheCheck.ts @@ -2,7 +2,7 @@ import type { Action } from "@browserbasehq/stagehand-protocol/types"; import { trimTrailingTextNode } from "../../utils.js"; import { ask, round, type AskContext, type Snapshot } from "./pick.js"; import { describeCandidate } from "./tree.js"; -import { noulAnswer } from "./typesafeClient.js"; +import { noulAnswer } from "./client.js"; /** * Cached actions replay blind: a selector that still resolves, but now points diff --git a/packages/extension/services/decisions/client.ts b/packages/extension/services/decisions/client.ts new file mode 100644 index 000000000..b0db4c41a --- /dev/null +++ b/packages/extension/services/decisions/client.ts @@ -0,0 +1,388 @@ +/** + * One client for every decision-model provider (see providers.ts for the + * wire formats). Raw fetch rather than vendor SDKs: this runs inside the + * extension service worker. + * + * What is shared here, so each provider behaves the same to the pipeline: + * request timeout, retry on overload, a circuit breaker per endpoint + key, + * splitting requests that exceed a provider's question limit, aliasing ids a + * provider would reject, and validating every answer before it is trusted. + */ +import { + providerFor, + type DecisionAnswer, + type DecisionChoiceAnswer, + type DecisionModelConfig, + type DecisionNoulAnswer, + type DecisionProvider, + type DecisionQuestion, + type JsonValue, +} from "./providers.js"; + +export { + DECISION_PROVIDERS, + TYPESAFE_DEFAULT_API_URL, + TYPESAFE_DEFAULT_MODEL, + type DecisionAnswer, + type DecisionChoiceAnswer, + type DecisionModelConfig, + type DecisionNoulAnswer, + type DecisionProviderName, + type DecisionQuestion, + type JsonValue, +} from "./providers.js"; + +export type DecisionResponse = { + model: string; + answers: Record; + usage: { inputTokens: number; outputTokens: number }; + durationMs: number; +}; + +const RETRY_STATUSES = new Set([429, 529]); +const MAX_ATTEMPTS = 3; +const MAX_RETRY_AFTER_MS = 2_000; +const AUTH_STATUSES = new Set([401, 403]); +const BREAKER_MS = 60_000; + +const FAILURES_TO_OPEN = 3; +const OUTAGE_BREAKER_MS = 30_000; +/** Probabilities are rounded by providers; allow each option half a unit of two-decimal rounding. */ +const SUM_SLACK_PER_OPTION = 0.005; + +// A bad key, exhausted quota, or a provider outage fails every act the same +// way; skip the decision model for a while instead of paying a doomed round +// trip before each LLM fallback. Keyed by provider + endpoint + key so one +// tenant's bad key does not pause another's in a shared worker. +const breakers = new Map(); + +function breakerFor(config: DecisionModelConfig, provider: DecisionProvider) { + const key = [provider.name, config.apiUrl ?? "", config.accountId ?? "", config.apiKey].join( + "\u0000", + ); + let breaker = breakers.get(key); + if (!breaker) breakers.set(key, (breaker = { openUntil: 0, failures: 0 })); + return breaker; +} + +export class DecisionRequestError extends Error { + constructor( + message: string, + readonly status?: number, + ) { + super(message); + this.name = "DecisionRequestError"; + } +} + +/** Asks one decision model a set of typed questions about one state. */ +export async function decide( + config: DecisionModelConfig, + state: JsonValue, + questions: Record, +): Promise { + const provider = providerFor(config); + const startedAt = Date.now(); + // A choice among one option has one possible answer (a softmax over one + // option is 1). Providers reject or waste tokens on it, so it never leaves. + const settled: Record = {}; + for (const [key, question] of Object.entries(questions)) { + if (question.type !== "choice") continue; + const options = Object.keys(question.criteria); + if (options.length === 0) throw new DecisionRequestError(`Question "${key}" has no options`); + if (options.length === 1) { + settled[key] = { + type: "choice", + choice: options[0]!, + probabilities: { [options[0]!]: 1 }, + confidence: 1, + }; + } + } + const keys = Object.keys(questions).filter((key) => !(key in settled)); + const chunks: string[][] = []; + for (let start = 0; start < keys.length; start += provider.maxQuestions) { + chunks.push(keys.slice(start, start + provider.maxQuestions)); + } + if (chunks.length === 0) { + return { + model: config.model ?? provider.defaultModel, + answers: settled, + usage: { inputTokens: 0, outputTokens: 0 }, + durationMs: 0, + }; + } + + // Every chunk sees the same state; the answers are independent per question. + const parts = await Promise.all( + chunks.map((chunk) => + requestOnce( + config, + provider, + state, + Object.fromEntries(chunk.map((key) => [key, questions[key]!])), + ), + ), + ); + return { + model: parts[0]!.model, + answers: Object.assign({}, settled, ...parts.map((part) => part.answers)) as Record< + string, + DecisionAnswer + >, + usage: { + inputTokens: parts.reduce((sum, part) => sum + part.usage.inputTokens, 0), + outputTokens: parts.reduce((sum, part) => sum + part.usage.outputTokens, 0), + }, + durationMs: Date.now() - startedAt, + }; +} + +type Aliases = { + questions: Record; + /** wire question key → caller's key */ + keyOf: Map; + /** caller's key → (wire option id → caller's option id) */ + optionOf: Map>; +}; + +function text(value: JsonValue): JsonValue { + return typeof value === "string" || value === null ? value : JSON.stringify(value); +} + +/** + * Rewrites questions into what the provider accepts: ids outside its alphabet + * get an alias (the description still says what the option is), and JSON + * instructions or descriptions become text where only text is taken. + */ +function prepare(provider: DecisionProvider, questions: Record): Aliases { + const keyOf = new Map(); + const optionOf = new Map>(); + const prepared: Record = {}; + const flatten = (value: JsonValue): JsonValue => (provider.structuredText ? value : text(value)); + let index = 0; + for (const [key, question] of Object.entries(questions)) { + const wireKey = provider.safeId.test(key) ? key : `q${index}`; + index++; + keyOf.set(wireKey, key); + if (question.type === "noul") { + prepared[wireKey] = { + type: "noul", + instructions: flatten(question.instructions), + ...(question.criteria + ? { + criteria: { + true: flatten(question.criteria.true), + false: flatten(question.criteria.false), + }, + } + : {}), + }; + continue; + } + const options = new Map(); + const ids = Object.keys(question.criteria); + const aliasAll = ids.some((id) => !provider.safeId.test(id)); + const criteria: Record = {}; + ids.forEach((id, position) => { + const wireId = aliasAll ? `o${position}` : id; + options.set(wireId, id); + const description = question.criteria[id]!; + // An aliased id no longer says anything: keep the original in the description. + criteria[wireId] = aliasAll + ? flatten( + description === null || description === "" + ? id + : typeof description === "string" + ? `${id}: ${description}` + : { + id, + ...(typeof description === "object" && !Array.isArray(description) + ? description + : { is: description }), + }, + ) + : flatten(description); + }); + optionOf.set(key, options); + prepared[wireKey] = { type: "choice", instructions: flatten(question.instructions), criteria }; + } + return { questions: prepared, keyOf, optionOf }; +} + +function probability(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) && value >= 0 && value <= 1 + ? value + : undefined; +} + +/** A provider's answer, checked against the question that was asked. Throws on anything unusable. */ +function validate( + provider: DecisionProvider, + key: string, + question: DecisionQuestion, + raw: unknown, + options: Map | undefined, +): DecisionAnswer { + const bad = (why: string): never => { + throw new DecisionRequestError( + `${provider.label} returned an invalid answer for "${key}" (${why})`, + ); + }; + const answer = raw && typeof raw === "object" ? (raw as Record) : bad("missing"); + if (question.type === "noul") { + const noul = probability(answer.noul ?? answer.probability); + return { type: "noul", noul: noul ?? bad("noul out of range") }; + } + const original = (wireId: unknown): string | undefined => + typeof wireId === "string" ? options?.get(wireId) : undefined; + const choice = original(answer.choice) ?? bad("choice is not one of the options"); + const probabilities: Record = {}; + const listed = + answer.probabilities && typeof answer.probabilities === "object" + ? Object.entries(answer.probabilities as Record) + : []; + for (const [wireId, value] of listed) { + const id = original(wireId) ?? bad("probability for an unknown option"); + probabilities[id] = probability(value) ?? bad("probability out of range"); + } + // Some providers report only the chosen option's probability. + if (listed.length === 0) { + probabilities[choice] = + probability(answer.probability ?? answer.confidence) ?? bad("no probabilities"); + } + if (probabilities[choice] === undefined) bad("no probability for the choice"); + const total = Object.values(probabilities).reduce((sum, value) => sum + value, 0); + const slack = 1e-6 + (options?.size ?? 1) * SUM_SLACK_PER_OPTION; + if (total > 1 + slack) bad("probabilities exceed 1"); + if (options && listed.length === options.size && total < 1 - slack) + bad("probabilities do not sum to 1"); + const top = Math.max(...Object.values(probabilities)); + if (probabilities[choice]! < top - 1e-6) bad("choice is not the most likely option"); + return { + type: "choice", + choice, + probabilities, + confidence: probability(answer.confidence) ?? probabilities[choice]!, + }; +} + +async function requestOnce( + config: DecisionModelConfig, + provider: DecisionProvider, + state: JsonValue, + questions: Record, +): Promise> { + const aliases = prepare(provider, questions); + const request = provider.encode(config, state, aliases.questions); + if (!request.url.startsWith("https://")) { + throw new DecisionRequestError(`${provider.label} apiUrl must be https`); + } + const body = JSON.stringify(request.body); + const breaker = breakerFor(config, provider); + if (Date.now() < breaker.openUntil) { + throw new DecisionRequestError(`${provider.label} requests are paused after repeated failures`); + } + const failed = () => { + if (++breaker.failures >= FAILURES_TO_OPEN) { + breaker.openUntil = Date.now() + OUTAGE_BREAKER_MS; + breaker.failures = 0; + } + }; + + for (let attempt = 1; ; attempt++) { + let response: Response; + try { + response = await fetch(request.url, { + signal: AbortSignal.timeout(provider.defaultTimeoutMs), + method: "POST", + headers: request.headers, + body, + }); + } catch (error) { + failed(); + const cause = error instanceof Error ? error.name : "network"; + throw new DecisionRequestError( + cause === "TimeoutError" + ? `${provider.label} decision request timed out after ${provider.defaultTimeoutMs} ms` + : `${provider.label} decision request failed (${cause})`, + ); + } + + if (response.ok) { + // A 2xx with a body that is not the documented shape (an intermediary's + // HTML, a truncated response) or with answers that do not fit the + // questions must surface as a typed error, not as something the + // pipeline acts on. It also counts as a failure: a provider returning + // nonsense is an outage too. + const decoded = provider.decode(await response.json().catch(() => undefined)); + if (!decoded) { + failed(); + throw new DecisionRequestError( + `${provider.label} response was not a decision payload`, + response.status, + ); + } + const answers: Record = {}; + try { + for (const [wireKey, key] of aliases.keyOf) { + answers[key] = validate( + provider, + key, + questions[key]!, + decoded.answers[wireKey], + aliases.optionOf.get(key), + ); + } + } catch (error) { + failed(); + throw error; + } + breaker.failures = 0; + return { model: decoded.model, answers, usage: decoded.usage }; + } + + if (RETRY_STATUSES.has(response.status) && attempt < MAX_ATTEMPTS) { + const retryAfter = Number(response.headers.get("retry-after")); + const wait = + Number.isFinite(retryAfter) && retryAfter > 0 + ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) + : 250 * 2 ** (attempt - 1); + await new Promise((resolve) => setTimeout(resolve, wait)); + continue; + } + + if (AUTH_STATUSES.has(response.status)) { + breaker.openUntil = Date.now() + BREAKER_MS; + breaker.failures = 0; + } else if (response.status >= 500 || RETRY_STATUSES.has(response.status)) { + failed(); + } else { + // A terminal client error (bad request) is not an outage: it must not + // let earlier counted failures open the outage breaker later. + breaker.failures = 0; + } + // Only the provider's error code reaches logs, never the raw body. + const detail = provider.errorCode(await response.text().catch(() => "")); + throw new DecisionRequestError( + `${provider.label} decision request failed (${response.status}${detail ? `: ${detail}` : ""})`, + response.status, + ); + } +} + +export function choiceAnswer(response: DecisionResponse, key: string): DecisionChoiceAnswer { + const answer = response.answers[key]; + if (!answer || !("choice" in answer)) { + throw new DecisionRequestError(`Decision response is missing choice answer "${key}"`); + } + return answer; +} + +export function noulAnswer(response: DecisionResponse, key: string): DecisionNoulAnswer { + const answer = response.answers[key]; + if (!answer || !("noul" in answer)) { + throw new DecisionRequestError(`Decision response is missing noul answer "${key}"`); + } + return answer; +} diff --git a/packages/extension/services/decisions/extract.ts b/packages/extension/services/decisions/extract.ts index 5c934ae03..583bc7b24 100644 --- a/packages/extension/services/decisions/extract.ts +++ b/packages/extension/services/decisions/extract.ts @@ -12,12 +12,7 @@ import { type TraceEntry, } from "./pick.js"; import { buildView, pageDigest, textOf, type OutlineNode } from "./tree.js"; -import { - choiceAnswer, - noulAnswer, - type DecisionModelConfig, - type JsonValue, -} from "./typesafeClient.js"; +import { choiceAnswer, noulAnswer, type DecisionModelConfig, type JsonValue } from "./client.js"; /** * Experimental extract() on the decision model: pick-and-copy. The decision model cannot write text, but an diff --git a/packages/extension/services/decisions/extractCheck.ts b/packages/extension/services/decisions/extractCheck.ts index 594da824e..17b366d3f 100644 --- a/packages/extension/services/decisions/extractCheck.ts +++ b/packages/extension/services/decisions/extractCheck.ts @@ -1,5 +1,5 @@ import { ask, round, type AskContext } from "./pick.js"; -import { noulAnswer } from "./typesafeClient.js"; +import { noulAnswer } from "./client.js"; const MAX_EXTRACTED_CHARS = 6000; diff --git a/packages/extension/services/decisions/observe.ts b/packages/extension/services/decisions/observe.ts index d3ecdb1fc..4cc1045a7 100644 --- a/packages/extension/services/decisions/observe.ts +++ b/packages/extension/services/decisions/observe.ts @@ -29,7 +29,7 @@ import { type OutlineNode, type ViewKind, } from "./tree.js"; -import { choiceAnswer, noulAnswer, type DecisionModelConfig } from "./typesafeClient.js"; +import { choiceAnswer, noulAnswer, type DecisionModelConfig } from "./client.js"; /** * Experimental observe() on the decision model. observe() answers "which element(s), and how diff --git a/packages/extension/services/decisions/pageState.ts b/packages/extension/services/decisions/pageState.ts index 4499fd997..e8e28beea 100644 --- a/packages/extension/services/decisions/pageState.ts +++ b/packages/extension/services/decisions/pageState.ts @@ -1,6 +1,6 @@ import { ask, round, type AskContext } from "./pick.js"; import { pageDigest, type OutlineNode } from "./tree.js"; -import { noulAnswer } from "./typesafeClient.js"; +import { noulAnswer } from "./client.js"; /** * Page-state signals from one decision-model request over a compact page digest. Used to diff --git a/packages/extension/services/decisions/pick.ts b/packages/extension/services/decisions/pick.ts index 8789bafb0..36972481d 100644 --- a/packages/extension/services/decisions/pick.ts +++ b/packages/extension/services/decisions/pick.ts @@ -12,12 +12,12 @@ import { import { choiceAnswer, noulAnswer, - systemOne, + decide, type DecisionModelConfig, type DecisionQuestion, type DecisionResponse, type JsonValue, -} from "./typesafeClient.js"; +} from "./client.js"; export type TraceEntry = Record; @@ -39,7 +39,7 @@ export type AskContext = { /** Called once per tier right before the decision model is asked, e.g. to fetch DOM hints only when needed. */ prepare?: (candidates: OutlineNode[]) => Promise; /** - * Applied to everything sent to TypeSafe and to the trace. Values an earlier + * Applied to everything sent to the decision provider and to the trace. Values an earlier * act typed from %variables% are on the page by now; they must not leave it. */ redact?: (text: string) => string; @@ -100,7 +100,7 @@ export async function ask( state = redactDeep(state, ctx.redact); questions = redactDeep(questions, ctx.redact); } - const response = await systemOne(ctx.config, state, questions); + const response = await decide(ctx.config, state, questions); ctx.trace.push({ node, ms: response.durationMs, diff --git a/packages/extension/services/decisions/pipeline.ts b/packages/extension/services/decisions/pipeline.ts index 893380b4c..7472cd84c 100644 --- a/packages/extension/services/decisions/pipeline.ts +++ b/packages/extension/services/decisions/pipeline.ts @@ -49,7 +49,7 @@ import { noulAnswer, type DecisionModelConfig, type DecisionResponse, -} from "./typesafeClient.js"; +} from "./client.js"; export { fillValueCandidates, parseKey, parsePercent } from "./args.js"; @@ -69,7 +69,7 @@ export type DecisionsConfig = DecisionModelConfig & { actConfidence?: number; /** * `"checks"` (default): deterministic checks only — fill read-back and the - * native-select flag. `"full"` adds a logged-only the decision model yes/no over the page + * native-select flag. `"full"` adds a logged-only decision-model yes/no over the page * diff. `"off"` disables every check. */ verify?: "off" | "checks" | "full"; @@ -98,7 +98,7 @@ export type DecisionsConfig = DecisionModelConfig & { * extract() on the decision model. `"judge"`: only the completion yes/no replaces the second * LLM call. `"pick"`: the decision model picks the elements holding each field's value and * code copies their text; the LLM extracts only when that does not fit. - * Both send page or extracted content to TypeSafe. Default `"off"`. + * Both send page or extracted content to the decision provider. Default `"off"`. */ extract?: "off" | "judge" | "pick"; /** Resolve observe() through the decision model first. Default false. */ @@ -114,7 +114,7 @@ export type DecisionsConfig = DecisionModelConfig & { * Let act() invoke a WebMCP tool the page registered when the decision model is sure the * tool is the request. Sends tool names and descriptions, and for the likely * tools their parameter names, descriptions, types and enum values, to - * TypeSafe. Default false. + * the decision provider. Default false. */ tools?: boolean; }; diff --git a/packages/extension/services/decisions/providers.ts b/packages/extension/services/decisions/providers.ts new file mode 100644 index 000000000..d9ed405a6 --- /dev/null +++ b/packages/extension/services/decisions/providers.ts @@ -0,0 +1,286 @@ +/** + * Decision-model providers. A decision model answers typed questions about a + * state (pick one option, yes/no) with probabilities instead of text. Three of + * the four providers here speak the same "System One" request shape; OpenAI's + * Decisions API carries the same information in a different one. Each provider + * is a codec: how to address it, how to encode a request, how to read answers. + * Everything else (timeouts, retries, circuit breaking, validation, splitting + * oversized requests) is shared in client.ts. + */ + +export type JsonValue = + | string + | number + | boolean + | null + | JsonValue[] + | { [key: string]: JsonValue }; + +export type DecisionQuestion = + | { type: "choice"; instructions: JsonValue; criteria: Record } + | { type: "noul"; instructions: JsonValue; criteria?: { true: JsonValue; false: JsonValue } }; + +export type DecisionChoiceAnswer = { + type: "choice"; + choice: string; + probabilities: Record; + confidence: number; +}; + +export type DecisionNoulAnswer = { type: "noul"; noul: number }; + +export type DecisionAnswer = DecisionChoiceAnswer | DecisionNoulAnswer; + +export const DECISION_PROVIDERS = ["typesafe", "cloudflare", "perplexity", "openai"] as const; +export type DecisionProviderName = (typeof DECISION_PROVIDERS)[number]; + +export type DecisionModelConfig = { + /** Which decision model service to call. Default `"typesafe"`. */ + provider?: DecisionProviderName; + apiKey: string; + model?: string; + /** Overrides the provider's base URL (https only). */ + apiUrl?: string; + /** Cloudflare account id; required for `provider: "cloudflare"` unless `apiUrl` names the account. */ + accountId?: string; +}; + +export type EncodedRequest = { url: string; headers: Record; body: JsonValue }; + +/** Answers as the provider returned them, keyed by the question keys that were sent. */ +export type DecodedResponse = { + model: string; + answers: Record; + usage: { inputTokens: number; outputTokens: number }; +}; + +export type DecisionProvider = { + name: DecisionProviderName; + /** Used in error messages and logs. */ + label: string; + defaultModel: string; + defaultTimeoutMs: number; + /** A request with more questions is split into several (same state) and the answers merged. */ + maxQuestions: number; + /** + * Question keys and option ids the provider accepts as they are. Others are + * sent under a generated alias and mapped back, so callers never care. + */ + safeId: RegExp; + /** Whether instructions and option descriptions may be JSON values, or must be text. */ + structuredText: boolean; + encode( + config: DecisionModelConfig, + state: JsonValue, + questions: Record, + ): EncodedRequest; + /** Undefined when the payload is not this provider's success shape. */ + decode(payload: unknown): DecodedResponse | undefined; + /** A short machine-readable reason from an error body; never the raw body. */ + errorCode(body: string): string | undefined; +}; + +const ANY_ID = /^[\s\S]+$/; +/** Cloudflare documents this alphabet for question ids; it is also a safe floor for the others. */ +const PLAIN_ID = /^[A-Za-z0-9_.-]{1,100}$/; + +function base(config: DecisionModelConfig, fallback: string): string { + return (config.apiUrl ?? fallback).replace(/\/+$/, ""); +} + +function bearer(config: DecisionModelConfig): Record { + return { Authorization: `Bearer ${config.apiKey}`, "Content-Type": "application/json" }; +} + +function record(value: unknown): Record | undefined { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function usageOf(payload: Record): DecodedResponse["usage"] { + const usage = record(payload.usage) ?? {}; + const number = (value: unknown): number => (typeof value === "number" ? value : 0); + return { + inputTokens: number(usage.input_tokens ?? usage.prompt_tokens), + outputTokens: number(usage.output_tokens ?? usage.completion_tokens), + }; +} + +/** `{ model, answers: { key: answer }, usage }`, optionally inside Workers AI's `{ result }` envelope. */ +function decodeSystemOne(payload: unknown): DecodedResponse | undefined { + const outer = record(payload); + const body = outer && record(outer.result) ? record(outer.result)! : outer; + const answers = body && record(body.answers); + if (!body || !answers) return undefined; + return { + model: typeof body.model === "string" ? body.model : "", + answers, + usage: usageOf(body), + }; +} + +function errorType(body: string): string | undefined { + return /"(?:error_type|type|code)"\s*:\s*"([\w.-]{1,64})"/.exec(body)?.[1]; +} + +export const TYPESAFE_DEFAULT_API_URL = "https://api.typesafe.ai"; +export const TYPESAFE_DEFAULT_MODEL = "jev-latest"; + +const typesafe: DecisionProvider = { + name: "typesafe", + label: "TypeSafe", + defaultModel: TYPESAFE_DEFAULT_MODEL, + defaultTimeoutMs: 8_000, + // No documented limit; requests are sent whole, as they always were. + maxQuestions: 1_000, + safeId: ANY_ID, + structuredText: true, + encode: (config, state, questions) => ({ + url: `${base(config, TYPESAFE_DEFAULT_API_URL)}/v1/systemone`, + headers: bearer(config), + body: { state, model: config.model ?? TYPESAFE_DEFAULT_MODEL, questions }, + }), + decode: decodeSystemOne, + errorCode: (body) => /"error_type"\s*:\s*"([\w-]{1,64})"/.exec(body)?.[1], +}; + +/** Clef on Workers AI: System One body, account-scoped URL, answers inside `{ result }`. */ +const cloudflare: DecisionProvider = { + name: "cloudflare", + label: "Cloudflare", + defaultModel: "clef", + defaultTimeoutMs: 15_000, + maxQuestions: 64, + safeId: PLAIN_ID, + structuredText: false, + encode: (config, state, questions) => { + const model = config.model ?? "clef"; + const slug = model.startsWith("@") ? model : `@cf/cloudflare/${model}`; + let root: string; + if (config.apiUrl) root = base(config, ""); + else if (config.accountId) { + root = `https://api.cloudflare.com/client/v4/accounts/${encodeURIComponent(config.accountId)}/ai/run`; + } else { + throw new Error('experimentalDecisions: provider "cloudflare" needs accountId (or apiUrl)'); + } + return { + url: `${root}/${slug}`, + headers: bearer(config), + body: { model: slug.slice(slug.lastIndexOf("/") + 1), state, questions }, + }; + }, + decode: decodeSystemOne, + errorCode: (body) => /"code"\s*:\s*(\d{3,6})/.exec(body)?.[1] ?? errorType(body), +}; + +const perplexity: DecisionProvider = { + name: "perplexity", + label: "Perplexity", + defaultModel: "pplx-decider-v1-27b", + defaultTimeoutMs: 15_000, + maxQuestions: 128, + safeId: PLAIN_ID, + structuredText: false, + encode: (config, state, questions) => ({ + url: `${base(config, "https://api.perplexity.ai")}/v1/decisions`, + headers: bearer(config), + body: { model: config.model ?? "pplx-decider-v1-27b", state, questions }, + }), + decode: decodeSystemOne, + errorCode: errorType, +}; + +/** + * OpenAI's Decisions API (limited preview). No schema is published; this + * follows the request and response recorded by preview users and OpenAI's own + * client: one `input` string, a list of named questions (`predicate`, + * `choice`), and probabilities as lists. Yes/no criteria have no field of + * their own and are folded into the instructions. + */ +const openai: DecisionProvider = { + name: "openai", + label: "OpenAI", + defaultModel: "gpt-6-luna", + defaultTimeoutMs: 15_000, + maxQuestions: 64, + safeId: PLAIN_ID, + structuredText: false, + encode: (config, state, questions) => ({ + url: `${base(config, "https://api.openai.com")}/v1/decisions`, + headers: bearer(config), + body: { + model: config.model ?? "gpt-6-luna", + input: typeof state === "string" ? state : JSON.stringify(state), + questions: Object.entries(questions).map( + ([name, question]): JsonValue => + question.type === "noul" + ? { + type: "predicate", + name, + instructions: question.criteria + ? `${String(question.instructions)}\nYes: ${String(question.criteria.true)}\nNo: ${String(question.criteria.false)}` + : question.instructions, + } + : { + type: "choice", + name, + instructions: question.instructions, + choices: Object.entries(question.criteria).map( + ([value, description]): JsonValue => + description === null || description === "" ? { value } : { value, description }, + ), + }, + ), + }, + }), + decode: (payload) => { + const body = record(payload); + if (!body || !Array.isArray(body.answers)) return undefined; + const answers: Record = {}; + for (const entry of body.answers) { + const answer = record(entry); + if (!answer || typeof answer.name !== "string") return undefined; + if (answer.type === "predicate") { + answers[answer.name] = { type: "noul", noul: answer.probability }; + } else if (answer.type === "choice" && Array.isArray(answer.probabilities)) { + const probabilities: Record = {}; + for (const item of answer.probabilities) { + const option = record(item); + if (!option || typeof option.value !== "string") return undefined; + probabilities[option.value] = option.probability; + } + answers[answer.name] = { + type: "choice", + choice: answer.choice, + probabilities, + confidence: answer.confidence, + }; + } else { + answers[answer.name] = answer; + } + } + return { + model: typeof body.model === "string" ? body.model : "", + answers, + usage: usageOf(body), + }; + }, + errorCode: (body) => + /Decision API is not enabled/i.test(body) ? "decisions_not_enabled" : errorType(body), +}; + +const PROVIDERS: Record = { + typesafe, + cloudflare, + perplexity, + openai, +}; + +export function providerFor(config: DecisionModelConfig): DecisionProvider { + const provider = PROVIDERS[config.provider ?? "typesafe"]; + if (!provider) { + throw new Error(`experimentalDecisions: unknown provider "${String(config.provider)}"`); + } + return provider; +} diff --git a/packages/extension/services/decisions/toolAct.ts b/packages/extension/services/decisions/toolAct.ts index 1f590d34c..056e0355b 100644 --- a/packages/extension/services/decisions/toolAct.ts +++ b/packages/extension/services/decisions/toolAct.ts @@ -6,7 +6,7 @@ import type { import type { Page } from "../../understudy/page.js"; import { redactor, substituteVariables } from "./args.js"; import type { TraceEntry } from "./pick.js"; -import type { JsonValue } from "./typesafeClient.js"; +import type { JsonValue } from "./client.js"; /** * act() through a WebMCP tool: when the page registers tools and the decision model is sure diff --git a/packages/extension/services/decisions/tools.ts b/packages/extension/services/decisions/tools.ts index d8838fea3..c25445a40 100644 --- a/packages/extension/services/decisions/tools.ts +++ b/packages/extension/services/decisions/tools.ts @@ -6,7 +6,7 @@ import { type DecisionQuestion, type DecisionResponse, type JsonValue, -} from "./typesafeClient.js"; +} from "./client.js"; /** * WebMCP tool selection on the decision model. A page that registers tools has already said diff --git a/packages/extension/services/decisions/tree.ts b/packages/extension/services/decisions/tree.ts index febf58d5f..e63e2cd33 100644 --- a/packages/extension/services/decisions/tree.ts +++ b/packages/extension/services/decisions/tree.ts @@ -1,4 +1,4 @@ -import type { JsonValue } from "./typesafeClient.js"; +import type { JsonValue } from "./client.js"; /** * Parses the hybrid snapshot outline back into nodes so the Decisions act pipeline diff --git a/packages/extension/services/decisions/typesafeClient.ts b/packages/extension/services/decisions/typesafeClient.ts deleted file mode 100644 index 16cbaa41f..000000000 --- a/packages/extension/services/decisions/typesafeClient.ts +++ /dev/null @@ -1,190 +0,0 @@ -/** - * Minimal client for TypeSafe's System One endpoint (`POST /v1/systemone`). - * Raw fetch rather than @typesafe-ai/sdk: the SDK targets Node 20+ and this - * code runs inside the extension service worker. - */ - -export type JsonValue = - | string - | number - | boolean - | null - | JsonValue[] - | { [key: string]: JsonValue }; - -export type DecisionQuestion = - | { type: "choice"; instructions: JsonValue; criteria: Record } - | { type: "noul"; instructions: JsonValue; criteria?: { true: JsonValue; false: JsonValue } }; - -export type DecisionChoiceAnswer = { - type: "choice"; - choice: string; - probabilities: Record; - confidence: number; -}; - -export type DecisionNoulAnswer = { type: "noul"; noul: number }; - -export type DecisionAnswer = DecisionChoiceAnswer | DecisionNoulAnswer; - -export type DecisionModelConfig = { apiKey: string; model?: string; apiUrl?: string }; - -export type DecisionResponse = { - model: string; - answers: Record; - usage: { inputTokens: number; outputTokens: number }; - durationMs: number; -}; - -export const TYPESAFE_DEFAULT_API_URL = "https://api.typesafe.ai"; -export const TYPESAFE_DEFAULT_MODEL = "jev-latest"; - -const RETRY_STATUSES = new Set([429, 529]); -const MAX_ATTEMPTS = 3; -const REQUEST_TIMEOUT_MS = 8_000; -const AUTH_STATUSES = new Set([401, 403]); -const BREAKER_MS = 60_000; - -const FAILURES_TO_OPEN = 3; -const OUTAGE_BREAKER_MS = 30_000; - -// A bad key, exhausted quota, or a TypeSafe outage fails every act the same -// way; skip the decision model for a while instead of paying a doomed (up to 8 s) round trip -// before each LLM fallback. Keyed by endpoint + key so one tenant's bad key -// does not pause another's in a shared worker. -const breakers = new Map(); - -function breakerFor(config: DecisionModelConfig) { - const key = `${config.apiUrl ?? TYPESAFE_DEFAULT_API_URL}\u0000${config.apiKey}`; - let breaker = breakers.get(key); - if (!breaker) breakers.set(key, (breaker = { openUntil: 0, failures: 0 })); - return breaker; -} - -export class DecisionRequestError extends Error { - constructor( - message: string, - readonly status?: number, - ) { - super(message); - this.name = "DecisionRequestError"; - } -} - -export async function systemOne( - config: DecisionModelConfig, - state: JsonValue, - questions: Record, -): Promise { - const url = `${(config.apiUrl ?? TYPESAFE_DEFAULT_API_URL).replace(/\/+$/, "")}/v1/systemone`; - const body = JSON.stringify({ state, model: config.model ?? TYPESAFE_DEFAULT_MODEL, questions }); - const startedAt = Date.now(); - const breaker = breakerFor(config); - if (startedAt < breaker.openUntil) { - throw new DecisionRequestError("TypeSafe requests are paused after repeated failures"); - } - const failed = () => { - if (++breaker.failures >= FAILURES_TO_OPEN) { - breaker.openUntil = Date.now() + OUTAGE_BREAKER_MS; - breaker.failures = 0; - } - }; - - for (let attempt = 1; ; attempt++) { - let response: Response; - try { - response = await fetch(url, { - signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS), - method: "POST", - headers: { - Authorization: `Bearer ${config.apiKey}`, - "Content-Type": "application/json", - }, - body, - }); - } catch (error) { - failed(); - const cause = error instanceof Error ? error.name : "network"; - throw new DecisionRequestError( - cause === "TimeoutError" - ? `TypeSafe systemone request timed out after ${REQUEST_TIMEOUT_MS} ms` - : `TypeSafe systemone request failed (${cause})`, - ); - } - - if (response.ok) { - // A 2xx with a body that is not the documented shape (an intermediary's - // HTML, a truncated response) must surface as a typed error, not as an - // undefined `answers` that the answer helpers trip over later. - const payload = (await response.json().catch(() => undefined)) as - | { - model?: string; - answers?: Record; - usage?: { input_tokens?: number; output_tokens?: number }; - } - | undefined; - if ( - !payload || - typeof payload !== "object" || - !payload.answers || - typeof payload.answers !== "object" - ) { - failed(); - throw new DecisionRequestError( - "TypeSafe systemone response was not a systemone payload", - response.status, - ); - } - breaker.failures = 0; - return { - model: payload.model ?? "", - answers: payload.answers, - usage: { - inputTokens: payload.usage?.input_tokens ?? 0, - outputTokens: payload.usage?.output_tokens ?? 0, - }, - durationMs: Date.now() - startedAt, - }; - } - - if (RETRY_STATUSES.has(response.status) && attempt < MAX_ATTEMPTS) { - await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** (attempt - 1))); - continue; - } - - if (AUTH_STATUSES.has(response.status)) { - breaker.openUntil = Date.now() + BREAKER_MS; - breaker.failures = 0; - } else if (response.status >= 500 || RETRY_STATUSES.has(response.status)) { - failed(); - } else { - // A terminal client error (bad request) is not an outage: it must not - // let earlier counted failures open the outage breaker later. - breaker.failures = 0; - } - // Only the vendor's error code reaches logs, never the raw body. - const detail = (await response.text().catch(() => "")).match( - /"error_type"\s*:\s*"([\w-]{1,64})"/, - ); - throw new DecisionRequestError( - `TypeSafe systemone request failed (${response.status}${detail ? `: ${detail[1]}` : ""})`, - response.status, - ); - } -} - -export function choiceAnswer(response: DecisionResponse, key: string): DecisionChoiceAnswer { - const answer = response.answers[key]; - if (!answer || !("choice" in answer)) { - throw new DecisionRequestError(`TypeSafe response is missing choice answer "${key}"`); - } - return answer; -} - -export function noulAnswer(response: DecisionResponse, key: string): DecisionNoulAnswer { - const answer = response.answers[key]; - if (!answer || !("noul" in answer)) { - throw new DecisionRequestError(`TypeSafe response is missing noul answer "${key}"`); - } - return answer; -} diff --git a/packages/extension/tests/decisionsCrossProvider.test.ts b/packages/extension/tests/decisionsCrossProvider.test.ts new file mode 100644 index 000000000..348e9c219 --- /dev/null +++ b/packages/extension/tests/decisionsCrossProvider.test.ts @@ -0,0 +1,268 @@ +import { trace } from "@opentelemetry/api"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { + Action, + ActResultData, + WebMCPToolDescriptor, +} from "@browserbasehq/stagehand-protocol/types"; +import { StagehandLogger } from "../logger.js"; +import { DECISION_PROVIDERS, type DecisionProviderName } from "../services/decisions/client.js"; +import { runDecisionsExtract } from "../services/decisions/extract.js"; +import { runDecisionsObserve } from "../services/decisions/observe.js"; +import { + runDecisionsAct, + type DecisionsActDeps, + type DecisionsConfig, +} from "../services/decisions/pipeline.js"; +import { parseOutline } from "../services/decisions/tree.js"; +import type { Page } from "../understudy/page.js"; +import { + fakeDecisionProvider, + type ScriptedAnswer, + type WireQuestion, +} from "./decisionsTestUtils.js"; + +vi.mock("../understudy/deepLocator.js", () => ({ + resolveLocatorWithHops: vi.fn(async () => ({ inputValue: async () => "Ada" })), +})); + +/** + * The same act / observe / extract / tool scenarios through every provider's + * wire format. The scripted model answers by what a question says, not by + * its key, because providers with a strict id alphabet see aliased keys. + */ +const SHOP = [ + "[0-1] RootWebArea: Shop", + " [0-3] textbox: Email", + " [0-4] textbox: Name", + " [0-5] button: Add to cart", + " [0-7] button: Checkout", + " [0-9] link: Help", + " [0-11] paragraph", + " [0-12] StaticText: Price:", + " [0-13] StaticText: $1,249.50", +].join("\n"); +const XPATHS = Object.fromEntries( + [3, 4, 5, 7, 9, 11, 12, 13].map((n) => [`0-${n}`, `/html/body/*[${n}]`]), +); + +function config(provider: DecisionProviderName): DecisionsConfig { + return { + provider, + apiKey: `key-${provider}-${Math.random()}`, + ...(provider === "cloudflare" ? { accountId: "acct123" } : {}), + }; +} + +function logger() { + return new StagehandLogger( + { tracer: trace.getTracer("decisions-cross-provider-test") }, + () => {}, + ); +} + +function actDeps(instruction: string, overrides: Partial = {}) { + const takeAction = vi.fn( + async (action: Action): Promise => ({ + success: true, + message: "ok", + actionDescription: action.description, + actions: [action], + }), + ); + const deps: DecisionsActDeps = { + page: { + captureSnapshot: vi.fn(async () => ({ + combinedTree: SHOP, + combinedXpathMap: XPATHS, + combinedUrlMap: {}, + })), + mainFrame: () => ({}), + url: () => "https://shop.test/cart?session=abc", + } as unknown as Page, + logger: logger(), + instruction, + snapshotOptions: {}, + ensureTimeRemaining: () => {}, + takeAction, + ...overrides, + }; + return { deps, takeAction }; +} + +/** Picks the option whose description mentions `needle`. */ +function optionAbout(question: WireQuestion, needle: string): string | undefined { + return Object.entries(question.descriptions).find(([, text]) => text.includes(needle))?.[0]; +} + +function intent(family: string) { + return (key: string): ScriptedAnswer | undefined => + key === "family" ? { choice: family, p: 0.98 } : undefined; +} + +function serve( + provider: DecisionProviderName, + script: (key: string, question: WireQuestion) => ScriptedAnswer | undefined, +) { + return fakeDecisionProvider(provider, script, (fetchMock) => + vi.stubGlobal("fetch", vi.fn(fetchMock)), + ); +} + +afterEach(() => vi.unstubAllGlobals()); + +describe.each(DECISION_PROVIDERS)("decisions pipeline on %s", (provider) => { + it("clicks the element the model picked", async () => { + const seen = serve(provider, (key, question) => { + if (key === "best" || key === "strict") { + return { choice: optionAbout(question, "Checkout"), p: 0.96 }; + } + return intent("click")(key); + }); + const { deps, takeAction } = actDeps("click the Checkout button"); + const outcome = await runDecisionsAct(config(provider), deps); + expect(outcome.kind).toBe("done"); + expect(takeAction).toHaveBeenCalledTimes(1); + expect(takeAction.mock.calls[0]![0]).toMatchObject({ + selector: "xpath=/html/body/*[7]", + method: "click", + }); + // Whatever the wire shape, the page URL leaves without its query string. + expect(JSON.stringify(seen.map((request) => request.body))).not.toContain("session=abc"); + }); + + it("types the quoted value into the field the model picked", async () => { + serve(provider, (key, question) => { + if (key === "fill_value") return { choice: optionAbout(question, "Ada"), p: 0.97 }; + if (key === "best" || key === "strict") + return { choice: optionAbout(question, "Name"), p: 0.95 }; + return intent("fill")(key); + }); + const { deps, takeAction } = actDeps("type 'Ada' into the Name field"); + const outcome = await runDecisionsAct(config(provider), deps); + expect(outcome.kind).toBe("done"); + expect(takeAction.mock.calls[0]![0]).toMatchObject({ + selector: "xpath=/html/body/*[4]", + method: "fill", + arguments: ["Ada"], + }); + }); + + it("hands off to the LLM when the model says nothing matches", async () => { + serve(provider, (key, question) => { + if (key === "strict") return { choice: "none_match", p: 0.97 }; + // Unsure, but still a real distribution: the leader is the most likely option. + if (key === "best") return { choice: question.options[0], p: 0.45 }; + return intent("click")(key); + }); + const { deps, takeAction } = actDeps("click the Wishlist button"); + const outcome = await runDecisionsAct(config(provider), deps); + expect(outcome.kind).toBe("fallback"); + expect(takeAction).not.toHaveBeenCalled(); + }); + + it("observes every element the model marks relevant", async () => { + serve(provider, (key, question) => { + if (key === "family") return { choice: "fill", p: 0.97 }; + if (key === "cardinality") return { choice: "several", p: 0.96 }; + if (question.type === "noul") return { p: key === "0-3" || key === "0-4" ? 0.93 : 0.04 }; + return undefined; + }); + const outcome = await runDecisionsObserve(config(provider), { + page: { + captureSnapshot: vi.fn(async () => ({ + combinedTree: SHOP, + combinedXpathMap: XPATHS, + combinedUrlMap: {}, + })), + } as never, + logger: logger(), + instruction: "find all the text fields", + snapshotOptions: {}, + ensureTimeRemaining: () => {}, + }); + expect(outcome.kind).toBe("done"); + expect(outcome.kind === "done" && outcome.actions.map((action) => action.selector)).toEqual([ + "xpath=/html/body/*[3]", + "xpath=/html/body/*[4]", + ]); + }); + + it("extracts by picking the element and copying its text", async () => { + serve(provider, (key, question) => { + if (key === "completed") return { p: 0.92 }; + if (question.type === "choice") { + return { choice: optionAbout(question, "1,249.50") ?? "none_match", p: 0.94 }; + } + return { p: 0.9 }; + }); + const outcome = await runDecisionsExtract(config(provider), { + logger: logger(), + instruction: "extract the price", + schema: { type: "object", required: ["price"], properties: { price: { type: "number" } } }, + snap: { tree: SHOP, xpathMap: {}, nodes: parseOutline(SHOP) }, + urlMap: {}, + ensureTimeRemaining: () => {}, + gate: true, + }); + expect(outcome).toMatchObject({ kind: "done", data: { price: 1249.5 } }); + }); + + it("invokes a page tool with arguments read from the instruction", async () => { + const tools: WebMCPToolDescriptor[] = [ + { + name: "add_to_cart", + description: "Add a product to the cart", + frameId: "main", + inputSchema: { + type: "object", + required: ["product_id"], + properties: { product_id: { type: "string" }, quantity: { type: "integer" } }, + }, + }, + { name: "clear_cart", description: "Empty the cart", frameId: "main" }, + ]; + const invoked: Array<{ name: string; input: unknown }> = []; + const seen = serve(provider, (key, question) => { + if (key === "family") return { choice: "not_an_action", p: 0.99 }; + if (key === "tool_best") return { choice: "add_to_cart", p: 0.97 }; + if (key === "tool_strict") return { choice: "add_to_cart", p: 0.96 }; + if (key === "tool_names_control") return { p: 0.03 }; + // Argument questions: keyed "tool_arg:add_to_cart:", aliased where colons are not allowed. + if (question.text.includes("parameter 'product_id'")) { + return { choice: optionAbout(question, "the exact words: p_102"), p: 0.97 }; + } + if (question.text.includes("parameter 'quantity'")) { + return { + choice: Object.entries(question.descriptions).find(([, text]) => + text.endsWith("the exact words: 2"), + )?.[0], + p: 0.95, + }; + } + return undefined; + }); + const { deps } = actDeps("add 2 of product p_102 to my cart", { + page: {} as Page, + webmcp: { + tools: Promise.resolve(tools), + page: { + invokeWebMCPTool: async (frameId, toolName, options) => { + invoked.push({ name: toolName, input: options?.input }); + return { invocationId: "inv-1", toolName, frameId, input: options?.input ?? {} }; + }, + waitForWebMCPInvocationResult: async () => ({ + invocationId: "inv-1", + status: "Completed", + output: { ok: true }, + }), + }, + }, + }); + const outcome = await runDecisionsAct({ ...config(provider), tools: true }, deps); + expect(outcome).toMatchObject({ kind: "done", viaTool: { argumentLlm: false } }); + expect(invoked).toEqual([{ name: "add_to_cart", input: { product_id: "p_102", quantity: 2 } }]); + // One request carried intent, tool choice and the arguments. + expect(seen).toHaveLength(1); + }); +}); diff --git a/packages/extension/tests/decisionsExtract.test.ts b/packages/extension/tests/decisionsExtract.test.ts index 35e3b6b32..37f6df974 100644 --- a/packages/extension/tests/decisionsExtract.test.ts +++ b/packages/extension/tests/decisionsExtract.test.ts @@ -1,6 +1,7 @@ import { trace } from "@opentelemetry/api"; import { afterEach, describe, expect, it, vi } from "vitest"; import { StagehandLogger } from "../logger.js"; +import { realisticAnswer } from "./decisionsTestUtils.js"; import { compactForJudge, findGroups, @@ -77,7 +78,12 @@ function stubByField(picks: Record, completed = 0.9) { : (picks[field] ?? "none_match"); return [ key, - { type: "choice", choice, confidence: 0.95, probabilities: { [choice]: 0.95 } }, + realisticAnswer(question, { + type: "choice", + choice, + confidence: 0.95, + probabilities: { [choice]: 0.95 }, + }), ]; }, ), @@ -120,7 +126,9 @@ describe("decisions extract (pick-and-copy)", () => { }); it("induces a list from the first item and reads the same position in every sibling", async () => { - stubByField({ author: "0-9", rating: "0-10" }); + // The rating candidate is the list item itself: "5 stars" inside it yields + // the same value, and one candidate per distinct value is offered. + stubByField({ author: "0-9", rating: "0-8" }); const outcome = await runDecisionsExtract( { apiKey: "test" }, deps({ @@ -220,7 +228,12 @@ describe("decisions extract (pick-and-copy)", () => { "none_match"; return [ key, - { type: "choice", choice: pick, confidence: 0.95, probabilities: { [pick]: 0.95 } }, + realisticAnswer(question, { + type: "choice", + choice: pick, + confidence: 0.95, + probabilities: { [pick]: 0.95 }, + }), ]; }), ); diff --git a/packages/extension/tests/decisionsModules.test.ts b/packages/extension/tests/decisionsModules.test.ts index 27ae44e27..2db3bdfd6 100644 --- a/packages/extension/tests/decisionsModules.test.ts +++ b/packages/extension/tests/decisionsModules.test.ts @@ -12,7 +12,7 @@ import { } from "../services/decisions/args.js"; import { blockingSignal, readPageState } from "../services/decisions/pageState.js"; import type { AskContext } from "../services/decisions/pick.js"; -import { systemOne } from "../services/decisions/typesafeClient.js"; +import { decide } from "../services/decisions/client.js"; import { describeCandidate, exactNameMatches, @@ -260,7 +260,10 @@ describe("argument parsing edge cases", () => { }); }); -describe("typesafe client errors", () => { +const ONE_QUESTION = { q: { type: "noul" as const, instructions: "Is it?" } }; +const ONE_ANSWER = { answers: { q: { type: "noul", noul: 0.5 } } }; + +describe("decision client errors", () => { afterEach(() => vi.unstubAllGlobals()); it("names the timeout budget and rejects a malformed success payload", async () => { @@ -272,7 +275,7 @@ describe("typesafe client errors", () => { throw error; }), ); - await expect(systemOne({ apiKey: `k-${Math.random()}` }, {}, {})).rejects.toThrow( + await expect(decide({ apiKey: `k-${Math.random()}` }, {}, ONE_QUESTION)).rejects.toThrow( /timed out after \d+ ms/, ); @@ -280,8 +283,8 @@ describe("typesafe client errors", () => { "fetch", vi.fn(async () => new Response("gateway", { status: 200 })), ); - await expect(systemOne({ apiKey: `k-${Math.random()}` }, {}, {})).rejects.toThrow( - /not a systemone payload/, + await expect(decide({ apiKey: `k-${Math.random()}` }, {}, ONE_QUESTION)).rejects.toThrow( + /not a decision payload/, ); }); @@ -292,18 +295,18 @@ describe("typesafe client errors", () => { vi.fn(async () => new Response("{}", { status: statuses.shift() ?? 200 })), ); const config = { apiKey: `k-${Math.random()}` }; - for (let i = 0; i < 4; i++) await systemOne(config, {}, {}).catch(() => undefined); + for (let i = 0; i < 4; i++) await decide(config, {}, ONE_QUESTION).catch(() => undefined); // Failures were 500, 400 (reset), 500, 500: two consecutive outage failures, // not three, so the breaker is still closed and the next call is attempted. vi.stubGlobal( "fetch", - vi.fn(async () => new Response(JSON.stringify({ answers: {} }), { status: 200 })), + vi.fn(async () => new Response(JSON.stringify(ONE_ANSWER), { status: 200 })), ); - await expect(systemOne(config, {}, {})).resolves.toMatchObject({ answers: {} }); + await expect(decide(config, {}, ONE_QUESTION)).resolves.toMatchObject(ONE_ANSWER); }); }); -describe("typesafe client breakers", () => { +describe("decision client breakers", () => { afterEach(() => vi.unstubAllGlobals()); it("opens the auth breaker on a 401 and rejects the next call while it is open", async () => { @@ -312,12 +315,10 @@ describe("typesafe client breakers", () => { "fetch", vi.fn(async () => new Response("{}", { status: 401 })), ); - await expect(systemOne(config, {}, {})).rejects.toThrow(/401/); - const fetchAfter = vi.fn( - async () => new Response(JSON.stringify({ answers: {} }), { status: 200 }), - ); + await expect(decide(config, {}, ONE_QUESTION)).rejects.toThrow(/401/); + const fetchAfter = vi.fn(async () => new Response(JSON.stringify(ONE_ANSWER), { status: 200 })); vi.stubGlobal("fetch", fetchAfter); - await expect(systemOne(config, {}, {})).rejects.toThrow(/paused/); + await expect(decide(config, {}, ONE_QUESTION)).rejects.toThrow(/paused/); expect(fetchAfter).not.toHaveBeenCalled(); }); @@ -329,17 +330,15 @@ describe("typesafe client breakers", () => { "fetch", vi.fn(async () => new Response(bodies.shift() ?? "{}", { status: statuses.shift() ?? 500 })), ); - for (let i = 0; i < 3; i++) await systemOne(config, {}, {}).catch(() => undefined); - const fetchAfter = vi.fn( - async () => new Response(JSON.stringify({ answers: {} }), { status: 200 }), - ); + for (let i = 0; i < 3; i++) await decide(config, {}, ONE_QUESTION).catch(() => undefined); + const fetchAfter = vi.fn(async () => new Response(JSON.stringify(ONE_ANSWER), { status: 200 })); vi.stubGlobal("fetch", fetchAfter); - await expect(systemOne(config, {}, {})).rejects.toThrow(/paused/); + await expect(decide(config, {}, ONE_QUESTION)).rejects.toThrow(/paused/); expect(fetchAfter).not.toHaveBeenCalled(); }); }); -describe("typesafe client breaker recovery", () => { +describe("decision client breaker recovery", () => { afterEach(() => { vi.unstubAllGlobals(); vi.restoreAllMocks(); @@ -353,14 +352,14 @@ describe("typesafe client breaker recovery", () => { "fetch", vi.fn(async () => new Response("{}", { status: 500 })), ); - for (let i = 0; i < 3; i++) await systemOne(config, {}, {}).catch(() => undefined); - await expect(systemOne(config, {}, {})).rejects.toThrow(/paused/); + for (let i = 0; i < 3; i++) await decide(config, {}, ONE_QUESTION).catch(() => undefined); + await expect(decide(config, {}, ONE_QUESTION)).rejects.toThrow(/paused/); // 30 s later the breaker lets a request through again. now += 30_001; - const ok = vi.fn(async () => new Response(JSON.stringify({ answers: {} }), { status: 200 })); + const ok = vi.fn(async () => new Response(JSON.stringify(ONE_ANSWER), { status: 200 })); vi.stubGlobal("fetch", ok); - await expect(systemOne(config, {}, {})).resolves.toMatchObject({ answers: {} }); + await expect(decide(config, {}, ONE_QUESTION)).resolves.toMatchObject(ONE_ANSWER); expect(ok).toHaveBeenCalledTimes(1); // The success reset the counter: two failures do not re-open it. @@ -368,9 +367,9 @@ describe("typesafe client breaker recovery", () => { "fetch", vi.fn(async () => new Response("{}", { status: 500 })), ); - for (let i = 0; i < 2; i++) await systemOne(config, {}, {}).catch(() => undefined); + for (let i = 0; i < 2; i++) await decide(config, {}, ONE_QUESTION).catch(() => undefined); vi.stubGlobal("fetch", ok); - await expect(systemOne(config, {}, {})).resolves.toMatchObject({ answers: {} }); + await expect(decide(config, {}, ONE_QUESTION)).resolves.toMatchObject(ONE_ANSWER); }); }); diff --git a/packages/extension/tests/decisionsObserve.test.ts b/packages/extension/tests/decisionsObserve.test.ts index 76e398062..5ddcc4da8 100644 --- a/packages/extension/tests/decisionsObserve.test.ts +++ b/packages/extension/tests/decisionsObserve.test.ts @@ -1,6 +1,7 @@ import { trace } from "@opentelemetry/api"; import { afterEach, describe, expect, it, vi } from "vitest"; import { StagehandLogger } from "../logger.js"; +import { realisticAnswer } from "./decisionsTestUtils.js"; import { runDecisionsObserve, type DecisionsObserveDeps } from "../services/decisions/observe.js"; const PAGE = [ @@ -38,7 +39,7 @@ function stub(answer: (key: string, question: { criteria?: Record }>; }; const answers = Object.fromEntries( - Object.entries(body.questions).map(([key, q]) => [key, answer(key, q)]), + Object.entries(body.questions).map(([key, q]) => [key, realisticAnswer(q, answer(key, q))]), ); return new Response( JSON.stringify({ diff --git a/packages/extension/tests/decisionsPipeline.test.ts b/packages/extension/tests/decisionsPipeline.test.ts index a57880085..6f0c32048 100644 --- a/packages/extension/tests/decisionsPipeline.test.ts +++ b/packages/extension/tests/decisionsPipeline.test.ts @@ -2,6 +2,7 @@ import { trace } from "@opentelemetry/api"; import { afterEach, describe, expect, it, vi } from "vitest"; import type { Action, ActResultData } from "@browserbasehq/stagehand-protocol/types"; import { StagehandLogger } from "../logger.js"; +import { realisticAnswer } from "./decisionsTestUtils.js"; import { fillValueCandidates, parseKey, @@ -78,9 +79,12 @@ function stubDecisions(table: Answers) { const answer = (table[key] ?? DEFAULT_ANSWERS[key]) as Answer; return [ key, - typeof answer === "function" - ? answer(question) - : (answer ?? { type: "noul", noul: 0.9 }), + realisticAnswer( + question as never, + typeof answer === "function" + ? answer(question as never) + : (answer ?? { type: "noul", noul: 0.9 }), + ), ]; }), ); diff --git a/packages/extension/tests/decisionsProviders.test.ts b/packages/extension/tests/decisionsProviders.test.ts new file mode 100644 index 000000000..8777433ef --- /dev/null +++ b/packages/extension/tests/decisionsProviders.test.ts @@ -0,0 +1,408 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + decide, + DECISION_PROVIDERS, + DecisionRequestError, + type DecisionModelConfig, + type DecisionProviderName, + type DecisionQuestion, +} from "../services/decisions/client.js"; +import { fakeDecisionProvider } from "./decisionsTestUtils.js"; + +/** + * Every provider must give the pipeline the same thing for the same question. + * `fakeProvider` is a server for each wire format: it reads the request the + * way that provider documents it and answers in that provider's own shape. + */ +type WireQuestion = { type: string; options: string[] }; +type Scripted = (key: string, question: WireQuestion) => { choice?: string; p?: number }; + +const CONFIGS: Record = { + typesafe: { provider: "typesafe", apiKey: "ts-key" }, + cloudflare: { provider: "cloudflare", apiKey: "cf-token", accountId: "acct123" }, + perplexity: { provider: "perplexity", apiKey: "pplx-key" }, + openai: { provider: "openai", apiKey: "sk-key" }, +}; + +const EXPECTED_URL: Record = { + typesafe: "https://api.typesafe.ai/v1/systemone", + cloudflare: "https://api.cloudflare.com/client/v4/accounts/acct123/ai/run/@cf/cloudflare/clef", + perplexity: "https://api.perplexity.ai/v1/decisions", + openai: "https://api.openai.com/v1/decisions", +}; + +function fakeProvider(name: DecisionProviderName, script: Scripted) { + return fakeDecisionProvider( + name, + (key, question) => script(key, { type: question.type, options: question.options }), + (fetchMock) => vi.stubGlobal("fetch", vi.fn(fetchMock)), + ); +} + +const QUESTIONS: Record = { + family: { + type: "choice", + instructions: "Which kind of browser action does the instruction ask for?", + criteria: { click: "A click", fill: "Typing text into a field", scroll: "Scrolling" }, + }, + // Ids a strict provider would reject: a colon in the key, spaces and slashes in the options. + "tool_arg:add_to_cart:size": { + type: "choice", + instructions: { task: "Which size?", request: "add the large one" }, + criteria: { "size / L": { label: "Large" }, "size / M": { label: "Medium" }, unset: null }, + }, + names_control: { + type: "noul", + instructions: { question: "Does the request name a control?", request: "add the large one" }, + criteria: { true: "It names a button or link", false: "It states a goal" }, + }, +}; + +afterEach(() => vi.unstubAllGlobals()); + +describe.each(DECISION_PROVIDERS)("decision provider %s", (name) => { + const config = () => ({ ...CONFIGS[name], apiKey: `${CONFIGS[name].apiKey}-${Math.random()}` }); + + it("answers the same questions with the same normalised result", async () => { + const seen = fakeProvider(name, (key, question) => { + if (key === "family") return { choice: "fill", p: 0.92 }; + if (question.type === "noul") return { p: 0.11 }; + // The second option, whatever it is called on the wire. + return { choice: question.options[1]!, p: 0.8 }; + }); + const response = await decide(config(), { instruction: "add the large one" }, QUESTIONS); + + expect(seen).toHaveLength(1); + expect(seen[0]!.url).toBe(EXPECTED_URL[name]); + expect(seen[0]!.headers.Authorization).toMatch(/^Bearer /); + expect(response.answers.family).toMatchObject({ + type: "choice", + choice: "fill", + confidence: 0.92, + }); + expect( + Object.keys((response.answers.family as { probabilities: object }).probabilities).sort(), + ).toEqual(["click", "fill", "scroll"]); + // Aliased on the wire where needed, but always the caller's own ids coming back. + expect(response.answers["tool_arg:add_to_cart:size"]).toMatchObject({ + type: "choice", + choice: "size / M", + confidence: 0.8, + }); + expect( + Object.keys( + (response.answers["tool_arg:add_to_cart:size"] as { probabilities: object }).probabilities, + ).sort(), + ).toEqual(["size / L", "size / M", "unset"]); + expect(response.answers.names_control).toEqual({ type: "noul", noul: 0.11 }); + expect(response.usage).toEqual({ inputTokens: 40, outputTokens: 3 }); + }); + + it("never sends a one-option choice and answers it as certain", async () => { + const seen = fakeProvider(name, () => ({ p: 0.3 })); + const response = await decide( + config(), + {}, + { + only: { type: "choice", instructions: "Which?", criteria: { "0-7": "the one candidate" } }, + also: { type: "noul", instructions: "Is it?" }, + }, + ); + expect(response.answers.only).toEqual({ + type: "choice", + choice: "0-7", + probabilities: { "0-7": 1 }, + confidence: 1, + }); + const sent = JSON.stringify(seen[0]!.body); + expect(sent).not.toContain("the one candidate"); + expect(response.answers.also).toEqual({ type: "noul", noul: 0.3 }); + }); + + it("splits a request past the provider's question limit and merges the answers", async () => { + const seen = fakeProvider(name, (key) => ({ p: key === "q149" ? 0.99 : 0.01 })); + const many = Object.fromEntries( + Array.from({ length: 150 }, (_, index) => [ + `q${index}`, + { type: "noul" as const, instructions: `Is candidate ${index} relevant?` }, + ]), + ); + const response = await decide(config(), { page: "x" }, many); + expect(Object.keys(response.answers)).toHaveLength(150); + expect(response.answers.q149).toEqual({ type: "noul", noul: 0.99 }); + const expectedRequests = { typesafe: 1, cloudflare: 3, perplexity: 2, openai: 3 }[name]; + expect(seen).toHaveLength(expectedRequests); + expect(response.usage.inputTokens).toBe(40 * expectedRequests); + }); + + it("rejects answers that do not fit the question instead of acting on them", async () => { + const question = { + pick: { type: "choice" as const, instructions: "Which?", criteria: { a: "A", b: "B" } }, + }; + const reply = (answer: Record) => { + vi.stubGlobal( + "fetch", + vi.fn(async () => { + if (name === "openai") { + const probabilities = Object.entries( + (answer.probabilities ?? {}) as Record, + ).map(([value, probability]) => ({ value, probability })); + return Response.json({ + answers: [{ type: "choice", name: "pick", ...answer, probabilities }], + }); + } + const payload = { answers: { pick: { type: "choice", ...answer } } }; + return Response.json( + name === "cloudflare" ? { result: payload, success: true } : payload, + ); + }), + ); + }; + reply({ choice: "c", confidence: 0.9, probabilities: { c: 0.9 } }); + await expect(decide(config(), {}, question)).rejects.toThrow(/not one of the options/); + reply({ choice: "a", confidence: 0.9, probabilities: { a: 1.4, b: 0.1 } }); + await expect(decide(config(), {}, question)).rejects.toThrow(/out of range/); + reply({ choice: "a", confidence: 0.2, probabilities: { a: 0.2, b: 0.8 } }); + await expect(decide(config(), {}, question)).rejects.toThrow(/not the most likely/); + reply({ choice: "a", confidence: 0.3, probabilities: { a: 0.3, b: 0.3 } }); + await expect(decide(config(), {}, question)).rejects.toThrow(/do not sum to 1/); + }); + + it("surfaces auth, overload and garbage as typed errors and pauses after a bad key", async () => { + const one = { q: { type: "noul" as const, instructions: "Is it?" } }; + const shared = config(); + const unauthorised = vi.fn( + async () => new Response('{"error":{"type":"auth"}}', { status: 401 }), + ); + vi.stubGlobal("fetch", unauthorised); + await expect(decide(shared, {}, one)).rejects.toBeInstanceOf(DecisionRequestError); + await expect(decide(shared, {}, one)).rejects.toThrow(/paused/); + expect(unauthorised).toHaveBeenCalledTimes(1); + + vi.stubGlobal( + "fetch", + vi.fn(async () => new Response("bad gateway", { status: 200 })), + ); + await expect(decide(config(), {}, one)).rejects.toThrow(/not a decision payload/); + + // 429 with Retry-After is retried, then succeeds. + let calls = 0; + vi.stubGlobal( + "fetch", + vi.fn(async () => { + if (++calls === 1) + return new Response("{}", { status: 429, headers: { "retry-after": "0.01" } }); + const noul = + name === "openai" + ? { answers: [{ type: "predicate", name: "q", probability: 0.7 }] } + : { answers: { q: { type: "noul", noul: 0.7 } } }; + return Response.json(name === "cloudflare" ? { result: noul, success: true } : noul); + }), + ); + await expect(decide(config(), {}, one)).resolves.toMatchObject({ + answers: { q: { type: "noul", noul: 0.7 } }, + }); + expect(calls).toBe(2); + }); +}); + +describe("provider wire formats", () => { + it("keeps TypeSafe's request exactly as the pipeline wrote it", async () => { + const seen = fakeProvider("typesafe", () => ({ p: 0.5 })); + await decide({ apiKey: `k-${Math.random()}` }, { url: "https://e.com" }, QUESTIONS); + expect(seen[0]!.body).toEqual({ + state: { url: "https://e.com" }, + model: "jev-latest", + questions: QUESTIONS, + }); + }); + + it("sends Cloudflare and Perplexity text-only instructions and wire-safe ids", async () => { + for (const name of ["cloudflare", "perplexity"] as const) { + const seen = fakeProvider(name, () => ({ p: 0.5 })); + await decide( + { ...CONFIGS[name], apiKey: `k-${Math.random()}` }, + { url: "https://e.com" }, + QUESTIONS, + ); + const body = seen[0]!.body as { + model: string; + state: unknown; + questions: Record }>; + }; + expect(body.model).toBe(name === "cloudflare" ? "clef" : "pplx-decider-v1-27b"); + expect(body.state).toEqual({ url: "https://e.com" }); + const keys = Object.keys(body.questions); + expect(keys.every((key) => /^[A-Za-z0-9_.-]{1,100}$/.test(key))).toBe(true); + expect(keys).toContain("family"); + expect(keys).not.toContain("tool_arg:add_to_cart:size"); + for (const question of Object.values(body.questions)) { + expect(typeof question.instructions).toBe("string"); + for (const [id, description] of Object.entries(question.criteria ?? {})) { + expect(id).toMatch(/^[A-Za-z0-9_.-]{1,100}$/); + expect(description === null || typeof description === "string").toBe(true); + } + } + // The aliased option still tells the model what it is. + const aliased = + body.questions[keys.find((key) => key !== "family" && key !== "names_control")!]!; + expect(JSON.stringify(aliased.criteria)).toContain("size / L"); + } + }); + + it("uses clef-flash and a custom gateway URL when asked", async () => { + const seen = fakeProvider("cloudflare", () => ({ p: 0.5 })); + await decide( + { + provider: "cloudflare", + apiKey: `k-${Math.random()}`, + accountId: "a1", + model: "clef-flash", + }, + {}, + { q: { type: "noul", instructions: "Is it?" } }, + ); + expect(seen[0]!.url).toBe( + "https://api.cloudflare.com/client/v4/accounts/a1/ai/run/@cf/cloudflare/clef-flash", + ); + await expect( + decide( + { provider: "cloudflare", apiKey: "k" }, + {}, + { q: { type: "noul", instructions: "?" } }, + ), + ).rejects.toThrow(/accountId/); + }); + + it("translates to and from OpenAI's Decisions shape", async () => { + // Response body recorded upstream on 2026-09-29 by a preview user + // (crmne/ruby_llm#1008, reused by pydantic/pydantic-ai#9646); the request + // below is ours, the answer names and shapes are OpenAI's. + const recorded = { + model: "gpt-6-luna", + answers: [ + { type: "predicate", name: "urgent", probability: 1.0 }, + { + type: "choice", + name: "department", + choice: "billing", + probabilities: [ + { value: "billing", probability: 1.0 }, + { value: "technical", probability: 0.0 }, + { value: "other", probability: 0.0 }, + ], + confidence: 1.0, + }, + ], + usage: { + input_tokens: 396, + input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 }, + output_tokens: 3, + output_tokens_details: { reasoning_tokens: 0 }, + total_tokens: 399, + }, + }; + let sent: Record = {}; + vi.stubGlobal( + "fetch", + vi.fn(async (_url: string, init: RequestInit) => { + sent = JSON.parse(init.body as string); + return Response.json(recorded); + }), + ); + const response = await decide( + { provider: "openai", apiKey: `sk-${Math.random()}` }, + { message: "I was charged twice. Please refund the duplicate charge today." }, + { + urgent: { + type: "noul", + instructions: "Does the customer explicitly need action today?", + criteria: { + true: "Explicitly asks for action today", + false: "No deadline or a later deadline", + }, + }, + department: { + type: "choice", + instructions: "Which team should handle this message?", + criteria: { + billing: "Payments and refunds", + technical: "Bugs and integrations", + other: null, + }, + }, + }, + ); + expect(sent).toEqual({ + model: "gpt-6-luna", + input: '{"message":"I was charged twice. Please refund the duplicate charge today."}', + questions: [ + { + type: "predicate", + name: "urgent", + instructions: + "Does the customer explicitly need action today?\nYes: Explicitly asks for action today\nNo: No deadline or a later deadline", + }, + { + type: "choice", + name: "department", + instructions: "Which team should handle this message?", + choices: [ + { value: "billing", description: "Payments and refunds" }, + { value: "technical", description: "Bugs and integrations" }, + { value: "other" }, + ], + }, + ], + }); + expect(response.answers).toEqual({ + urgent: { type: "noul", noul: 1 }, + department: { + type: "choice", + choice: "billing", + probabilities: { billing: 1, technical: 0, other: 0 }, + confidence: 1, + }, + }); + expect(response.usage).toEqual({ inputTokens: 396, outputTokens: 3 }); + }); + + it("names OpenAI's preview gate instead of a bare 403", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response( + '{"error":{"message":"Decision API is not enabled for this user.","type":"invalid_request_error"}}', + { status: 403 }, + ), + ), + ); + await expect( + decide( + { provider: "openai", apiKey: `sk-${Math.random()}` }, + {}, + { + q: { type: "noul", instructions: "Is it?" }, + }, + ), + ).rejects.toThrow(/403: decisions_not_enabled/); + }); + + it("refuses a non-https endpoint and an unknown provider", async () => { + await expect( + decide( + { apiKey: "k", apiUrl: "http://localhost:1" }, + {}, + { q: { type: "noul", instructions: "?" } }, + ), + ).rejects.toThrow(/https/); + await expect( + decide( + { apiKey: "k", provider: "nope" as never }, + {}, + { q: { type: "noul", instructions: "?" } }, + ), + ).rejects.toThrow(/unknown provider/); + }); +}); diff --git a/packages/extension/tests/decisionsProvidersLive.test.ts b/packages/extension/tests/decisionsProvidersLive.test.ts new file mode 100644 index 000000000..b3c5d81b0 --- /dev/null +++ b/packages/extension/tests/decisionsProvidersLive.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it } from "vitest"; +import { + decide, + type DecisionChoiceAnswer, + type DecisionModelConfig, + type DecisionNoulAnswer, + type DecisionQuestion, +} from "../services/decisions/client.js"; + +/** + * Live conformance: the same questions against every provider a key exists + * for. Never runs in CI; opt in with DECISIONS_LIVE=1 and any of + * TYPESAFE_API_KEY + * CLOUDFLARE_API_TOKEN + CLOUDFLARE_ACCOUNT_ID + * PERPLEXITY_API_KEY + * OPENAI_API_KEY (needs Decisions preview access) + * The questions are unambiguous on purpose: this checks that a provider is + * wired correctly and answers in a usable shape, not how good its model is. + */ +const env = process.env; +const LIVE = env.DECISIONS_LIVE === "1"; + +const PROVIDERS: Array<{ name: string; config: DecisionModelConfig | undefined }> = [ + { name: "typesafe", config: env.TYPESAFE_API_KEY ? { apiKey: env.TYPESAFE_API_KEY } : undefined }, + { + name: "cloudflare", + config: + env.CLOUDFLARE_API_TOKEN && env.CLOUDFLARE_ACCOUNT_ID + ? { + provider: "cloudflare", + apiKey: env.CLOUDFLARE_API_TOKEN, + accountId: env.CLOUDFLARE_ACCOUNT_ID, + ...(env.DECISIONS_CLOUDFLARE_MODEL ? { model: env.DECISIONS_CLOUDFLARE_MODEL } : {}), + } + : undefined, + }, + { + name: "perplexity", + config: env.PERPLEXITY_API_KEY + ? { provider: "perplexity", apiKey: env.PERPLEXITY_API_KEY } + : undefined, + }, + { + name: "openai", + config: env.OPENAI_API_KEY ? { provider: "openai", apiKey: env.OPENAI_API_KEY } : undefined, + }, +]; + +const STATE = { + instruction: "click the Checkout button", + page: "Shop — cart with 2 items. Buttons: Add to cart, Checkout. Link: Help.", +}; + +const QUESTIONS: Record = { + family: { + type: "choice", + instructions: "Which kind of browser action does the instruction ask for?", + criteria: { + click: "Clicking or pressing an element with the pointer", + fill: "Typing text into a field", + scroll: "Scrolling the page", + }, + }, + // A key and option ids outside every strict provider's alphabet, and JSON + // instructions and descriptions: the client has to make these acceptable. + "target:pointer/0": { + type: "choice", + instructions: { task: "Which element should receive the click?", request: STATE.instruction }, + criteria: { + "0-5 / a": { role: "button", name: "Add to cart" }, + "0-7 / b": { role: "button", name: "Checkout" }, + "0-9 / c": { role: "link", name: "Help" }, + none_match: "None of these elements is what the instruction refers to", + }, + }, + is_click: { + type: "noul", + instructions: "Does the instruction ask to click something?", + criteria: { true: "It asks for a click", false: "It asks for something else" }, + }, + mentions_refund: { type: "noul", instructions: "Does the instruction mention a refund?" }, +}; + +describe.skipIf(!LIVE)("live decision providers", () => { + for (const { name, config } of PROVIDERS) { + it.skipIf(!config)( + `${name} answers the canonical questions`, + async (ctx) => { + let response; + try { + response = await decide(config!, STATE, QUESTIONS); + } catch (error) { + // A key without access to a gated preview is a skip, not a failure of the adapter. + if (error instanceof Error && /decisions_not_enabled/.test(error.message)) { + ctx.skip(`${name}: ${error.message}`); + return; + } + throw error; + } + const family = response.answers.family as DecisionChoiceAnswer; + const target = response.answers["target:pointer/0"] as DecisionChoiceAnswer; + const isClick = response.answers.is_click as DecisionNoulAnswer; + const refund = response.answers.mentions_refund as DecisionNoulAnswer; + // eslint-disable-next-line no-console -- the point of a live run is to see the numbers + console.info( + `[${name}] model=${response.model} ${response.durationMs} ms in=${response.usage.inputTokens} ` + + `family=${family.choice}@${family.confidence.toFixed(2)} target=${target.choice}@${target.confidence.toFixed(2)} ` + + `is_click=${isClick.noul.toFixed(2)} refund=${refund.noul.toFixed(2)}`, + ); + expect(family.choice).toBe("click"); + expect(family.confidence).toBeGreaterThan(0.5); + expect(target.choice).toBe("0-7 / b"); + expect( + Object.keys(target.probabilities).every( + (id) => id in QUESTIONS["target:pointer/0"]!.criteria!, + ), + ).toBe(true); + expect(isClick.noul).toBeGreaterThan(0.5); + expect(refund.noul).toBeLessThan(0.5); + expect(response.usage.inputTokens).toBeGreaterThan(0); + }, + 30_000, + ); + } +}); diff --git a/packages/extension/tests/decisionsTestUtils.ts b/packages/extension/tests/decisionsTestUtils.ts new file mode 100644 index 000000000..f8eef0e30 --- /dev/null +++ b/packages/extension/tests/decisionsTestUtils.ts @@ -0,0 +1,171 @@ +/** + * Shared by the decisions test stubs. A real decision model can only ever + * answer with one of the options it was offered; the client now rejects + * anything else. Stubs that script "whatever, nothing matches" answers go + * through this so they stay short and still look like a provider's reply. + */ +type Question = { type?: string; criteria?: object | null }; +type Choice = { + type: "choice"; + choice: string; + confidence: number; + probabilities: Record; +}; + +/** Options that mean "none of the above" in the pipeline's questions, in order of preference. */ +const NEUTRAL = [ + "none_match", + "none_of_these", + "unset", + "other", + "unspecified", + "nothing", + "not_scroll", + "left", +]; + +export function realisticAnswer(question: Question, answer: T): T | Choice { + const scripted = answer as Partial | null; + if (!scripted || typeof scripted !== "object" || typeof scripted.choice !== "string") { + return answer; + } + const options = Object.keys(question.criteria ?? {}); + if (options.length === 0 || options.includes(scripted.choice)) return answer; + const neutral = NEUTRAL.find((id) => options.includes(id)); + // No way to say "none": a real model still names an option, just without conviction. + const choice = neutral ?? options[0]!; + const confidence = neutral ? (scripted.confidence ?? 0.95) : 0.2; + return { type: "choice", choice, confidence, probabilities: { [choice]: confidence } }; +} + +export type FakeProviderName = "typesafe" | "cloudflare" | "perplexity" | "openai"; +export type WireQuestion = { + type: "choice" | "noul"; + /** Option ids exactly as they arrived on the wire (possibly aliased). */ + options: string[]; + /** Instructions plus option descriptions, as text, to script answers by content. */ + text: string; + descriptions: Record; +}; +export type ScriptedAnswer = { choice?: string; p?: number }; +export type SeenRequest = { + url: string; + headers: Record; + body: Record; +}; + +/** + * A stand-in server for each provider's wire format: it reads the request the + * way that provider documents it and replies in that provider's own shape. + * Unscripted choices fall on a "none of the above" option when there is one. + */ +export function fakeDecisionProvider( + name: FakeProviderName, + script: (key: string, question: WireQuestion) => ScriptedAnswer | undefined, + stub: (fetchMock: (url: string, init: RequestInit) => Promise) => void, +): SeenRequest[] { + const seen: SeenRequest[] = []; + const asText = (value: unknown): string => + typeof value === "string" ? value : value === undefined ? "" : JSON.stringify(value); + const distribution = (options: string[], choice: string, p: number) => + Object.fromEntries( + options.map((id) => [id, id === choice ? p : (1 - p) / Math.max(1, options.length - 1)]), + ); + const answerChoice = (key: string, question: WireQuestion) => { + const scripted = script(key, question) ?? {}; + const fallback = NEUTRAL.find((id) => question.options.includes(id)) ?? question.options[0]!; + const choice = + scripted.choice && question.options.includes(scripted.choice) ? scripted.choice : fallback; + return { choice, p: scripted.p ?? 0.95 }; + }; + + stub(async (url, init) => { + const body = JSON.parse(init.body as string) as Record; + seen.push({ url, headers: init.headers as Record, body }); + + if (name === "openai") { + const answers = (body.questions as Array>).map((question) => { + const key = question.name as string; + if (question.type === "predicate") { + const wire: WireQuestion = { + type: "noul", + options: [], + text: asText(question.instructions), + descriptions: {}, + }; + return { type: "predicate", name: key, probability: script(key, wire)?.p ?? 0.5 }; + } + const choices = question.choices as Array<{ value: string; description?: unknown }>; + const descriptions = Object.fromEntries( + choices.map((entry) => [entry.value, asText(entry.description)]), + ); + const wire: WireQuestion = { + type: "choice", + options: choices.map((entry) => entry.value), + text: `${asText(question.instructions)} ${Object.values(descriptions).join(" ")}`, + descriptions, + }; + const { choice, p } = answerChoice(key, wire); + return { + type: "choice", + name: key, + choice, + confidence: p, + probabilities: Object.entries(distribution(wire.options, choice, p)).map( + ([value, probability]) => ({ value, probability }), + ), + }; + }); + return Response.json({ + model: "gpt-6-luna", + answers, + usage: { input_tokens: 40, output_tokens: 3, total_tokens: 43 }, + }); + } + + const questions = body.questions as Record< + string, + { type: string; instructions?: unknown; criteria?: Record | null } + >; + const answers = Object.fromEntries( + Object.entries(questions).map(([key, question]) => { + const descriptions = Object.fromEntries( + Object.entries(question.criteria ?? {}).map(([id, value]) => [id, asText(value)]), + ); + const text = `${asText(question.instructions)} ${Object.values(descriptions).join(" ")}`; + if (question.type === "noul") { + const wire: WireQuestion = { type: "noul", options: [], text, descriptions }; + return [key, { type: "noul", noul: script(key, wire)?.p ?? 0.5 }]; + } + const wire: WireQuestion = { + type: "choice", + options: Object.keys(descriptions), + text, + descriptions, + }; + const { choice, p } = answerChoice(key, wire); + return [ + key, + { + type: "choice", + choice, + confidence: p, + probabilities: distribution(wire.options, choice, p), + }, + ]; + }), + ); + const payload = { + model: String(body.model), + answers, + usage: { input_tokens: 40, output_tokens: 3 }, + }; + // Workers AI wraps every model's output. + return Response.json( + name === "cloudflare" + ? { result: payload, success: true, errors: [], messages: [] } + : payload, + ); + }); + return seen; +} diff --git a/packages/extension/tests/decisionsTools.test.ts b/packages/extension/tests/decisionsTools.test.ts index d03f05b0a..f39d6691d 100644 --- a/packages/extension/tests/decisionsTools.test.ts +++ b/packages/extension/tests/decisionsTools.test.ts @@ -7,6 +7,7 @@ import type { } from "@browserbasehq/stagehand-protocol/types"; import { toolArguments } from "../inference.js"; import { StagehandLogger } from "../logger.js"; +import { realisticAnswer } from "./decisionsTestUtils.js"; import type { Variables } from "@browserbasehq/stagehand-protocol/types"; import { runDecisionsAct, type DecisionsActDeps } from "../services/decisions/pipeline.js"; import type { DecisionsToolDeps } from "../services/decisions/toolAct.js"; @@ -97,8 +98,24 @@ function stubDecisions(script: Scripted) { )?.[0] ?? "unset"; confidence = p; } - probabilities[choice] = confidence; - answers[key] = { type: "choice", choice, confidence, probabilities }; + if (key === "tool_strict") { + // One distribution: what "none" takes, the tool cannot also have. + const none = script.none ?? 0; + confidence = Math.min(confidence, 1 - none); + probabilities[choice] = confidence; + if (none > confidence) { + choice = "none_of_these"; + confidence = none; + } + } else { + probabilities[choice] = confidence; + } + answers[key] = realisticAnswer(question, { + type: "choice", + choice, + confidence, + probabilities, + }); } return new Response(JSON.stringify({ answers, usage: { input_tokens: 10 } }), { status: 200, @@ -161,7 +178,7 @@ function harness( const config = { apiKey: "test", tools: true }; -describe("the decision model WebMCP tool act", () => { +describe("decisions WebMCP tool act", () => { afterEach(() => vi.unstubAllGlobals()); it("invokes a tool that takes no input inside the intent request", async () => { @@ -280,7 +297,7 @@ describe("the decision model WebMCP tool act", () => { expect(h.skipReason()).toBe("arguments_not_filled"); }); - it("never sends a variable's value to TypeSafe and resolves it only for the page", async () => { + it("never sends a variable's value to the provider and resolves it only for the page", async () => { const requests = stubDecisions({ tool: "add_to_cart", args: { product_id: ["%sku%", 0.96] } }); const h = harness("add product %sku% to my cart", {}, { sku: "secret-sku-9" }); const outcome = await runDecisionsAct(config, h.deps); diff --git a/packages/protocol/schemas.ts b/packages/protocol/schemas.ts index 163ee3019..79f133dd2 100644 --- a/packages/protocol/schemas.ts +++ b/packages/protocol/schemas.ts @@ -1650,13 +1650,22 @@ export const StagehandInitParamsSchema = z }), experimentalDecisions: z .strictObject({ + provider: z.enum(["typesafe", "cloudflare", "perplexity", "openai"]).optional().meta({ + description: + "Decision-model provider: TypeSafe Jev (default), Cloudflare Clef, Perplexity Decider, or OpenAI's Decisions API", + }), apiKey: z.string().min(1), model: z.string().min(1).optional(), apiUrl: z .string() // A regex (not a refine) so the constraint reaches the generated JSON schema and SDK types. - .regex(/^https:\/\/[^\s?#]+$/, "TypeSafe apiUrl must be https, without query or fragment") + .regex(/^https:\/\/[^\s?#]+$/, "apiUrl must be https, without query or fragment") .optional(), + accountId: z + .string() + .regex(/^[A-Za-z0-9_-]{1,64}$/) + .optional() + .meta({ description: 'Cloudflare account id; required for provider "cloudflare"' }), enabled: z.boolean().optional(), retryNoEffect: z.boolean().optional(), focusFallback: z.boolean().optional(), @@ -1674,7 +1683,7 @@ export const StagehandInitParamsSchema = z .optional() .meta({ description: - "Experimental: resolve act() through TypeSafe Jev decisions before falling back to the LLM pipeline", + "Experimental: resolve act(), observe() and extract() through a decision model (typed questions answered with probabilities) before falling back to the LLM pipeline", }), }) .meta({ id: "StagehandInitParams" }); diff --git a/packages/protocol/stagehand.v4.json b/packages/protocol/stagehand.v4.json index 0b47d35a2..7e7a75a4f 100644 --- a/packages/protocol/stagehand.v4.json +++ b/packages/protocol/stagehand.v4.json @@ -1224,9 +1224,14 @@ "$ref": "#/$defs/Caching" }, "experimental_decisions": { - "description": "Experimental: resolve act() through TypeSafe Jev decisions before falling back to the LLM pipeline", + "description": "Experimental: resolve act(), observe() and extract() through a decision model (typed questions answered with probabilities) before falling back to the LLM pipeline", "type": "object", "properties": { + "provider": { + "description": "Decision-model provider: TypeSafe Jev (default), Cloudflare Clef, Perplexity Decider, or OpenAI's Decisions API", + "type": "string", + "enum": ["typesafe", "cloudflare", "perplexity", "openai"] + }, "api_key": { "type": "string", "minLength": 1 @@ -1239,6 +1244,11 @@ "type": "string", "pattern": "^https:\\/\\/[^\\s?#]+$" }, + "account_id": { + "description": "Cloudflare account id; required for provider \"cloudflare\"", + "type": "string", + "pattern": "^[A-Za-z0-9_-]{1,64}$" + }, "enabled": { "type": "boolean" }, diff --git a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip index 8500d4673..65e17cddd 100644 Binary files a/packages/sdk-go/internal/extensionassets/stagehand-extension.zip and b/packages/sdk-go/internal/extensionassets/stagehand-extension.zip differ diff --git a/packages/sdk-go/models.gen.go b/packages/sdk-go/models.gen.go index fd32936a1..62e276ca0 100644 --- a/packages/sdk-go/models.gen.go +++ b/packages/sdk-go/models.gen.go @@ -1998,8 +1998,9 @@ type StagehandInitParams struct { // "dom_settle_timeout_ms". DOMSettleTimeoutMs *int `json:"dom_settle_timeout_ms,omitempty,omitzero"` - // Experimental: resolve act() through TypeSafe Jev decisions before falling back - // to the LLM pipeline + // Experimental: resolve act(), observe() and extract() through a decision model + // (typed questions answered with probabilities) before falling back to the LLM + // pipeline ExperimentalDecisions *StagehandInitParamsExperimentalDecisions `json:"experimental_decisions,omitempty,omitzero"` // LogLevel corresponds to the JSON schema field "log_level". @@ -2021,9 +2022,13 @@ type StagehandInitParams struct { Telemetry *TelemetryConfig `json:"telemetry,omitempty,omitzero"` } -// Experimental: resolve act() through TypeSafe Jev decisions before falling back -// to the LLM pipeline +// Experimental: resolve act(), observe() and extract() through a decision model +// (typed questions answered with probabilities) before falling back to the LLM +// pipeline type StagehandInitParamsExperimentalDecisions struct { + // Cloudflare account id; required for provider "cloudflare" + AccountID *string `json:"account_id,omitempty,omitzero"` + // ActConfidence corresponds to the JSON schema field "act_confidence". ActConfidence *float64 `json:"act_confidence,omitempty,omitzero"` @@ -2060,6 +2065,10 @@ type StagehandInitParamsExperimentalDecisions struct { // PageState corresponds to the JSON schema field "page_state". PageState *bool `json:"page_state,omitempty,omitzero"` + // Decision-model provider: TypeSafe Jev (default), Cloudflare Clef, Perplexity + // Decider, or OpenAI's Decisions API + Provider *StagehandInitParamsExperimentalDecisionsProvider `json:"provider,omitempty,omitzero"` + // RetryNoEffect corresponds to the JSON schema field "retry_no_effect". RetryNoEffect *bool `json:"retry_no_effect,omitempty,omitzero"` @@ -2079,6 +2088,13 @@ const StagehandInitParamsExperimentalDecisionsExtractJudge StagehandInitParamsEx const StagehandInitParamsExperimentalDecisionsExtractOff StagehandInitParamsExperimentalDecisionsExtract = "off" const StagehandInitParamsExperimentalDecisionsExtractPick StagehandInitParamsExperimentalDecisionsExtract = "pick" +type StagehandInitParamsExperimentalDecisionsProvider string + +const StagehandInitParamsExperimentalDecisionsProviderCloudflare StagehandInitParamsExperimentalDecisionsProvider = "cloudflare" +const StagehandInitParamsExperimentalDecisionsProviderOpenai StagehandInitParamsExperimentalDecisionsProvider = "openai" +const StagehandInitParamsExperimentalDecisionsProviderPerplexity StagehandInitParamsExperimentalDecisionsProvider = "perplexity" +const StagehandInitParamsExperimentalDecisionsProviderTypesafe StagehandInitParamsExperimentalDecisionsProvider = "typesafe" + type StagehandInitParamsExperimentalDecisionsVerify string const StagehandInitParamsExperimentalDecisionsVerifyChecks StagehandInitParamsExperimentalDecisionsVerify = "checks" diff --git a/packages/sdk-python/src/stagehand/_generated/input_types.py b/packages/sdk-python/src/stagehand/_generated/input_types.py index f169cfbf2..50c482b99 100644 --- a/packages/sdk-python/src/stagehand/_generated/input_types.py +++ b/packages/sdk-python/src/stagehand/_generated/input_types.py @@ -225,9 +225,11 @@ class EmptyParams(TypedDict): class ExperimentalDecisions(TypedDict): + provider: NotRequired[Literal["typesafe", "cloudflare", "perplexity", "openai"]] api_key: str model: NotRequired[str] api_url: NotRequired[str] + account_id: NotRequired[str] enabled: NotRequired[bool] retry_no_effect: NotRequired[bool] focus_fallback: NotRequired[bool] diff --git a/packages/sdk-python/src/stagehand/_generated/models.py b/packages/sdk-python/src/stagehand/_generated/models.py index f7b4eff25..e33790932 100644 --- a/packages/sdk-python/src/stagehand/_generated/models.py +++ b/packages/sdk-python/src/stagehand/_generated/models.py @@ -561,17 +561,23 @@ class EmptyParams(WireModel): class ExperimentalDecisions(WireModel): - """Experimental: resolve act() through TypeSafe Jev decisions before falling back to the LLM pipeline""" + """Experimental: resolve act(), observe() and extract() through a decision model (typed questions answered with probabilities) before falling back to the LLM pipeline""" model_config = ConfigDict( extra="forbid", validate_by_name=True, ) + provider: Optional[Provider] = None + """Decision-model provider: TypeSafe Jev (default), Cloudflare Clef, Perplexity Decider, or OpenAI's Decisions API""" api_key: Annotated[StrictStr, Field(min_length=1)] model: Annotated[Optional[StrictStr], Field(min_length=1)] = None api_url: Annotated[Optional[StrictStr], Field(pattern="^https:\\/\\/[^\\s?#]+$")] = ( None ) + account_id: Annotated[Optional[StrictStr], Field(pattern="^[A-Za-z0-9_-]{1,64}$")] = ( + None + ) + """Cloudflare account id; required for provider "cloudflare\"""" enabled: Optional[StrictBool] = None retry_no_effect: Optional[StrictBool] = None focus_fallback: Optional[StrictBool] = None @@ -2115,6 +2121,17 @@ class PageWebMCPToolsResult(WireModel): tools: list[WebMCPToolDescriptor] +class Provider(StrEnum): + """ + Decision-model provider: TypeSafe Jev (default), Cloudflare Clef, Perplexity Decider, or OpenAI's Decisions API + """ + + typesafe = "typesafe" + cloudflare = "cloudflare" + perplexity = "perplexity" + openai = "openai" + + class ProxyConfig(RootModel[Union[BrowserbaseProxyConfig, ExternalProxyConfig]]): root: Union[BrowserbaseProxyConfig, ExternalProxyConfig] @@ -2283,7 +2300,7 @@ class StagehandInitParams(WireModel): cache: Optional[Caching] = None """Server-side caching of act/observe/extract results for this instance: a boolean toggle, or an object with an optional hit-count threshold. Requires a Browserbase apiKey and browser sessionId. Can be overridden per request via options.cache.""" experimental_decisions: Optional[ExperimentalDecisions] = None - """Experimental: resolve act() through TypeSafe Jev decisions before falling back to the LLM pipeline""" + """Experimental: resolve act(), observe() and extract() through a decision model (typed questions answered with probabilities) before falling back to the LLM pipeline""" class StagehandInitResult(WireModel):