diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d407bc..b3edbd8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- Add `--jev`, which plans `ACT` with TypeSafe's Jev first and falls back to the `model` option when Jev is unsure, cannot supply an argument, or fails. `TYPESAFE_API_KEY` holds Jev's key, and `WHIRL_JEV_ENDPOINT` sends its requests to another server. Whirl asks Jev through `lithos-llm`, which prices its requests. JSON reports record the planner of each action and Jev's requests, tokens, and cost, which the step's cost includes. - When the text `ACT` types matches a string the instruction quotes, type the instruction's characters, so a model that changes the case or spacing of `"AbC 123"` still types `AbC 123`. - After `ACT` fills a field, read the value back. When the field does not hold it, as when a `maxlength` cuts it short, the step fails with `act-fill-mismatch` instead of passing silently. Case, spaces, and punctuation do not count, so a field that formats its value still passes. - When `ACT` clicks a native checkbox or radio input, focus it and press Space, as `CHECK` does, so a styled control that covers the input does not make the click wait out the step. diff --git a/Cargo.lock b/Cargo.lock index d944174..ab5da42 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2327,7 +2327,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.4.3", + "getrandom 0.3.4", "once_cell", "rustix", "windows-sys 0.61.2", @@ -2664,7 +2664,7 @@ dependencies = [ [[package]] name = "twin-core" version = "0.1.5" -source = "git+https://github.com/lithoscomputer/twins?rev=552576b5c50cf178da37dd6a0c97c472ffc6ffcb#552576b5c50cf178da37dd6a0c97c472ffc6ffcb" +source = "git+https://github.com/lithoscomputer/twins?rev=73164984026a2901f133140a467195c1d924f244#73164984026a2901f133140a467195c1d924f244" dependencies = [ "anyhow", "async-stream", @@ -2679,7 +2679,7 @@ dependencies = [ [[package]] name = "twin-openai" version = "0.1.5" -source = "git+https://github.com/lithoscomputer/twins?rev=552576b5c50cf178da37dd6a0c97c472ffc6ffcb#552576b5c50cf178da37dd6a0c97c472ffc6ffcb" +source = "git+https://github.com/lithoscomputer/twins?rev=73164984026a2901f133140a467195c1d924f244#73164984026a2901f133140a467195c1d924f244" dependencies = [ "anyhow", "async-stream", diff --git a/SPEC.md b/SPEC.md index b9756e4..27cd90c 100644 --- a/SPEC.md +++ b/SPEC.md @@ -498,6 +498,27 @@ its value, such as a phone number, passes. Whirl skips the check when the element has no value to read, such as a `contenteditable` element, or when the page has removed it. +With `--jev` (section 13), TypeSafe's Jev plans first, as in Stagehand's +experimental Jev path. Jev is a classifier: it answers closed questions in a +few hundred milliseconds but cannot write text. Whirl asks it which kind of +action the instruction wants, then which element, from the snapshot elements +that kind of action can target. Each element is described by its role, name, +and value, the text of its table row or list item, the named sections around +it, the nearest heading, and its place among elements that look the same. +When there are more than 40 such elements, Jev sees the 30 whose descriptions +share the most words with the instruction. For `press`, a focused control is +the target without a question. Arguments come from the instruction itself: the one quoted string or +placeholder that does not name the field for `fill`, the named key for +`press`, and the one option of a native select that the instruction names. +Jev's choice is checked like a model answer. When Jev is not confident, when +it cannot supply an argument, for step two of a two-step action, and when a +Jev request fails, the model plans the step instead, and its prompt lists +Jev's likely matches when Jev has some. Jev receives the instruction, with +masked values as placeholders, and the element descriptions. Whirl asks Jev +through `lithos-llm` as `typesafe/jev-latest` and prices its requests from the +catalog. Jev's usage is reported apart from the model's, and the step's cost +includes both. + A custom dropdown that must open before an option can be chosen is a two-step action. The model marks its first answer as two-step. Whirl runs that action, takes a new snapshot, and asks for the second action. When the second answer @@ -536,9 +557,10 @@ snapshot contains text that the page shows, including values typed by earlier steps; Whirl does not mask page content. The step's report text is the authored line. The JSON report adds an `act` -object to the step: the model, each action that ran as a Whirl line with the -model's description of the element, and the token usage and cost of the model -calls. A rendered line such as `CLICK role:button "Sign in"` describes the +object to the step: the model, the planner, each action that ran as a Whirl +line with the description of the element and the planner that chose it, and +the token usage and cost of the model calls, and with `--jev` Jev's requests, +tokens, and cost. A rendered line such as `CLICK role:button "Sign in"` describes the element; it is not guaranteed to be unique on the page. ## 8. PAGE @@ -832,7 +854,7 @@ Whirl masks every value sourced from `env.*` in the textual output it generates: - **Failure.** The first failing step fails the entry, and a failed entry stops its file; remaining entries in that file are skipped and reported as skipped. Other files still run. On failure Whirl saves a full-page screenshot and, with `--trace`, a Playwright trace to the artifacts directory. - **Navigation.** `VISIT` completes when the new document reaches `DOMContentLoaded`: the HTML is parsed and its synchronous scripts have run. It does not wait for the `load` event, because images, fonts, iframes, and media hold `load` open for reasons a flow never asserted, and every later line waits for what it needs anyway: actions wait for their element to be actionable, asserts and `PAGE` retry. A page that only becomes usable after `load` needs an assert on that state before an `EVAL` or `SCREENSHOT`, which run once without waiting. - **Retries.** Page checks and page captures read their value again on the schedule of section 9.7 until they pass or the step timeout expires. Response checks, response captures, and `eval` captures read once. -- **ACT.** An `ACT` line is one step. Its snapshots, model calls, and actions share its step timeout. Model calls take seconds, so an `ACT` line that needs more than the step timeout sets its own, such as `@60s`. +- **ACT.** An `ACT` line is one step. Its snapshots, model calls, Jev requests, and actions share its step timeout. Model calls take seconds, so an `ACT` line that needs more than the step timeout sets its own, such as `@60s`. - **Timeouts.** Each action, PAGE, assert, and capture line gets the step timeout (`step-timeout` option, default 10s); `VISIT` gets the navigation timeout (`nav-timeout` option, default 30s). A trailing `@duration` on any such line overrides its own budget: `CLICK "Generate report" @60s`. The optional `entry-timeout` option caps an entry's total time across all of its lines; when it expires, the in-flight step fails with an entry-timeout error. An entry without one is still bounded by its per-step timeouts. The suffix must be bare: a line’s final bare token of the form `@duration` is always its timeout, and a quoted `"@60s"` is an ordinary value. Timeouts are enforced from outside the page, so they hold even when the page cannot respond — an `EVAL` script blocking the renderer or returning a Promise that never settles. When a timed-out step cannot be cancelled cleanly, Whirl closes that flow's browser context; if closing also stalls, it terminates and restarts only that worker's shim process. Either way the flow fails and reports normally, and other files are unaffected. - **Setup.** Files with a `setup` option run after their setup flows. Whirl first runs every distinct setup flow named by the inputs, once each and in parallel like any files, then runs the remaining files, each starting from its setup flow's saved state with the setup flow's captures as `{{setup.name}}`. A setup flow that is also an input runs once, as the setup. A failed setup flow reports normally, and each of its dependents reports a `[setup]` failure naming the setup flow and its first failing step, without opening a browser. Setup flows are one level deep. - **Parallelism.** Files run in parallel across worker slots (`--jobs`, default: logical CPU count). A single file is never parallelized. @@ -885,6 +907,7 @@ nothing and exits with code 1 when any file would change. | `--save-storage PATH` | Write the final storage state after a successful run (single file only) | | `--entry-timeout DURATION` | Override the entry-timeout option | | `--user-agent UA` | Override the user-agent option with `chrome`, `firefox`, `safari`, or a literal string | +| `--jev` | Plan `ACT` with TypeSafe's Jev first, and the `model` option when Jev is unsure (section 7.4) | Exit codes: @@ -904,6 +927,13 @@ does not check it against the catalog. `WHIRL_LLM_API_KEY`, when set, is sent to that server as a bearer token. These variables serve local model servers, proxies, and tests. +`--jev` reads Jev's key from `TYPESAFE_API_KEY`. Without it, a run whose files +use `ACT` fails before any flow starts, with a runtime error (exit 3). +`WHIRL_JEV_ENDPOINT=http://host:port` sends Jev's requests to +`/v1/systemone` on another server, such as a test double. `--jev` does +nothing in a run whose files do not use `ACT`. `--rerun-failed` does not read +it from a report; pass it again. + `--video` records the `main` tab (section 7.1). On Chromium, Whirl records the page's own screencast frames through Playwright's bundled ffmpeg at 60 frames per second, or at the rate `--video-fps` names; a still page holds its last frame, so the recording always plays at a constant rate. Firefox and WebKit use Playwright's recorder at its fixed rate of 25 frames per second. `--video-fps` on those engines is not an error: the file records at 25 frames per second and reports a warning, because the recording is evidence, not a result. A missing ffmpeg fails the file as a runtime error; `whirl install` provisions it with every browser build, and `whirl doctor` checks for it. `--rerun-failed` reads a version 1 or version 2 report with an absolute `workingDirectory`. Relative file paths resolve against that directory, even when the report is moved. Each selected file runs from the beginning, including its setup. Existing CLI overrides and secrets must be supplied again; a report is not executable configuration. An unsupported or malformed report is a usage error. A report with no failed or errored files exits 0 with a message and launches no browser. @@ -991,7 +1021,7 @@ The artifact override applies to each input's relative paths. Destination protec Rust source, configuration, and project setup follow the [Brynary Rust Style Guide](https://github.com/brynary/rust-style-guide). TypeScript source and language tooling follow the [Brynary TypeScript Style Guide](https://github.com/brynary/typescript-style-guide) for language-level and authoring conventions. The shim targets the pinned private Node runtime specified here, so the TypeScript guide's Bun-specific runtime, API, package-management, and test-runner policies do not apply. This specification and accepted Whirl ADRs take precedence over both guides. -- `whirl` is a single Rust binary containing the parser, the runner, the check engine, the reporters, and the shim manager. It also makes `ACT`'s language model calls itself, through the `lithos-llm` client; the shim only takes the page snapshot and runs the chosen action. +- `whirl` is a single Rust binary containing the parser, the runner, the check engine, the reporters, and the shim manager. It also makes `ACT`'s language model calls and, with `--jev`, its Jev requests itself, through the `lithos-llm` client; the shim only takes the page snapshot and runs the chosen action. - Whirl drives browsers through a thin Node shim that Whirl owns: a small, stable JSON API over stdio pipes, shaped like Whirl's closed vocabulary and implemented on the Playwright library. The Rust binary launches the shim as a child process. Whirl does not reimplement browser automation and does not speak CDP or Playwright's internal driver protocol, so it inherits Playwright's auto-waiting, retrying assertions, locator engine, tracing, and three browser engines — and Playwright upgrades stay internal to the shim. - `whirl install` downloads the pinned shim bundle (a private Node runtime, the shim, and the `@playwright/test` package) and the browser builds. Users do not need Node installed. Each Whirl release pins exactly one Playwright version. - Rust evaluates every filter and predicate (ADR [evaluate-checks-in-rust](docs/engineering/decisions/evaluate-checks-in-rust.md)). The shim reads raw values: page strings, `eval` results, and each response's status, headers, and body bytes. Rust owns the retry loop for page checks and page captures, on Playwright's poll schedule. diff --git a/crates/whirl/Cargo.toml b/crates/whirl/Cargo.toml index fafc118..fd07896 100644 --- a/crates/whirl/Cargo.toml +++ b/crates/whirl/Cargo.toml @@ -46,7 +46,7 @@ axum = "0.8" proptest = "=1.11.0" tiny_http = "0.12.0" # The OpenAI-compatible model twin that ACT acceptance tests run against. -twin-openai = { git = "https://github.com/lithoscomputer/twins", rev = "552576b5c50cf178da37dd6a0c97c472ffc6ffcb" } +twin-openai = { git = "https://github.com/lithoscomputer/twins", rev = "73164984026a2901f133140a467195c1d924f244" } trycmd = "1.2.1" [lints] diff --git a/crates/whirl/src/cli.rs b/crates/whirl/src/cli.rs index 16c99f5..e819896 100644 --- a/crates/whirl/src/cli.rs +++ b/crates/whirl/src/cli.rs @@ -223,6 +223,10 @@ struct RunArgs { #[arg(long)] har: bool, + /// Plan ACT with TypeSafe Jev first, falling back to the model option. + #[arg(long)] + jev: bool, + /// Override the `storage` option. #[arg(long, value_name = "PATH")] storage: Option, @@ -935,6 +939,7 @@ fn run_command(args: &RunArgs) -> Exit { }, overrides, base_vars, + jev: args.jev, }; let files: Vec = checked.inputs.into_iter().map(|input| input.file).collect(); let setups: Vec = checked.setups.into_iter().map(|input| input.file).collect(); diff --git a/crates/whirl/src/report/console.rs b/crates/whirl/src/report/console.rs index 3bff439..6e1462c 100644 --- a/crates/whirl/src/report/console.rs +++ b/crates/whirl/src/report/console.rs @@ -210,9 +210,11 @@ mod tests { let act_line = entry.steps[0].line; entry.steps[0].act = Some(ActReport { model: "gpt-test".to_owned(), + planner: "llm".to_owned(), actions: vec![ActActionReport { line: "CLICK role:button \"Sign in\"".to_owned(), description: "the sign-in button".to_owned(), + planned_by: "llm".to_owned(), }], usage: ActUsage::default(), }); diff --git a/crates/whirl/src/report/html.rs b/crates/whirl/src/report/html.rs index 931b146..93080bb 100644 --- a/crates/whirl/src/report/html.rs +++ b/crates/whirl/src/report/html.rs @@ -441,10 +441,22 @@ fn render_act(output: &mut impl io::Write, step: &StepReport) -> io::Result<()> act.usage.input_tokens, act.usage.output_tokens )?; + if let Some(jev) = act.usage.jev { + write!( + output, + "
Jev
{} request(s) · {} input and {} output tokens
", + jev.requests, jev.input_tokens, jev.output_tokens + )?; + } for action in &act.actions { + let chosen_by = if action.planned_by == "jev" { + " · chosen by Jev" + } else { + "" + }; write!( output, - "
Ran
{} {}
", + "
Ran
{} {}{chosen_by}
", escape(&action.line), escape(&action.description) )?; diff --git a/crates/whirl/src/report/model.rs b/crates/whirl/src/report/model.rs index 96ac9b5..08cc5c5 100644 --- a/crates/whirl/src/report/model.rs +++ b/crates/whirl/src/report/model.rs @@ -143,10 +143,18 @@ pub(crate) struct StepReport { #[serde(rename_all = "camelCase")] pub(crate) struct ActReport { pub(crate) model: String, + /// The planner that chose the actions: `llm`, or `jev` with `--jev`. + #[serde(default = "llm_planner")] + pub(crate) planner: String, pub(crate) actions: Vec, pub(crate) usage: ActUsage, } +/// Reports written before planners existed used the language model. +fn llm_planner() -> String { + "llm".to_owned() +} + /// One action an `ACT` step ran. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] @@ -155,6 +163,9 @@ pub(crate) struct ActActionReport { pub(crate) line: String, /// The model's description of the element. pub(crate) description: String, + /// Which planner chose this action: `llm` or `jev`. + #[serde(default = "llm_planner")] + pub(crate) planned_by: String, } /// Token usage summed over an `ACT` step's model calls. @@ -169,6 +180,22 @@ pub(crate) struct ActUsage { /// Present only when every call was priced. #[serde(default, skip_serializing_if = "Option::is_none")] pub(crate) cost_usd_micros: Option, + /// Jev's requests, tokens, and cost; present only with `--jev`. + /// `costUsdMicros` above includes Jev's cost. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) jev: Option, +} + +/// What an `ACT` step's Jev requests used. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ActJevUsage { + pub(crate) requests: u32, + pub(crate) input_tokens: u64, + pub(crate) output_tokens: u64, + /// Present only when every answered request was priced. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) cost_usd_micros: Option, } /// One entry's result: its steps, captures, and artifacts. diff --git a/crates/whirl/src/run/act.rs b/crates/whirl/src/run/act.rs index 64bc348..b40ecb3 100644 --- a/crates/whirl/src/run/act.rs +++ b/crates/whirl/src/run/act.rs @@ -4,6 +4,7 @@ mod decision; mod instruction; +mod jev; mod model; mod planner; mod prompt; @@ -11,6 +12,9 @@ mod snapshot; pub(crate) use decision::{ActDecision, FollowUp}; pub(crate) use instruction::Instruction; +pub(crate) use jev::{JevClient, JevPlanner, JevSetupError}; pub(crate) use model::{ModelCatalog, ModelClient, ModelSetupError}; -pub(crate) use planner::{ActPlanner, LlmPlanner, PlanError, PlanRequest, PlanStep, PlanUsage}; +pub(crate) use planner::{ + ActPlanner, LlmPlanner, PlanError, PlanRequest, PlanStep, PlanUsage, PlannedBy, +}; pub(crate) use snapshot::PageSnapshot; diff --git a/crates/whirl/src/run/act/decision.rs b/crates/whirl/src/run/act/decision.rs index f301ec3..dfd9560 100644 --- a/crates/whirl/src/run/act/decision.rs +++ b/crates/whirl/src/run/act/decision.rs @@ -25,6 +25,27 @@ struct InferredAction { arguments: Vec, } +impl ActInference { + /// A one-step answer that another planner chose, in the model's + /// terms, so it goes through [`PageSnapshot::decide`] like any answer. + pub(crate) fn chosen( + element_id: String, + description: String, + method: ActMethod, + arguments: Vec, + ) -> Self { + Self { + action: Some(InferredAction { + element_id, + description, + method, + arguments, + }), + two_step: false, + } + } +} + /// The methods the model may choose. The names are Stagehand's, so its /// prompts carry over; each maps to one Whirl verb. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq)] diff --git a/crates/whirl/src/run/act/instruction.rs b/crates/whirl/src/run/act/instruction.rs index b305815..ed29b32 100644 --- a/crates/whirl/src/run/act/instruction.rs +++ b/crates/whirl/src/run/act/instruction.rs @@ -175,7 +175,7 @@ pub(crate) fn same_text(left: &str, right: &str) -> bool { /// curly single quotes, or in straight single quotes that stand at word /// boundaries, so the apostrophe in "user's" opens no quote. A quote does /// not cross a line. The rules are those of Stagehand's `quotedStrings`. -fn quoted_strings(text: &str) -> Vec<&str> { +pub(crate) fn quoted_strings(text: &str) -> Vec<&str> { let is_word = |ch: Option| ch.is_some_and(|ch| ch.is_alphanumeric() || ch == '_'); let mut found = Vec::new(); let mut start = 0; diff --git a/crates/whirl/src/run/act/jev.rs b/crates/whirl/src/run/act/jev.rs new file mode 100644 index 0000000..d3a7206 --- /dev/null +++ b/crates/whirl/src/run/act/jev.rs @@ -0,0 +1,425 @@ +//! The `--jev` planner (SPEC 7.4): TypeSafe's Jev chooses the action when +//! it is confident, and the language model plans otherwise. +//! +//! Jev answers closed questions in a few hundred milliseconds but cannot +//! write text. One request asks which kind of action the instruction wants. +//! A second asks which element, from the candidates that kind of action can +//! target, twice as Stagehand's Jev path does: `best` must choose, and +//! `strict` may say that none fits. Arguments come from the instruction +//! itself (see [`args`]). When Jev is unsure, when an argument is missing, +//! or when a request fails, the fallback planner plans the step, with Jev's +//! likely matches added to its prompt. + +mod args; +mod client; +mod outline; + +use std::collections::HashMap; +use std::fmt::Write as _; +use std::sync::Arc; +use std::time::Instant; + +pub(crate) use client::{JevClient, JevSetupError}; +use serde_json::{Map, Value as Json, json}; + +use self::client::{JevAnswer, JevQuestion, JevResponse, choice, noul}; +use self::outline::{Outline, View, shortlist}; +use crate::run::act::decision::{ActInference, ActMethod}; +use crate::run::act::planner::{ + ActPlanner, JevUsage, Plan, PlanFuture, PlanRequest, PlanStep, PlanUsage, PlannedBy, +}; + +/// The key of the none-of-these option. +const NONE: &str = "none_match"; +const NONE_DESCRIPTION: &str = "None of these elements matches the instruction"; +/// The confidence Jev needs to decide the intent or the element. +const ACCEPT: f64 = 0.7; +/// Strict's none-of-these vetoes a pick above this probability. +const NONE_VETO: f64 = 0.9; +/// Above this many candidates, Jev sees only the best word matches. +const PRUNE_ABOVE: usize = 40; +const PRUNE_KEEP: usize = 30; +/// Jev's likely matches reach the fallback when its top pick is at least +/// this likely. +const CREDIBLE: f64 = 0.5; +const HINT_ITEMS: usize = 5; + +/// The kinds of action Jev can tell apart. +const INTENTS: &[(&str, &str)] = &[ + ( + "click", + "Click, tap, open, follow, or toggle an element such as a button, link, tab, checkbox, or radio button", + ), + ("fill", "Type or enter text into a field"), + ("select", "Choose an option from a dropdown list"), + ("press", "Press a keyboard key, such as Enter or Escape"), + ( + "hover", + "Move the mouse over an element without clicking it", + ), + ("double_click", "Double-click an element"), + ( + "other", + "Something else, such as scrolling, dragging, or several actions at once", + ), +]; + +/// Asks Jev first and the fallback planner when Jev is unsure. +#[derive(Debug)] +pub(crate) struct JevPlanner { + jev: JevClient, + fallback: Arc, +} + +impl JevPlanner { + pub(crate) fn new(jev: JevClient, fallback: Arc) -> Self { + Self { jev, fallback } + } +} + +/// What Jev decided for one planning step. +enum Outcome { + Chosen(ActInference), + /// The fallback plans, with Jev's likely matches when it has some. + Unsure(Option), +} + +/// An element Jev chose, or what it found when it did not choose. +type Pick = Result>; + +impl ActPlanner for JevPlanner { + fn plan<'a>(&'a self, request: PlanRequest<'a>) -> PlanFuture<'a> { + Box::pin(async move { + let mut jev = JevUsage::default(); + let outcome = self.choose(request, &mut jev).await; + let spent = PlanUsage { + jev, + ..PlanUsage::default() + }; + match outcome { + Outcome::Chosen(inference) => Plan { + answer: Ok(inference), + usage: spent, + planned_by: PlannedBy::Jev, + }, + Outcome::Unsure(hint) => { + let mut plan = self + .fallback + .plan(PlanRequest { + hint: hint.as_deref(), + ..request + }) + .await; + plan.usage = plan.usage.saturating_add(spent); + plan + } + } + }) + } + + fn name(&self) -> &'static str { + PlannedBy::Jev.as_str() + } +} + +impl JevPlanner { + async fn choose(&self, request: PlanRequest<'_>, usage: &mut JevUsage) -> Outcome { + // Step two of a two-step action needs the model's plan of step one. + if !matches!(request.step, PlanStep::First) { + return Outcome::Unsure(None); + } + let instruction = request.instruction.prompt(); + let Some(intent) = self.intent(instruction, request.deadline, usage).await else { + return Outcome::Unsure(None); + }; + let outline = Outline::parse(request.snapshot.raw()); + let (method, view) = match intent { + "click" => (ActMethod::Click, View::Pointer), + "double_click" => (ActMethod::DoubleClick, View::Pointer), + "hover" => (ActMethod::Hover, View::Pointer), + "fill" => (ActMethod::Fill, View::Input), + "select" => (ActMethod::SelectOptionFromDropdown, View::Select), + "press" => (ActMethod::Press, View::Keyboard), + _ => return Outcome::Unsure(None), + }; + let key = match method { + ActMethod::Press => match args::key(instruction) { + Some(key) => Some(key), + None => return Outcome::Unsure(None), + }, + _ => None, + }; + let focused = key.as_ref().and(outline.focused()); + let index = match focused { + Some(index) => index, + None => match self.pick(&outline, view, &request, usage).await { + Ok(index) => index, + Err(hint) => return Outcome::Unsure(hint), + }, + }; + let node = outline.node(index); + let description = describe_line(&outline.describe(index, &HashMap::new())); + let argument = match method { + ActMethod::Fill => args::fill_value(request.instruction, node.name.as_deref()), + ActMethod::SelectOptionFromDropdown => { + args::option(instruction, &outline.options(index), node.name.as_deref()) + .map(str::to_owned) + } + ActMethod::Press => key, + _ => None, + }; + let needs_argument = !matches!( + method, + ActMethod::Click | ActMethod::DoubleClick | ActMethod::Hover + ); + if needs_argument && argument.is_none() { + // Jev found the element but cannot write its text. + return Outcome::Unsure(Some(hint(&[(element(&outline, index), description)]))); + } + Outcome::Chosen(ActInference::chosen( + element(&outline, index), + description, + method, + argument.into_iter().collect(), + )) + } + + /// Which kind of action the instruction asks for, when Jev is sure. + async fn intent( + &self, + instruction: &str, + deadline: Instant, + usage: &mut JevUsage, + ) -> Option<&'static str> { + let criteria: Map = INTENTS + .iter() + .map(|(intent, meaning)| ((*intent).to_owned(), json!(meaning))) + .collect(); + let questions = vec![( + "intent", + choice( + json!({ + "question": "Which kind of browser action does the instruction ask for?", + "instruction": instruction, + }), + criteria, + ), + )]; + let response = self.ask(instruction, questions, deadline, usage).await?; + let JevAnswer::Choice { + choice, confidence, .. + } = response.answers.get("intent")? + else { + return None; + }; + let intent = INTENTS + .iter() + .map(|(intent, _)| *intent) + .find(|intent| intent == choice)?; + (*confidence >= ACCEPT).then_some(intent) + } + + /// The element Jev chooses among a view's candidates. + async fn pick( + &self, + outline: &Outline, + view: View, + request: &PlanRequest<'_>, + usage: &mut JevUsage, + ) -> Pick { + let instruction = request.instruction.prompt(); + let candidates = outline.view(view); + if candidates.is_empty() { + return Err(None); + } + let twins = outline.twins(&candidates); + let descriptions: HashMap = candidates + .iter() + .map(|&index| (index, outline.describe(index, &twins))) + .collect(); + let asked = if candidates.len() > PRUNE_ABOVE { + shortlist(instruction, &candidates, &descriptions, PRUNE_KEEP).ok_or(None)? + } else { + candidates + }; + let criteria: Map = asked + .iter() + .map(|&index| (element(outline, index), descriptions[&index].clone())) + .collect(); + let mut strict = criteria.clone(); + strict.insert(NONE.to_owned(), json!(NONE_DESCRIPTION)); + let single = match asked.as_slice() { + [only] => Some(*only), + _ => None, + }; + let best = match single { + Some(only) => noul(json!({ + "question": "Is this element a plausible target for the instruction?", + "instruction": instruction, + "element": descriptions[&only], + })), + None => choice( + json!({ + "question": instruction, + "note": "Pick the best available element even if the wording does not match exactly.", + }), + criteria, + ), + }; + let questions = vec![ + ("strict", choice(json!(instruction), strict)), + ("best", best), + ]; + let response = self + .ask(instruction, questions, request.deadline, usage) + .await + .ok_or(None)?; + + let none = match response.answers.get("strict") { + Some(JevAnswer::Choice { probabilities, .. }) => { + probabilities.get(NONE).copied().unwrap_or(0.0) + } + _ => return Err(None), + }; + let by_ref: HashMap = asked + .iter() + .map(|&index| (element(outline, index), index)) + .collect(); + let (chosen, confidence, ranked) = match (single, response.answers.get("best")) { + (Some(only), Some(JevAnswer::Noul { noul })) => (only, *noul, vec![(only, *noul)]), + ( + None, + Some(JevAnswer::Choice { + choice, + confidence, + probabilities, + }), + ) => { + let Some(&chosen) = by_ref.get(choice) else { + return Err(None); + }; + let mut ranked: Vec<(usize, f64)> = probabilities + .iter() + .filter_map(|(key, probability)| { + by_ref.get(key).map(|&index| (index, *probability)) + }) + .collect(); + ranked.sort_by(|left, right| right.1.total_cmp(&left.1)); + (chosen, *confidence, ranked) + } + _ => return Err(None), + }; + if confidence >= ACCEPT && none <= NONE_VETO { + return Ok(chosen); + } + if confidence < CREDIBLE { + return Err(None); + } + let likely: Vec<(String, String)> = ranked + .into_iter() + .take(HINT_ITEMS) + .map(|(index, _)| { + ( + element(outline, index), + describe_line(&descriptions[&index]), + ) + }) + .collect(); + Err(Some(hint(&likely))) + } + + /// One request; its usage counts even when it fails. + async fn ask( + &self, + instruction: &str, + questions: Vec<(&'static str, JevQuestion)>, + deadline: Instant, + usage: &mut JevUsage, + ) -> Option { + usage.requests = usage.requests.saturating_add(1); + let response = self + .jev + .ask(json!({ "instruction": instruction }), questions, deadline) + .await + .ok()?; + let spend = response.spend; + usage.input_tokens = usage.input_tokens.saturating_add(spend.input_tokens); + usage.output_tokens = usage.output_tokens.saturating_add(spend.output_tokens); + match spend.cost_usd_micros { + Some(cost) => usage.cost_usd_micros = usage.cost_usd_micros.saturating_add(cost), + None => usage.unpriced = true, + } + Some(response) + } +} + +fn element(outline: &Outline, index: usize) -> String { + outline + .node(index) + .element + .clone() + .expect("every candidate has a ref") +} + +/// A description on one line, such as `button "Delete" · row: 1037 · Stark`. +fn describe_line(description: &Json) -> String { + let Json::Object(fields) = description else { + return description.to_string(); + }; + let mut parts = Vec::new(); + let role = fields + .get("role") + .and_then(Json::as_str) + .unwrap_or_default(); + match fields.get("name").and_then(Json::as_str) { + Some(name) => parts.push(format!("{role} {}", json!(name))), + None => parts.push(role.to_owned()), + } + for (key, value) in fields { + if key != "role" && key != "name" { + let text = value + .as_str() + .map_or_else(|| value.to_string(), str::to_owned); + parts.push(format!("{key}: {text}")); + } + } + parts.join(" · ") +} + +/// Jev's likely matches, as the fallback's prompt shows them. +fn hint(likely: &[(String, String)]) -> String { + let mut text = String::from( + "A classifier found these likely matches. Use one only if it fits the instruction and the tree:", + ); + for (element, description) in likely { + write!(text, "\n- {element}: {description}").expect("writing to a String cannot fail"); + } + text +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn a_description_reads_on_one_line() { + assert_eq!( + describe_line( + &json!({"role": "button", "name": "Delete", "row": "1037 · Stark", "position": "2 of 12"}) + ), + r#"button "Delete" · row: 1037 · Stark · position: 2 of 12"# + ); + assert_eq!( + describe_line(&json!({"role": "textbox", "label": "Last name"})), + "textbox · label: Last name" + ); + } + + #[test] + fn the_hint_lists_refs_and_descriptions() { + assert_eq!( + hint(&[("e5".to_owned(), "button \"Save\"".to_owned())]), + "A classifier found these likely matches. Use one only if it fits the instruction and the tree:\n- e5: button \"Save\"" + ); + } +} diff --git a/crates/whirl/src/run/act/jev/args.rs b/crates/whirl/src/run/act/jev/args.rs new file mode 100644 index 0000000..cf65293 --- /dev/null +++ b/crates/whirl/src/run/act/jev/args.rs @@ -0,0 +1,178 @@ +//! Arguments the Jev planner reads from the instruction itself. Jev cannot +//! write text, so an argument must be a quoted string, a placeholder, a key +//! name, or an option the page shows. Anything else goes to the language +//! model. + +use crate::run::act::instruction::{Instruction, quoted_strings, same_text}; + +/// The text to fill: the one quoted string or placeholder in the +/// instruction that is not the field's own name, which an instruction +/// such as `type "Ada" into "Name"` also quotes. +pub(crate) fn fill_value(instruction: &Instruction, field_name: Option<&str>) -> Option { + let prompt = instruction.prompt(); + let mut values: Vec = instruction + .bindings() + .placeholders() + .into_iter() + .filter(|placeholder| prompt.contains(placeholder.as_str())) + .collect(); + for quoted in quoted_strings(prompt) { + let is_field = field_name.is_some_and(|name| same_text(name, quoted)); + let in_placeholder = values.iter().any(|value| value.contains(quoted)); + if !is_field && !in_placeholder && !values.iter().any(|value| value == quoted) { + values.push(quoted.to_owned()); + } + } + match values.as_slice() { + [value] => Some(value.clone()), + _ => None, + } +} + +/// The key a `press` instruction names, as Playwright spells it: `press +/// Enter`, `press the tab key`, or one character. +pub(crate) fn key(instruction: &str) -> Option { + const NAMED: &[(&str, &str)] = &[ + ("enter", "Enter"), + ("return", "Enter"), + ("tab", "Tab"), + ("escape", "Escape"), + ("esc", "Escape"), + ("space", "Space"), + ("backspace", "Backspace"), + ("delete", "Delete"), + ("arrowdown", "ArrowDown"), + ("arrowup", "ArrowUp"), + ]; + let lower = instruction.to_lowercase(); + let words: Vec<&str> = lower + .split(|ch: char| !ch.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .collect(); + let press = words + .iter() + .position(|&word| word == "press" || word == "hit")?; + let word = words + .get(press + 1) + .filter(|&&word| word != "the") + .or_else(|| words.get(press + 2))?; + if let Some((_, key)) = NAMED.iter().find(|(name, _)| name == word) { + return Some((*key).to_owned()); + } + let mut chars = word.chars(); + match (chars.next(), chars.next()) { + (Some(ch), None) => Some(ch.to_string()), + _ => None, + } +} + +/// The one option of a native select that the instruction names, spelled +/// as the page spells it. The select's own name does not count. +pub(crate) fn option<'a>( + instruction: &str, + options: &[&'a str], + select_name: Option<&str>, +) -> Option<&'a str> { + let words = |text: &str| -> Vec { + text.split(|ch: char| !ch.is_alphanumeric()) + .filter(|word| !word.is_empty()) + .map(str::to_lowercase) + .collect() + }; + let said = words(instruction); + let named: Vec<&str> = options + .iter() + .copied() + .filter(|option| !select_name.is_some_and(|name| same_text(name, option))) + .filter(|option| { + let option = words(option); + !option.is_empty() && said.windows(option.len()).any(|window| window == option) + }) + .collect(); + match named.as_slice() { + [option] => Some(option), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::lang::ast::{Span, Value, ValueSegment}; + use crate::run::vars::VarStore; + + fn instruction(segments: Vec, vars: &mut VarStore) -> Instruction { + let value = Value { + segments, + span: Span { + line: 1, + column: 1, + len: 1, + }, + quoted: true, + }; + Instruction::try_new(&value, vars).expect("resolves") + } + + fn literal(text: &str) -> Instruction { + instruction( + vec![ValueSegment::Literal(text.to_owned())], + &mut VarStore::new(), + ) + } + + #[test] + fn the_fill_value_is_the_quoted_text_that_is_not_the_field() { + let typed = literal("type \"Ada\" into the \"Name\" field"); + assert_eq!(fill_value(&typed, Some("Name")), Some("Ada".to_owned())); + assert_eq!(fill_value(&typed, Some("First name")), None); + assert_eq!( + fill_value(&literal("type Ada into Name"), Some("Name")), + None + ); + } + + #[test] + fn a_placeholder_is_a_fill_value() { + let mut vars = VarStore::new(); + vars.record_secret("hunter2"); + vars.set_input("password", "hunter2"); + let typed = instruction( + vec![ + ValueSegment::Literal("type ".to_owned()), + ValueSegment::Var("password".to_owned()), + ValueSegment::Literal(" into the password field".to_owned()), + ], + &mut vars, + ); + assert_eq!( + fill_value(&typed, Some("Password")), + Some("%secret1%".to_owned()) + ); + } + + #[test] + fn keys_are_read_from_press_instructions() { + assert_eq!(key("press enter"), Some("Enter".to_owned())); + assert_eq!(key("Press the Tab key"), Some("Tab".to_owned())); + assert_eq!(key("press k"), Some("k".to_owned())); + assert_eq!(key("press the big button"), None); + assert_eq!(key("submit the form"), None); + } + + #[test] + fn the_option_is_the_one_the_instruction_names() { + let sizes = ["Small", "Medium", "Large"]; + assert_eq!( + option("choose Large from the size dropdown", &sizes, Some("Size")), + Some("Large") + ); + assert_eq!(option("choose a size", &sizes, Some("Size")), None); + assert_eq!(option("pick small or large", &sizes, Some("Size")), None); + let countries = ["United States", "United Kingdom"]; + assert_eq!( + option("choose united kingdom", &countries, None), + Some("United Kingdom") + ); + } +} diff --git a/crates/whirl/src/run/act/jev/client.rs b/crates/whirl/src/run/act/jev/client.rs new file mode 100644 index 0000000..cb948d2 --- /dev/null +++ b/crates/whirl/src/run/act/jev/client.rs @@ -0,0 +1,326 @@ +//! TypeSafe's Jev through `lithos-llm`'s evaluation API: the classifier the +//! `--jev` planner asks first (SPEC 7.4, 13). +//! +//! Jev cannot write text. It answers typed questions about a JSON state +//! with probabilities: a choice among named options, or the probability +//! that a statement holds. `lithos-llm` routes `typesafe/jev-latest` to +//! TypeSafe's `/v1/systemone`, checks each verdict, retries throttled and +//! failed calls, and prices the verdict from its catalog. + +use std::collections::HashMap; +use std::env; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; + +use lithos_llm::catalog::{Catalog, CatalogError}; +use lithos_llm::client::ClientBuildError; +use lithos_llm::credentials::ConventionalCredentials; +use lithos_llm::middleware::{CallContext, RetryMiddleware, RetryPolicy}; +use lithos_llm::types::ErrorKind; +use lithos_llm::{Client, Evaluation, Verdict}; +use serde_json::{Map, Value as Json}; + +/// Jev's key (SPEC 13). `lithos-llm` reads it for the `typesafe` provider. +pub(crate) const API_KEY_ENV: &str = "TYPESAFE_API_KEY"; +/// Another Jev server, such as a twin in tests (SPEC 13). +pub(crate) const ENDPOINT_ENV: &str = "WHIRL_JEV_ENDPOINT"; +/// The catalog row for Jev on TypeSafe's own API. +const MODEL: &str = "typesafe/jev-latest"; + +/// One request's cap, below any `ACT` budget. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(8); +/// A rejected key fails every request the same way, so Jev pauses. +const AUTH_PAUSE: Duration = Duration::from_secs(60); +/// After this many failures in a row, Jev pauses for [`OUTAGE_PAUSE`]. +const FAILURES_TO_PAUSE: u32 = 3; +const OUTAGE_PAUSE: Duration = Duration::from_secs(30); + +/// A question to Jev. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum JevQuestion { + /// Which of the named options fits; `options` maps each option's key + /// to its description, in order. + Choice { + instructions: Json, + options: Map, + }, + /// How likely the statement is to hold. + Noul { instructions: Json }, +} + +/// Jev's answer to one question. +#[derive(Clone, Debug, PartialEq)] +pub(crate) enum JevAnswer { + Choice { + choice: String, + confidence: f64, + probabilities: HashMap, + }, + Noul { + noul: f64, + }, +} + +/// What one request used. +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct JevSpend { + pub(crate) input_tokens: u64, + pub(crate) output_tokens: u64, + /// `None` when the catalog could not price the request. + pub(crate) cost_usd_micros: Option, +} + +/// Jev's answers, by question name. +#[derive(Clone, Debug, PartialEq)] +pub(crate) struct JevResponse { + pub(crate) answers: HashMap, + pub(crate) spend: JevSpend, +} + +/// Why a request got no answers. The planner falls back either way, so +/// the kinds only steer the breaker. +#[derive(Clone, Copy, Debug, Eq, PartialEq, thiserror::Error)] +pub(crate) enum JevError { + #[error("Jev rejected the key")] + Auth, + #[error("the Jev request failed")] + Failed, + #[error("Jev is paused after repeated failures")] + Paused, +} + +/// A failure to set up the Jev client. The run stops with a runtime error +/// (exit 3). +#[derive(Debug, thiserror::Error)] +pub(crate) enum JevSetupError { + #[error("--jev needs {API_KEY_ENV}")] + MissingKey, + #[error("the Jev catalog could not be built")] + Catalog(#[source] Box), + #[error("the Jev client could not be built")] + Client(#[source] Box), +} + +/// Pauses Jev after a rejected key or repeated failures, so a broken setup +/// does not cost each `ACT` line a doomed request before its fallback. +#[derive(Debug, Default)] +struct Breaker { + paused_until: Option, + failures: u32, +} + +impl Breaker { + fn record(&mut self, error: Option) { + match error { + None => self.failures = 0, + Some(JevError::Auth) => self.paused_until = Some(Instant::now() + AUTH_PAUSE), + Some(_) => { + self.failures += 1; + if self.failures >= FAILURES_TO_PAUSE { + self.failures = 0; + self.paused_until = Some(Instant::now() + OUTAGE_PAUSE); + } + } + } + } + + fn paused(&self) -> bool { + self.paused_until + .is_some_and(|until| Instant::now() < until) + } +} + +/// The run's Jev client, shared by every flow. +#[derive(Debug)] +pub(crate) struct JevClient { + client: Client, + breaker: Mutex, +} + +impl JevClient { + /// Builds the client from the environment (SPEC 13). + pub(crate) fn from_env() -> Result { + if env::var(API_KEY_ENV).map_or(true, |key| key.is_empty()) { + return Err(JevSetupError::MissingKey); + } + let endpoint = env::var(ENDPOINT_ENV).ok().filter(|url| !url.is_empty()); + let catalog = catalog(endpoint.as_deref()) + .map_err(|source| JevSetupError::Catalog(Box::new(source)))?; + let build = Client::builder() + .catalog(catalog) + .credentials(ConventionalCredentials::new()) + .application("whirl") + .middleware(RetryMiddleware::new(RetryPolicy::default())) + .build() + .map_err(|source| JevSetupError::Client(Box::new(source)))?; + Ok(Self { + client: build.client, + breaker: Mutex::new(Breaker::default()), + }) + } + + /// Asks the questions about `state`. The request, with its retries, + /// ends by `deadline`. + pub(crate) async fn ask( + &self, + state: Json, + questions: Vec<(&'static str, JevQuestion)>, + deadline: Instant, + ) -> Result { + if self.breaker().paused() { + return Err(JevError::Paused); + } + let result = self.evaluate(state, questions, deadline).await; + self.breaker().record(result.as_ref().err().copied()); + result + } + + async fn evaluate( + &self, + state: Json, + questions: Vec<(&'static str, JevQuestion)>, + deadline: Instant, + ) -> Result { + let mut builder = Evaluation::builder() + .model(MODEL) + .state(state) + .timeout(REQUEST_TIMEOUT); + let asked: Vec<(&'static str, bool)> = questions + .iter() + .map(|(id, question)| (*id, matches!(question, JevQuestion::Choice { .. }))) + .collect(); + for (id, question) in questions { + builder = match question { + JevQuestion::Choice { + instructions, + options, + } => builder.choice( + id, + instructions, + options + .into_iter() + .map(|(key, description)| (key, Some(description))), + ), + JevQuestion::Noul { instructions } => builder.boolean(id, instructions), + }; + } + let evaluation = builder.build().map_err(|_| JevError::Failed)?; + let mut context = CallContext::new(); + context.set_deadline(deadline); + let verdict = self + .client + .evaluate_with_context(evaluation, context) + .await + .map_err(|error| match error.kind() { + ErrorKind::Authentication | ErrorKind::AccessDenied => JevError::Auth, + _ => JevError::Failed, + })?; + Ok(response(&verdict, &asked)) + } + + fn breaker(&self) -> MutexGuard<'_, Breaker> { + self.breaker + .lock() + .expect("breaker users do not panic while holding the lock") + } +} + +/// The built-in catalog, with the `typesafe` provider pointed at another +/// server when `endpoint` is set. +fn catalog(endpoint: Option<&str>) -> Result { + let builder = Catalog::builder().with_builtin(); + let Some(url) = endpoint else { + return builder.build(); + }; + let base_url = serde_json::to_string(&format!("{}/v1", url.trim_end_matches('/'))) + .expect("a string always serializes"); + let overlay = format!("schema_version = 1\n\n[providers.typesafe]\nbase_url = {base_url}\n"); + builder.toml_layer(ENDPOINT_ENV, &overlay)?.build() +} + +/// A verdict in the planner's terms. `asked` names each question and +/// whether it was a choice; `lithos-llm` checked that every one has an +/// answer of that kind. +fn response(verdict: &Verdict, asked: &[(&'static str, bool)]) -> JevResponse { + let answers = asked + .iter() + .filter_map(|&(id, is_choice)| { + let answer = if is_choice { + let choice = verdict.choice(id).ok()?; + JevAnswer::Choice { + choice: choice.choice.clone(), + confidence: choice.confidence.unwrap_or_default(), + probabilities: choice + .probabilities + .iter() + .flatten() + .map(|(key, probability)| (key.clone(), *probability)) + .collect(), + } + } else { + JevAnswer::Noul { + noul: verdict.boolean(id).ok()?.probability, + } + }; + Some((id.to_owned(), answer)) + }) + .collect(); + JevResponse { + answers, + spend: JevSpend { + input_tokens: verdict.usage.input.saturating_add(verdict.usage.cache_read), + output_tokens: verdict.usage.output.saturating_add(verdict.usage.reasoning), + cost_usd_micros: verdict.cost.map(|cost| cost.usd_micros), + }, + } +} + +/// A choice question. +pub(crate) fn choice(instructions: Json, options: Map) -> JevQuestion { + JevQuestion::Choice { + instructions, + options, + } +} + +/// A yes-or-no question, answered as a probability. +pub(crate) fn noul(instructions: Json) -> JevQuestion { + JevQuestion::Noul { instructions } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_breaker_pauses_after_a_rejected_key_or_three_failures() { + let mut breaker = Breaker::default(); + breaker.record(Some(JevError::Failed)); + breaker.record(Some(JevError::Failed)); + assert!(!breaker.paused()); + breaker.record(None); + for _ in 0..3 { + breaker.record(Some(JevError::Failed)); + } + assert!(breaker.paused()); + + let mut breaker = Breaker::default(); + breaker.record(Some(JevError::Auth)); + assert!(breaker.paused()); + } + + #[test] + fn the_catalog_routes_jev_to_typesafe_or_to_the_endpoint() { + let builtin = catalog(None).expect("the built-in catalog is valid"); + let local = catalog(Some("http://127.0.0.1:3000/")).expect("the overlay is valid"); + for (catalog, base_url) in [ + (builtin, "https://api.typesafe.ai/v1"), + (local, "http://127.0.0.1:3000/v1"), + ] { + let provider = catalog + .provider("typesafe") + .expect("the catalog has the typesafe provider"); + assert_eq!(provider.base_url(), base_url); + } + } +} diff --git a/crates/whirl/src/run/act/jev/outline.rs b/crates/whirl/src/run/act/jev/outline.rs new file mode 100644 index 0000000..1339408 --- /dev/null +++ b/crates/whirl/src/run/act/jev/outline.rs @@ -0,0 +1,528 @@ +//! The snapshot as a tree, and the candidates Jev chooses among. +//! +//! Jev cannot read a whole snapshot the way a language model does. It picks +//! from a short list, so each candidate carries the context that tells +//! twins apart: the text of its table row or card, the named sections +//! around it, the nearest heading, and its place among elements that look +//! the same. This follows Stagehand's Jev candidate views +//! (browserbase/stagehand#2952). + +use std::collections::{HashMap, HashSet}; +use std::iter; + +use serde_json::{Map, Value as Json, json}; + +/// One line of an AI snapshot. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub(crate) struct OutlineNode { + depth: usize, + parent: Option, + pub(crate) role: String, + pub(crate) name: Option, + /// The element ref, for lines that have one. + pub(crate) element: Option, + /// Text after the line's colon, such as a text box's value. + text: Option, + /// `[active]`: the focused element. + active: bool, + /// `[cursor=pointer]`: the page styles it as clickable. + pointer: bool, +} + +/// The snapshot as a tree, in document order. +#[derive(Clone, Debug, Default)] +pub(crate) struct Outline { + nodes: Vec, +} + +/// Which elements an action can target. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum View { + /// Click, double-click, and hover. + Pointer, + /// Fill. + Input, + /// A native `\"\n\ + ACT \"type {{env.WHIRL_ACT_SECRET}} into the password field\"\n\ + [Asserts]\nlabel:Password value == {{env.WHIRL_ACT_SECRET}}\n", + ); + let secret = "hunter2-jev-secret"; + let output = twin.run_jev(&dir, &flow, &[("WHIRL_ACT_SECRET", secret)]); + let stdout = stdout_text(&output); + assert_eq!(exit_code(&output), 0, "stdout:\n{stdout}"); + + let log = twin.request_log(); + assert!(!log.contains(secret), "the secret reached Jev:\n{log}"); + assert!(log.contains("%env.WHIRL_ACT_SECRET%"), "log:\n{log}"); + let step = act_step(&dir); + assert_eq!(step["act"]["actions"][0]["plannedBy"], "jev"); + assert_eq!( + step["act"]["actions"][0]["line"], + "FILL role:textbox \"Password\" \"%env.WHIRL_ACT_SECRET%\"" + ); +} + +#[test] +fn jev_without_a_key_is_a_runtime_error() { + let dir = TestDir::new(); + let twin = ModelTwin::start(); + let flow = dir.file( + "jev-no-key.whirl", + &format!("[Options]\nmodel: gpt-test\n{SHOP}ACT \"add the item to the cart\"\n"), + ); + let output = twin.run_with_args(&dir, &flow, &[("TYPESAFE_API_KEY", "")], &["--jev"]); + assert_eq!(exit_code(&output), 3); + let stderr = String::from_utf8_lossy(&output.stderr); + assert!( + stderr.contains("--jev needs TYPESAFE_API_KEY"), + "stderr:\n{stderr}" + ); +} diff --git a/crates/whirl/tests/cmd/help.stdout b/crates/whirl/tests/cmd/help.stdout index 7a26274..72e4bdc 100644 --- a/crates/whirl/tests/cmd/help.stdout +++ b/crates/whirl/tests/cmd/help.stdout @@ -35,6 +35,7 @@ Options: --video Record a .webm video of each file's run --video-fps Frames per second for --video on Chromium (1 to 60; default 60) --har Record a .har network log per file + --jev Plan ACT with TypeSafe Jev first, falling back to the model option --storage Override the `storage` option --save-storage Write the final storage state after a successful run --entry-timeout Override the `entry-timeout` option diff --git a/docs/engineering/machine-output.md b/docs/engineering/machine-output.md index fdf94b9..d7df04d 100644 --- a/docs/engineering/machine-output.md +++ b/docs/engineering/machine-output.md @@ -46,10 +46,14 @@ Checks and captures add `type-mismatch`, `filter-error`, and `missing-value`, and report a false predicate as `assert` (SPEC 9.7). An `eval` subject reports the shim kinds `eval` and `eval-result`. `ACT` steps add `act-no-match`, `act-invalid-decision`, `act-model`, -`act-fill-mismatch`, and `stale-ref` (SPEC 7.4). An `ACT` step also has an `act` object: `model`; `actions`, each with a -`line` in Whirl syntax and the model's `description`; and `usage`, with -`modelCalls`, `inputTokens`, `outputTokens`, and `costUsdMicros` when every -call was priced. Arguments keep their `%name%` placeholders. +`act-fill-mismatch`, and `stale-ref` (SPEC 7.4). An `ACT` step also has an `act` object: `model`; +`planner`, `llm` or with `--jev` `jev`; `actions`, each with a `line` in +Whirl syntax, the `description` of the element, and `plannedBy`, the planner +that chose it; and `usage`, with `modelCalls`, `inputTokens`, `outputTokens`, +`costUsdMicros` when every call was priced, and with `--jev` a `jev` object +of `requests`, `inputTokens`, `outputTokens`, and `costUsdMicros` when every +answered request was priced. The step's `costUsdMicros` is the model's cost +plus Jev's; a step with no model call costs only what Jev did. Arguments keep their `%name%` placeholders. `internal` covers an invalid shim result. Codes are stable; message text may change and must not be parsed. Secret masking covers diagnostic action logs in run reports. `check` reports source text without resolving environment variables. diff --git a/docs/engineering/report.schema.json b/docs/engineering/report.schema.json index 23ed40e..8ce8d31 100644 --- a/docs/engineering/report.schema.json +++ b/docs/engineering/report.schema.json @@ -190,6 +190,12 @@ "model": { "type": "string" }, + "planner": { + "enum": [ + "llm", + "jev" + ] + }, "actions": { "type": "array", "items": { @@ -204,6 +210,12 @@ }, "description": { "type": "string" + }, + "plannedBy": { + "enum": [ + "llm", + "jev" + ] } } } @@ -231,6 +243,32 @@ "costUsdMicros": { "type": "integer", "minimum": 0 + }, + "jev": { + "type": "object", + "required": [ + "requests", + "inputTokens", + "outputTokens" + ], + "properties": { + "requests": { + "type": "integer", + "minimum": 0 + }, + "inputTokens": { + "type": "integer", + "minimum": 0 + }, + "outputTokens": { + "type": "integer", + "minimum": 0 + }, + "costUsdMicros": { + "type": "integer", + "minimum": 0 + } + } } } }