[STG-2515] feat(evals): vercel_ai_sdk bare-loop harness adapter

packages/evals/framework/benchHarness.ts

@@ -31,6 +31,7 @@ import {
   prepareExternalHarnessAdapter,
   type PreparedExternalHarnessAdapter,
 } from "./externalHarnessToolAdapter.js";
+import { runVercelAiSdkAgent } from "./vercelAiSdkRunner.js";
 import type { DiscoveredTask, TaskResult } from "./types.js";
 import type {
   AdapterBackedHarness,
@@ -439,10 +440,16 @@ export function buildAdapterBackedHarness(
   };
 }
 
+export const vercelAiSdkHarness = buildAdapterBackedHarness(
+  "vercel_ai_sdk",
+  runVercelAiSdkAgent,
+);
+
 const harnessRegistry = new Map<Harness, BenchHarness>([
   ["stagehand", stagehandHarness],
   ["claude_code", claudeCodeHarness],
   ["codex", codexHarness],
+  ["vercel_ai_sdk", vercelAiSdkHarness],
 ]);
 
 export function getBenchHarness(harness: Harness): BenchHarness {

packages/evals/framework/benchTypes.ts

@@ -31,6 +31,7 @@ export const EXECUTABLE_BENCH_HARNESSES = [
   "stagehand",
   "claude_code",
   "codex",
+  "vercel_ai_sdk",
 ] as const satisfies readonly Harness[];
 
 /**

packages/evals/framework/vercelAiSdkRunner.ts

@@ -0,0 +1,167 @@
+/**
+ * vercelAiSdkRunner — bare-loop harness via the Vercel AI SDK (`ai` package).
+ *
+ * The loop is `generateText` with `stopWhen: stepCountIs(N)` and a single
+ * `browse` tool — the AI SDK's own multi-step tool loop IS the harness, with
+ * zero additional scaffolding, mirroring the default way a JS developer
+ * wires a tool loop with this SDK.
+ *
+ * Testability: pass `generateTextFn` to drive the loop with a mock. The mock
+ * receives the fully-built options (system, prompt, tools, stopWhen) and can
+ * invoke `options.tools.browse.execute(...)` to exercise tool wiring.
+ */
+import type { AvailableModel } from "@browserbasehq/stagehand";
+import { getAISDKLanguageModel } from "@browserbasehq/stagehand";
+import { EvalsError } from "../errors.js";
+import type { EvalLogger } from "../logger.js";
+import type { TaskResult } from "./types.js";
+import type { ExternalHarnessTaskPlan } from "./externalHarnessPlan.js";
+import type { PreparedExternalHarnessAdapter } from "./externalHarnessToolAdapter.js";
+import { readBareLoopMaxSteps } from "./bareLoopConfig.js";
+import {
+  BROWSE_TOOL_DESCRIPTION,
+  BROWSE_TOOL_NAME,
+  buildBareLoopUserPrompt,
+  createBareLoopToolRecorder,
+  finalizeBareLoopResult,
+  stringifyLoopError,
+} from "./bareLoopRunner.js";
+import type { ExternalHarnessVerifierConfig } from "./verifierAdapter.js";
+
+const HARNESS = "vercel_ai_sdk";
+const MAX_STEPS_ENV = "EVAL_VERCEL_AI_SDK_MAX_STEPS";
+
+export interface VercelAiSdkGenerateTextResult {
+  text: string;
+  steps?: unknown[];
+  totalUsage?: {
+    inputTokens?: number;
+    outputTokens?: number;
+    cachedInputTokens?: number;
+    reasoningTokens?: number;
+  };
+  finishReason?: string;
+}
+
+export type VercelAiSdkGenerateTextFn = (
+  options: Record<string, unknown>,
+) => Promise<VercelAiSdkGenerateTextResult>;
+
+export interface VercelAiSdkRunnerInput {
+  plan: ExternalHarnessTaskPlan;
+  model: AvailableModel;
+  logger: EvalLogger;
+  toolAdapter: PreparedExternalHarnessAdapter;
+  signal?: AbortSignal;
+  /** Injectable for unit tests; defaults to the real `ai` generateText. */
+  generateTextFn?: VercelAiSdkGenerateTextFn;
+  verifier?: ExternalHarnessVerifierConfig;
+}
+
+/**
+ * Resolve "provider/model" to an AI SDK LanguageModel via stagehand's own
+ * provider map — the exact resolution the `-m` flag already uses for the
+ * stagehand harness, so model names mean the same thing on every harness.
+ */
+export function resolveVercelAiSdkModel(model: AvailableModel): unknown {
+  if (!model.includes("/")) {
+    throw new EvalsError(
+      `vercel_ai_sdk harness requires a provider-prefixed model (e.g. anthropic/claude-sonnet-4-6); received "${model}".`,
+    );
+  }
+  const slash = model.indexOf("/");
+  return getAISDKLanguageModel(model.slice(0, slash), model.slice(slash + 1));
+}
+
+export async function runVercelAiSdkAgent(
+  input: VercelAiSdkRunnerInput,
+): Promise<TaskResult> {
+  const maxSteps = readBareLoopMaxSteps(MAX_STEPS_ENV);
+  const recorder = createBareLoopToolRecorder(
+    input.toolAdapter,
+    input.logger,
+    HARNESS,
+  );
+
+  let finalText = "";
+  let stepsUsed = 0;
+  let usage: VercelAiSdkGenerateTextResult["totalUsage"];
+  let loopError: unknown;
+  let cappedOut = false;
+
+  try {
+    const ai = await import("ai");
+    const { z } = await import("zod");
+    const generateTextFn =
+      input.generateTextFn ??
+      (ai.generateText as unknown as VercelAiSdkGenerateTextFn);
+
+    const result = await generateTextFn({
+      model:
+        input.generateTextFn === undefined
+          ? resolveVercelAiSdkModel(input.model)
+          : input.model,
+      system: input.toolAdapter.systemPromptAddendum,
+      prompt: buildBareLoopUserPrompt(input.plan),
+      stopWhen: ai.stepCountIs(maxSteps),
+      ...(input.signal && { abortSignal: input.signal }),
+      tools: {
+        [BROWSE_TOOL_NAME]: ai.tool({
+          description: BROWSE_TOOL_DESCRIPTION,
+          inputSchema: z.object({
+            args: z
+              .string()
+              .describe(
+                'Everything after "browse", e.g. "open https://example.com".',
+              ),
+          }),
+          execute: async ({ args }: { args: string }) => recorder.execute(args),
+        }),
+      },
+    });
+
+    finalText = result.text ?? "";
+    stepsUsed = result.steps?.length ?? 0;
+    usage = result.totalUsage;
+    // stopWhen: stepCountIs(maxSteps) ends the loop silently -- generateText
+    // returns normally either way, so a clean stop and a cap-truncated one are
+    // otherwise indistinguishable. finishReason on the last step is "stop"
+    // when the model was actually done, or "tool-calls" when it still wanted
+    // to call a tool but the step cap cut it off first. Mirrors anthropic_sdk's
+    // stopReason shape so all bare-loop harnesses report the cap the same way.
+    cappedOut = stepsUsed >= maxSteps && result.finishReason === "tool-calls";
+  } catch (error) {
+    loopError = error;
+    input.logger.warn({
+      category: HARNESS,
+      message: `vercel_ai_sdk loop stopped before a normal result: ${stringifyLoopError(error)}`,
+      level: 0,
+    });
+  }
+
+  return finalizeBareLoopResult({
+    harness: HARNESS,
+    toolCalls: recorder.calls,
+    finalText,
+    status: loopError ? "error" : "complete",
+    stopReason: loopError
+      ? stringifyLoopError(loopError)
+      : cappedOut
+        ? `step cap reached (${maxSteps})`
+        : undefined,
+    usage: {
+      input_tokens: usage?.inputTokens ?? 0,
+      output_tokens: usage?.outputTokens ?? 0,
+      ...(usage?.cachedInputTokens !== undefined && {
+        cached_input_tokens: usage.cachedInputTokens,
+      }),
+      ...(usage?.reasoningTokens !== undefined && {
+        reasoning_tokens: usage.reasoningTokens,
+      }),
+    },
+    stepsUsed,
+    maxSteps,
+    logger: input.logger,
+    verifier: input.verifier,
+  });
+}

packages/evals/tests/framework/benchHarness.test.ts

@@ -3,7 +3,9 @@ import {
   claudeCodeHarness,
   codexHarness,
   getBenchHarness,
+  vercelAiSdkHarness,
 } from "../../framework/benchHarness.js";
+import { isExecutableBenchHarness } from "../../framework/benchTypes.js";
 
 describe("bench harness registry", () => {
   it("registers claude_code as a concrete executable harness", () => {
@@ -23,4 +25,18 @@ describe("bench harness registry", () => {
     expect(harness.supportsApi).toBe(false);
     expect(harness.execute).toBeDefined();
   });
+
+  it("registers vercel_ai_sdk as a concrete executable harness", async () => {
+    const harness = getBenchHarness("vercel_ai_sdk");
+
+    expect(isExecutableBenchHarness("vercel_ai_sdk")).toBe(true);
+    expect(harness).toBe(vercelAiSdkHarness);
+    expect(harness.supportedTaskKinds).toEqual(["agent", "suite"]);
+    expect(harness.supportsApi).toBe(false);
+    expect(harness.execute).toBeDefined();
+    // Like claude_code/codex, this harness runs via the execute path only.
+    await expect(harness.start({} as never)).rejects.toThrow(
+      /external harness execute path/,
+    );
+  });
 });

packages/evals/tests/framework/vercelAiSdkRunner.test.ts

@@ -0,0 +1,166 @@
+import fsp from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, describe, expect, it } from "vitest";
+import type { AvailableModel } from "@browserbasehq/stagehand";
+import { EvalLogger } from "../../logger.js";
+import type { ExternalHarnessTaskPlan } from "../../framework/externalHarnessPlan.js";
+import type { PreparedExternalHarnessAdapter } from "../../framework/externalHarnessToolAdapter.js";
+import { BARE_LOOP_DEFAULT_SYSTEM_PROMPT } from "../../framework/externalHarnessToolAdapter.js";
+import {
+  resolveVercelAiSdkModel,
+  runVercelAiSdkAgent,
+} from "../../framework/vercelAiSdkRunner.js";
+
+const plan: ExternalHarnessTaskPlan = {
+  dataset: "webtailbench",
+  taskId: "wtb-1",
+  startUrl: "https://example.com",
+  instruction: "Find the checkout button",
+};
+
+let tempDir: string | undefined;
+
+afterEach(async () => {
+  if (tempDir) {
+    await fsp.rm(tempDir, { recursive: true, force: true });
+    tempDir = undefined;
+  }
+  delete process.env.EVAL_VERCEL_AI_SDK_MAX_STEPS;
+});
+
+async function makeAdapter(): Promise<PreparedExternalHarnessAdapter> {
+  tempDir = await fsp.mkdtemp(
+    path.join(os.tmpdir(), "stagehand-evals-vercel-test-"),
+  );
+  const bin = path.join(tempDir, "browse");
+  await fsp.writeFile(bin, '#!/usr/bin/env bash\necho "browse-output:$@"\n', {
+    mode: 0o755,
+  });
+  return {
+    cwd: tempDir,
+    env: { ...process.env } as Record<string, string>,
+    browseBinPath: bin,
+    skillMode: "none",
+    systemPromptAddendum: BARE_LOOP_DEFAULT_SYSTEM_PROMPT,
+    metadata: { toolCommand: "browse", browseCliEntrypoint: bin },
+    cleanup: async () => {},
+  };
+}
+
+describe("vercel_ai_sdk runner", () => {
+  it("requires a provider-prefixed model", () => {
+    expect(() =>
+      resolveVercelAiSdkModel("claude-sonnet-4-6" as AvailableModel),
+    ).toThrow(/provider-prefixed/);
+  });
+
+  it("drives the loop through generateText with the bare system prompt and records tool calls", async () => {
+    const adapter = await makeAdapter();
+    process.env.EVAL_VERCEL_AI_SDK_MAX_STEPS = "7";
+    let captured: Record<string, unknown> | undefined;
+
+    const result = await runVercelAiSdkAgent({
+      plan,
+      model: "anthropic/claude-sonnet-4-6" as AvailableModel,
+      logger: new EvalLogger(false),
+      toolAdapter: adapter,
+      generateTextFn: async (options) => {
+        captured = options;
+        const tools = options.tools as Record<
+          string,
+          { execute: (input: { args: string }) => Promise<string> }
+        >;
+        const first = await tools.browse.execute({ args: "--help" });
+        expect(first).toContain("browse-output:--help");
+        await tools.browse.execute({ args: "open https://example.com" });
+        return {
+          text: 'done\nEVAL_RESULT: {"success":true,"summary":"found it","finalAnswer":"checkout"}',
+          steps: [{}, {}, {}],
+          totalUsage: { inputTokens: 120, outputTokens: 30 },
+        };
+      },
+    });
+
+    expect(captured?.system).toBe(BARE_LOOP_DEFAULT_SYSTEM_PROMPT);
+    expect(String(captured?.prompt)).toContain("Find the checkout button");
+    expect(String(captured?.prompt)).toContain(
+      "Start URL: https://example.com",
+    );
+    expect(result._success).toBe(true);
+    expect(result.finalAnswer).toBe("checkout");
+    expect(result.vercelAiSdkStatus).toBe("completed");
+    const metrics = result.metrics as Record<string, { value: number }>;
+    expect(metrics.vercel_ai_sdk_tool_calls.value).toBe(2);
+    expect(metrics.vercel_ai_sdk_steps.value).toBe(3);
+    expect(metrics.vercel_ai_sdk_max_steps.value).toBe(7);
+    expect(metrics.vercel_ai_sdk_input_tokens.value).toBe(120);
+    expect(metrics.vercel_ai_sdk_output_tokens.value).toBe(30);
+  });
+
+  it("stops at the step cap and reports it as the stop reason", async () => {
+    const adapter = await makeAdapter();
+    process.env.EVAL_VERCEL_AI_SDK_MAX_STEPS = "2";
+
+    const result = await runVercelAiSdkAgent({
+      plan,
+      model: "anthropic/claude-sonnet-4-6" as AvailableModel,
+      logger: new EvalLogger(false),
+      toolAdapter: adapter,
+      generateTextFn: async () => ({
+        // The model was still mid tool-call when stepCountIs(2) cut the loop
+        // off -- generateText returns normally either way, so finishReason is
+        // the only signal that distinguishes this from a clean stop.
+        text: "",
+        steps: [{}, {}],
+        finishReason: "tool-calls",
+        totalUsage: { inputTokens: 10, outputTokens: 5 },
+      }),
+    });
+
+    expect(result._success).toBe(false);
+    expect(result.vercelAiSdkStatus).toBe("completed");
+    expect(result.vercelAiSdkStopReason).toContain("step cap reached (2)");
+    const metrics = result.metrics as Record<string, { value: number }>;
+    expect(metrics.vercel_ai_sdk_steps.value).toBe(2);
+    expect(metrics.vercel_ai_sdk_max_steps.value).toBe(2);
+  });
+
+  it("does not report a step cap on a clean stop even at the max step count", async () => {
+    const adapter = await makeAdapter();
+    process.env.EVAL_VERCEL_AI_SDK_MAX_STEPS = "2";
+
+    const result = await runVercelAiSdkAgent({
+      plan,
+      model: "anthropic/claude-sonnet-4-6" as AvailableModel,
+      logger: new EvalLogger(false),
+      toolAdapter: adapter,
+      generateTextFn: async () => ({
+        text: 'EVAL_RESULT: {"success":true,"summary":"done","finalAnswer":"checkout"}',
+        steps: [{}, {}],
+        finishReason: "stop",
+        totalUsage: { inputTokens: 10, outputTokens: 5 },
+      }),
+    });
+
+    expect(result._success).toBe(true);
+    expect(result.vercelAiSdkStopReason).toBeUndefined();
+  });
+
+  it("returns a failed task result instead of throwing on loop errors", async () => {
+    const adapter = await makeAdapter();
+    const result = await runVercelAiSdkAgent({
+      plan,
+      model: "anthropic/claude-sonnet-4-6" as AvailableModel,
+      logger: new EvalLogger(false),
+      toolAdapter: adapter,
+      generateTextFn: async () => {
+        throw new Error("provider exploded");
+      },
+    });
+
+    expect(result._success).toBe(false);
+    expect(result.vercelAiSdkStatus).toBe("error");
+    expect(result.vercelAiSdkStopReason).toContain("provider exploded");
+  });
+});