Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions packages/core/lib/modelUtils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,14 @@ import {
AvailableCuaModel,
} from "./v3/types/public/agent.js";

//useful when resolving a model from string or object formats we accept
export function extractModelName(
model?: string | { modelName: string },
): string | undefined {
if (!model) return undefined;
return typeof model === "string" ? model : model.modelName;
}

export function splitModelName(model: string): {
provider: string;
modelName: string;
Expand All @@ -20,8 +28,7 @@ export function resolveModel(model: string | ModelConfiguration): {
clientOptions: ClientOptions;
isCua: boolean;
} {
// Extract the model string and client options
const modelString = typeof model === "string" ? model : model.modelName;
const modelString = extractModelName(model)!;
const clientOptions =
typeof model === "string"
? {}
Expand Down
3 changes: 2 additions & 1 deletion packages/core/lib/v3/agent/tools/act.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,9 @@ import { tool } from "ai";
import { z } from "zod";
import type { V3 } from "../../v3.js";
import type { Action } from "../../types/public/methods.js";
import type { AgentModelConfig } from "../../types/public/agent.js";

export const actTool = (v3: V3, executionModel?: string) =>
export const actTool = (v3: V3, executionModel?: string | AgentModelConfig) =>
tool({
description:
"Perform an action on the page (click, type). Provide a short, specific phrase that mentions the element type.",
Expand Down
6 changes: 5 additions & 1 deletion packages/core/lib/v3/agent/tools/extract.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { tool } from "ai";
import { z, ZodTypeAny } from "zod";
import type { V3 } from "../../v3.js";
import type { AgentModelConfig } from "../../types/public/agent.js";

interface JsonSchema {
type?: string;
Expand Down Expand Up @@ -44,7 +45,10 @@ function jsonSchemaToZod(schema: JsonSchema): ZodTypeAny {
}
}

export const extractTool = (v3: V3, executionModel?: string) =>
export const extractTool = (
v3: V3,
executionModel?: string | AgentModelConfig,
) =>
tool({
description: `Extract structured data from the current page based on a provided schema.

Expand Down
6 changes: 5 additions & 1 deletion packages/core/lib/v3/agent/tools/fillform.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,12 @@ import { tool } from "ai";
import { z } from "zod";
import type { V3 } from "../../v3.js";
import type { Action } from "../../types/public/methods.js";
import type { AgentModelConfig } from "../../types/public/agent.js";

export const fillFormTool = (v3: V3, executionModel?: string) =>
export const fillFormTool = (
v3: V3,
executionModel?: string | AgentModelConfig,
) =>
tool({
description: `📝 FORM FILL - MULTI-FIELD INPUT TOOL\nFor any form with 2+ inputs/textareas. Faster than individual typing.`,
inputSchema: z.object({
Expand Down
7 changes: 5 additions & 2 deletions packages/core/lib/v3/agent/tools/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,13 @@ import { searchTool } from "./search.js";
import type { ToolSet, InferUITools } from "ai";
import type { V3 } from "../../v3.js";
import type { LogLine } from "../../types/public/logs.js";
import type { AgentToolMode } from "../../types/public/agent.js";
import type {
AgentToolMode,
AgentModelConfig,
} from "../../types/public/agent.js";

export interface V3AgentToolOptions {
executionModel?: string;
executionModel?: string | AgentModelConfig;
logger?: (message: LogLine) => void;
/**
* Tool mode determines which set of tools are available.
Expand Down
5 changes: 3 additions & 2 deletions packages/core/lib/v3/handlers/v3AgentHandler.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import {
AgentStreamResult,
AgentStreamCallbacks,
AgentToolMode,
AgentModelConfig,
} from "../types/public/agent.js";
import { V3FunctionName } from "../types/public/methods.js";
import { mapToolResultToActions } from "../agent/utils/actionMapping.js";
Expand Down Expand Up @@ -68,7 +69,7 @@ export class V3AgentHandler {
private v3: V3;
private logger: (message: LogLine) => void;
private llmClient: LLMClient;
private executionModel?: string;
private executionModel?: string | AgentModelConfig;
private systemInstructions?: string;
private mcpTools?: ToolSet;
private mode: AgentToolMode;
Expand All @@ -77,7 +78,7 @@ export class V3AgentHandler {
v3: V3,
logger: (message: LogLine) => void,
llmClient: LLMClient,
executionModel?: string,
executionModel?: string | AgentModelConfig,
systemInstructions?: string,
mcpTools?: ToolSet,
mode?: AgentToolMode,
Expand Down
16 changes: 8 additions & 8 deletions packages/core/lib/v3/v3.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import {
toJsonSchema,
} from "./zodCompat.js";
import { loadApiKeyFromEnv } from "../utils.js";
import { extractModelName } from "../modelUtils.js";
import { StagehandLogger, LoggerOptions } from "../logger.js";
import { ActCache } from "./cache/ActCache.js";
import { AgentCache } from "./cache/AgentCache.js";
Expand Down Expand Up @@ -1678,13 +1679,13 @@ export class V3 {
? this.resolveLlmClient(options.model)
: this.llmClient;

const resolvedExecutionModel = options?.executionModel ?? options?.model;

const handler = new V3AgentHandler(
this,
this.logger,
agentLlmClient,
typeof options?.executionModel === "string"
? options.executionModel
: options?.executionModel?.modelName,
resolvedExecutionModel,
options?.systemPrompt,
tools,
options?.mode,
Expand Down Expand Up @@ -1782,11 +1783,10 @@ export class V3 {
auxiliary: {
cua: { value: isCuaMode ? "true" : "false", type: "boolean" },
mode: { value: options?.mode ?? "dom", type: "string" },
model: options?.model
? typeof options?.model === "string"
? { value: options.model, type: "string" }
: { value: options.model.modelName, type: "string" }
: { value: this.llmClient.modelName, type: "string" },
model: {
value: extractModelName(options?.model) ?? this.llmClient.modelName,
type: "string",
},
systemPrompt: { value: options?.systemPrompt ?? "", type: "string" },
tools: { value: JSON.stringify(options?.tools ?? {}), type: "object" },
...(options?.integrations && {
Expand Down
170 changes: 170 additions & 0 deletions packages/core/tests/agent-execution-model.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
import { describe, expect, it, vi } from "vitest";
import { actTool } from "../lib/v3/agent/tools/act.js";
import { extractTool } from "../lib/v3/agent/tools/extract.js";
import { fillFormTool } from "../lib/v3/agent/tools/fillform.js";
import type { V3 } from "../lib/v3/v3.js";

/**
* Minimal mock of V3 that captures how tools pass `model` options
* into v3.act(), v3.extract(), and v3.observe().
*/
function createMockV3() {
const calls: { method: string; model: unknown }[] = [];

const mock = {
logger: vi.fn(),
recordAgentReplayStep: vi.fn(),
act: vi.fn(async (_instruction: unknown, options?: { model?: unknown }) => {
calls.push({ method: "act", model: options?.model });
return {
success: true,
message: "ok",
actionDescription: "clicked",
actions: [],
};
}),
extract: vi.fn(
async (
_instruction: unknown,
_schema: unknown,
options?: { model?: unknown },
) => {
calls.push({ method: "extract", model: options?.model });
return { extraction: "data" };
},
),
observe: vi.fn(
async (_instruction: unknown, options?: { model?: unknown }) => {
calls.push({ method: "observe", model: options?.model });
return [];
},
),
calls,
};

return mock as unknown as V3 & { calls: typeof calls };
}

describe("agent tools pass full executionModel config to v3 methods", () => {
const modelConfig = {
modelName: "openai/gpt-4o-mini",
apiKey: "sk-test-key",
baseURL: "https://custom.api",
};

it("actTool passes AgentModelConfig object to v3.act()", async () => {
const v3 = createMockV3();
const tool = actTool(v3, modelConfig);
await tool.execute!(
{ action: "click the button" },
{
toolCallId: "t1",
messages: [],
abortSignal: new AbortController().signal,
},
);

expect(v3.calls).toHaveLength(1);
expect(v3.calls[0].method).toBe("act");
expect(v3.calls[0].model).toBe(modelConfig);
});

it("extractTool passes AgentModelConfig object to v3.extract()", async () => {
const v3 = createMockV3();
const tool = extractTool(v3, modelConfig);
await tool.execute!(
{ instruction: "get the title", schema: undefined },
{
toolCallId: "t2",
messages: [],
abortSignal: new AbortController().signal,
},
);

expect(v3.calls).toHaveLength(1);
expect(v3.calls[0].method).toBe("extract");
expect(v3.calls[0].model).toBe(modelConfig);
});

it("fillFormTool passes AgentModelConfig object to v3.observe()", async () => {
const v3 = createMockV3();
const tool = fillFormTool(v3, modelConfig);
await tool.execute!(
{ fields: [{ action: "type hello into name", value: "hello" }] },
{
toolCallId: "t3",
messages: [],
abortSignal: new AbortController().signal,
},
);

expect(v3.calls).toHaveLength(1);
expect(v3.calls[0].method).toBe("observe");
expect(v3.calls[0].model).toBe(modelConfig);
});

it("actTool passes undefined when no executionModel is set", async () => {
const v3 = createMockV3();
const tool = actTool(v3, undefined);
await tool.execute!(
{ action: "click the button" },
{
toolCallId: "t4",
messages: [],
abortSignal: new AbortController().signal,
},
);

expect(v3.calls).toHaveLength(1);
expect(v3.calls[0].model).toBeUndefined();
});

it("actTool passes plain string executionModel to v3.act()", async () => {
const v3 = createMockV3();
const tool = actTool(v3, "openai/gpt-4o-mini");
await tool.execute!(
{ action: "click the button" },
{
toolCallId: "t5",
messages: [],
abortSignal: new AbortController().signal,
},
);

expect(v3.calls).toHaveLength(1);
expect(v3.calls[0].model).toBe("openai/gpt-4o-mini");
});
});

describe("executionModel fallback logic", () => {
// This mirrors the resolution in V3.prepareAgentExecution (v3.ts:1682):
// const resolvedExecutionModel = options?.executionModel ?? options?.model;
function resolveExecutionModel(options?: {
executionModel?: string | { modelName: string };
model?: string | { modelName: string };
}) {
return options?.executionModel ?? options?.model;
}

it("prefers explicit executionModel over model", () => {
const result = resolveExecutionModel({
executionModel: "openai/gpt-4o-mini",
model: "anthropic/claude-sonnet-4-20250514",
});
expect(result).toBe("openai/gpt-4o-mini");
});

it("falls back to model when executionModel is not set", () => {
const modelConfig = {
modelName: "anthropic/claude-sonnet-4-20250514",
apiKey: "sk-test",
};
const result = resolveExecutionModel({ model: modelConfig });
expect(result).toBe(modelConfig);
});

it("returns undefined when neither is set", () => {
expect(resolveExecutionModel({})).toBeUndefined();
expect(resolveExecutionModel(undefined)).toBeUndefined();
});
});
49 changes: 49 additions & 0 deletions packages/core/tests/model-utils.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
import { describe, expect, it } from "vitest";
import { extractModelName, resolveModel } from "../lib/modelUtils.js";

describe("extractModelName", () => {
it("returns undefined for undefined input", () => {
expect(extractModelName(undefined)).toBeUndefined();
});

it("returns the string as-is for a string input", () => {
expect(extractModelName("openai/gpt-4o")).toBe("openai/gpt-4o");
});

it("returns modelName from an object input", () => {
expect(
extractModelName({ modelName: "anthropic/claude-sonnet-4-20250514" }),
).toBe("anthropic/claude-sonnet-4-20250514");
});

it("returns modelName from an object with extra properties", () => {
expect(
extractModelName({
modelName: "openai/gpt-4o-mini",
apiKey: "sk-test",
baseURL: "https://custom.endpoint",
}),
).toBe("openai/gpt-4o-mini");
});
});

describe("resolveModel", () => {
it("extracts provider and modelName from a string", () => {
const result = resolveModel("openai/gpt-4o");
expect(result.provider).toBe("openai");
expect(result.modelName).toBe("gpt-4o");
expect(result.clientOptions).toEqual({});
});

it("extracts clientOptions from an object config", () => {
const result = resolveModel({
modelName: "openai/gpt-4o" as never,
apiKey: "sk-test",
});
expect(result.provider).toBe("openai");
expect(result.modelName).toBe("gpt-4o");
expect(result.clientOptions).toMatchObject({ apiKey: "sk-test" });
// modelName should not leak into clientOptions
expect(result.clientOptions).not.toHaveProperty("modelName");
});
});
Loading