Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
from packaging.version import Version
from pydantic import BaseModel
from pydantic_ai import Agent, ModelSettings
from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart
from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, UserPromptPart
from pydantic_ai.usage import UsageLimits


Expand Down Expand Up @@ -1561,14 +1561,24 @@ def get_weather(city: str) -> str:
_assert_metrics_are_valid(agent_span["metrics"], start, end)


@pytest.mark.vcr
def test_tool_execution_creates_spans(memory_logger):
"""Test that executing tools with agents works and creates traced spans."""
assert not memory_logger.pop()
@pytest.mark.vcr(cassette_name="test_tool_execution_creates_spans")
@pytest.mark.parametrize("trace_tool_execution", [True, False], ids=["live", "from-messages"])
def test_tool_execution_creates_spans(memory_logger, monkeypatch, trace_tool_execution):
"""A synchronous agent run emits one span per tool call."""
if not trace_tool_execution:
from braintrust.integrations.pydantic_ai.patchers import ToolManagerFunctionToolPatcher
from pydantic_ai._agent_graph import ToolManager

start = time.time()
# Exercise fallback tracing with the real tool execution method.
for patcher in ToolManagerFunctionToolPatcher.sub_patchers:
if patcher.applies(pydantic_ai, None):
name = patcher.target_path.rsplit(".", 1)[-1]
method = getattr(ToolManager, name)
monkeypatch.setattr(ToolManager, name, method.__wrapped__)

assert not memory_logger.pop()

agent = Agent(MODEL, model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500))
agent = Agent(MODEL, name="calculator", model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500))

@agent.tool_plain
def calculate(operation: str, a: float, b: float) -> float:
Expand All @@ -1582,7 +1592,6 @@ def calculate(operation: str, a: float, b: float) -> float:

# Run the agent with a query that will use the tool
result = agent.run_sync("What is 127 multiplied by 49?")
end = time.time()

# Verify the tool was actually called and result is correct
assert result.output
Expand All @@ -1594,16 +1603,16 @@ def calculate(operation: str, a: float, b: float) -> float:
spans = memory_logger.pop()

# Find spans by type
agent_span = _find_agent_span(spans)
sync_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run_sync [calculator]")
agent_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run [calculator]")
chat_spans = [s for s in spans if "chat" in s["span_attributes"]["name"]]

# Assertions - verify basic tracing works with tools
assert agent_span is not None, "agent_run span should exist"
assert len(chat_spans) >= 1, f"Expected at least 1 chat span, got {len(chat_spans)}"

# Verify agent span has tool information in input
assert "toolsets" in agent_span["input"], "Tool information should be captured in agent input"
toolsets = agent_span["input"]["toolsets"]
assert "toolsets" in sync_span["input"], "Tool information should be captured in agent input"
toolsets = sync_span["input"]["toolsets"]
agent_toolset = next((ts for ts in toolsets if ts.get("id") == "<agent>"), None)
assert agent_toolset is not None, "Agent toolset should be in input"

Expand All @@ -1612,18 +1621,18 @@ def calculate(operation: str, a: float, b: float) -> float:
tool_names = [t["name"] for t in tools if isinstance(t, dict)]
assert "calculate" in tool_names, f"calculate tool should be in toolset, got: {tool_names}"

tool_call_count = sum(
isinstance(part, ToolCallPart) for message in result.new_messages() for part in message.parts
)
tool_spans = [s for s in spans if s["span_attributes"].get("type") == SpanTypeAttribute.TOOL]
assert len(tool_spans) >= 1, f"Expected at least 1 TOOL span, got {len(tool_spans)}"

calc_tool_span = next((s for s in tool_spans if s["span_attributes"]["name"] == "calculate"), None)
assert calc_tool_span is not None, "calculate TOOL span not found"
assert len(tool_spans) == tool_call_count >= 1, (
f"Expected one TOOL span per tool call ({tool_call_count}), got {len(tool_spans)}"
)
calc_tool_span = _single_tool_span(tool_spans, "calculate")
assert calc_tool_span["input"] is not None, "tool span should have input"
assert calc_tool_span["output"] is not None, "tool span should have output"
# Verify tool span is nested within the agent span tree
all_span_ids = {s["span_id"] for s in spans}
assert calc_tool_span["span_parents"][0] in all_span_ids, (
"tool span should be nested under a span in the agent tree"
)
assert agent_span["span_parents"] == [sync_span["span_id"]]
assert calc_tool_span["span_parents"] == [agent_span["span_id"]]


def _single_tool_span(spans, name):
Expand Down
31 changes: 25 additions & 6 deletions py/src/braintrust/integrations/pydantic_ai/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import time
from collections.abc import Mapping
from contextlib import AbstractAsyncContextManager
from dataclasses import dataclass
from typing import Any

from braintrust.integrations.utils import _materialize_attachment, _timing_metrics
Expand All @@ -28,7 +29,19 @@ def start_span(*args, **kwargs):


logger = logging.getLogger(__name__)
_tool_trace_state: contextvars.ContextVar[list[int] | None] = contextvars.ContextVar(


@dataclass
class _ToolTraceState:
emitted: bool = False


@dataclass
class _SyncToolTraceState(_ToolTraceState):
traced_run_result: Any = None


_tool_trace_state: contextvars.ContextVar[_ToolTraceState | None] = contextvars.ContextVar(
"braintrust_pydantic_ai_tool_trace_state", default=None
)

Expand Down Expand Up @@ -67,7 +80,7 @@ def _direct_prepare_model_wrapper(wrapped: Any, instance: Any, args: Any, kwargs


def _start_tool_trace_capture() -> Any:
return _tool_trace_state.set([0])
return _tool_trace_state.set(_ToolTraceState())


def _reset_tool_trace_capture(token: Any) -> None:
Expand All @@ -77,12 +90,12 @@ def _reset_tool_trace_capture(token: Any) -> None:
def _mark_tool_span_emitted() -> None:
state = _tool_trace_state.get()
if state is not None:
state[0] += 1
state.emitted = True


def _maybe_create_tool_spans_from_messages(result: Any) -> None:
state = _tool_trace_state.get()
if state is not None and state[0] > 0:
if state is not None and state.emitted:
return
_create_tool_spans_from_messages(result)

Expand Down Expand Up @@ -141,6 +154,9 @@ async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any
_shape_result_output(result),
_wrapper_span_metrics(start_time, end_time),
)
parent_capture = tool_trace_token.old_value
if isinstance(parent_capture, _SyncToolTraceState):
parent_capture.traced_run_result = result
return result
finally:
_reset_tool_trace_capture(tool_trace_token)
Expand All @@ -155,13 +171,16 @@ def _agent_run_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any)
input=input_data if input_data else None,
metadata=metadata,
) as agent_span:
tool_trace_token = _start_tool_trace_capture()
tool_trace_state = _SyncToolTraceState()
tool_trace_token = _tool_trace_state.set(tool_trace_state)
try:
start_time = time.time()
result = wrapped(*args, **kwargs)
end_time = time.time()

_maybe_create_tool_spans_from_messages(result)
# run_sync normally delegates to run, which already traced this result.
if result is not tool_trace_state.traced_run_result:
_maybe_create_tool_spans_from_messages(result)

_log_agent_result(
agent_span,
Expand Down
Loading