diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index e7b575f8..7f7ef812 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -16,7 +16,7 @@ from packaging.version import Version from pydantic import BaseModel from pydantic_ai import Agent, ModelSettings -from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart +from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, UserPromptPart from pydantic_ai.usage import UsageLimits @@ -1561,14 +1561,24 @@ def get_weather(city: str) -> str: _assert_metrics_are_valid(agent_span["metrics"], start, end) -@pytest.mark.vcr -def test_tool_execution_creates_spans(memory_logger): - """Test that executing tools with agents works and creates traced spans.""" - assert not memory_logger.pop() +@pytest.mark.vcr(cassette_name="test_tool_execution_creates_spans") +@pytest.mark.parametrize("trace_tool_execution", [True, False], ids=["live", "from-messages"]) +def test_tool_execution_creates_spans(memory_logger, monkeypatch, trace_tool_execution): + """A synchronous agent run emits one span per tool call.""" + if not trace_tool_execution: + from braintrust.integrations.pydantic_ai.patchers import ToolManagerFunctionToolPatcher + from pydantic_ai._agent_graph import ToolManager - start = time.time() + # Exercise fallback tracing with the real tool execution method. + for patcher in ToolManagerFunctionToolPatcher.sub_patchers: + if patcher.applies(pydantic_ai, None): + name = patcher.target_path.rsplit(".", 1)[-1] + method = getattr(ToolManager, name) + monkeypatch.setattr(ToolManager, name, method.__wrapped__) + + assert not memory_logger.pop() - agent = Agent(MODEL, model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500)) + agent = Agent(MODEL, name="calculator", model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500)) @agent.tool_plain def calculate(operation: str, a: float, b: float) -> float: @@ -1582,7 +1592,6 @@ def calculate(operation: str, a: float, b: float) -> float: # Run the agent with a query that will use the tool result = agent.run_sync("What is 127 multiplied by 49?") - end = time.time() # Verify the tool was actually called and result is correct assert result.output @@ -1594,16 +1603,16 @@ def calculate(operation: str, a: float, b: float) -> float: spans = memory_logger.pop() # Find spans by type - agent_span = _find_agent_span(spans) + sync_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run_sync [calculator]") + agent_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run [calculator]") chat_spans = [s for s in spans if "chat" in s["span_attributes"]["name"]] # Assertions - verify basic tracing works with tools - assert agent_span is not None, "agent_run span should exist" assert len(chat_spans) >= 1, f"Expected at least 1 chat span, got {len(chat_spans)}" # Verify agent span has tool information in input - assert "toolsets" in agent_span["input"], "Tool information should be captured in agent input" - toolsets = agent_span["input"]["toolsets"] + assert "toolsets" in sync_span["input"], "Tool information should be captured in agent input" + toolsets = sync_span["input"]["toolsets"] agent_toolset = next((ts for ts in toolsets if ts.get("id") == ""), None) assert agent_toolset is not None, "Agent toolset should be in input" @@ -1612,18 +1621,18 @@ def calculate(operation: str, a: float, b: float) -> float: tool_names = [t["name"] for t in tools if isinstance(t, dict)] assert "calculate" in tool_names, f"calculate tool should be in toolset, got: {tool_names}" + tool_call_count = sum( + isinstance(part, ToolCallPart) for message in result.new_messages() for part in message.parts + ) tool_spans = [s for s in spans if s["span_attributes"].get("type") == SpanTypeAttribute.TOOL] - assert len(tool_spans) >= 1, f"Expected at least 1 TOOL span, got {len(tool_spans)}" - - calc_tool_span = next((s for s in tool_spans if s["span_attributes"]["name"] == "calculate"), None) - assert calc_tool_span is not None, "calculate TOOL span not found" + assert len(tool_spans) == tool_call_count >= 1, ( + f"Expected one TOOL span per tool call ({tool_call_count}), got {len(tool_spans)}" + ) + calc_tool_span = _single_tool_span(tool_spans, "calculate") assert calc_tool_span["input"] is not None, "tool span should have input" assert calc_tool_span["output"] is not None, "tool span should have output" - # Verify tool span is nested within the agent span tree - all_span_ids = {s["span_id"] for s in spans} - assert calc_tool_span["span_parents"][0] in all_span_ids, ( - "tool span should be nested under a span in the agent tree" - ) + assert agent_span["span_parents"] == [sync_span["span_id"]] + assert calc_tool_span["span_parents"] == [agent_span["span_id"]] def _single_tool_span(spans, name): diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index c4ee2c14..955a1e1a 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -6,6 +6,7 @@ import time from collections.abc import Mapping from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass from typing import Any from braintrust.integrations.utils import _materialize_attachment, _timing_metrics @@ -28,7 +29,19 @@ def start_span(*args, **kwargs): logger = logging.getLogger(__name__) -_tool_trace_state: contextvars.ContextVar[list[int] | None] = contextvars.ContextVar( + + +@dataclass +class _ToolTraceState: + emitted: bool = False + + +@dataclass +class _SyncToolTraceState(_ToolTraceState): + traced_run_result: Any = None + + +_tool_trace_state: contextvars.ContextVar[_ToolTraceState | None] = contextvars.ContextVar( "braintrust_pydantic_ai_tool_trace_state", default=None ) @@ -67,7 +80,7 @@ def _direct_prepare_model_wrapper(wrapped: Any, instance: Any, args: Any, kwargs def _start_tool_trace_capture() -> Any: - return _tool_trace_state.set([0]) + return _tool_trace_state.set(_ToolTraceState()) def _reset_tool_trace_capture(token: Any) -> None: @@ -77,12 +90,12 @@ def _reset_tool_trace_capture(token: Any) -> None: def _mark_tool_span_emitted() -> None: state = _tool_trace_state.get() if state is not None: - state[0] += 1 + state.emitted = True def _maybe_create_tool_spans_from_messages(result: Any) -> None: state = _tool_trace_state.get() - if state is not None and state[0] > 0: + if state is not None and state.emitted: return _create_tool_spans_from_messages(result) @@ -141,6 +154,9 @@ async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any _shape_result_output(result), _wrapper_span_metrics(start_time, end_time), ) + parent_capture = tool_trace_token.old_value + if isinstance(parent_capture, _SyncToolTraceState): + parent_capture.traced_run_result = result return result finally: _reset_tool_trace_capture(tool_trace_token) @@ -155,13 +171,16 @@ def _agent_run_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) input=input_data if input_data else None, metadata=metadata, ) as agent_span: - tool_trace_token = _start_tool_trace_capture() + tool_trace_state = _SyncToolTraceState() + tool_trace_token = _tool_trace_state.set(tool_trace_state) try: start_time = time.time() result = wrapped(*args, **kwargs) end_time = time.time() - _maybe_create_tool_spans_from_messages(result) + # run_sync normally delegates to run, which already traced this result. + if result is not tool_trace_state.traced_run_result: + _maybe_create_tool_spans_from_messages(result) _log_agent_result( agent_span,