From 38d0157e0da8ee5570602b47cf25eefeb85d29d3 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 12:01:30 -0500 Subject: [PATCH] fix(pydantic_ai): deduplicate tool spans from run_sync MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Agent.run_sync() delegates to Agent.run(). The inner run traces each tool call, and the outer wrapper could reconstruct the same calls from the returned messages. ```text Before After agent_run_sync agent_run_sync |-- agent_run `-- agent_run | `-- calculate `-- calculate `-- calculate (duplicate) ``` Record the result handled by the inner run so the outer wrapper can skip reconstruction of that result. Keep the fallback available for runs whose tool calls still need to be reconstructed from messages. Fixes #1137 --- .../test_pydantic_ai_integration.py | 51 +++++++++++-------- .../integrations/pydantic_ai/tracing.py | 31 ++++++++--- 2 files changed, 55 insertions(+), 27 deletions(-) diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index e7b575f8a..7f7ef8129 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -16,7 +16,7 @@ from packaging.version import Version from pydantic import BaseModel from pydantic_ai import Agent, ModelSettings -from pydantic_ai.messages import ModelRequest, ModelResponse, UserPromptPart +from pydantic_ai.messages import ModelRequest, ModelResponse, ToolCallPart, UserPromptPart from pydantic_ai.usage import UsageLimits @@ -1561,14 +1561,24 @@ def get_weather(city: str) -> str: _assert_metrics_are_valid(agent_span["metrics"], start, end) -@pytest.mark.vcr -def test_tool_execution_creates_spans(memory_logger): - """Test that executing tools with agents works and creates traced spans.""" - assert not memory_logger.pop() +@pytest.mark.vcr(cassette_name="test_tool_execution_creates_spans") +@pytest.mark.parametrize("trace_tool_execution", [True, False], ids=["live", "from-messages"]) +def test_tool_execution_creates_spans(memory_logger, monkeypatch, trace_tool_execution): + """A synchronous agent run emits one span per tool call.""" + if not trace_tool_execution: + from braintrust.integrations.pydantic_ai.patchers import ToolManagerFunctionToolPatcher + from pydantic_ai._agent_graph import ToolManager - start = time.time() + # Exercise fallback tracing with the real tool execution method. + for patcher in ToolManagerFunctionToolPatcher.sub_patchers: + if patcher.applies(pydantic_ai, None): + name = patcher.target_path.rsplit(".", 1)[-1] + method = getattr(ToolManager, name) + monkeypatch.setattr(ToolManager, name, method.__wrapped__) + + assert not memory_logger.pop() - agent = Agent(MODEL, model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500)) + agent = Agent(MODEL, name="calculator", model_settings=ModelSettings(**OPENAI_SETTINGS, max_tokens=500)) @agent.tool_plain def calculate(operation: str, a: float, b: float) -> float: @@ -1582,7 +1592,6 @@ def calculate(operation: str, a: float, b: float) -> float: # Run the agent with a query that will use the tool result = agent.run_sync("What is 127 multiplied by 49?") - end = time.time() # Verify the tool was actually called and result is correct assert result.output @@ -1594,16 +1603,16 @@ def calculate(operation: str, a: float, b: float) -> float: spans = memory_logger.pop() # Find spans by type - agent_span = _find_agent_span(spans) + sync_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run_sync [calculator]") + agent_span = next(s for s in spans if s["span_attributes"]["name"] == "agent_run [calculator]") chat_spans = [s for s in spans if "chat" in s["span_attributes"]["name"]] # Assertions - verify basic tracing works with tools - assert agent_span is not None, "agent_run span should exist" assert len(chat_spans) >= 1, f"Expected at least 1 chat span, got {len(chat_spans)}" # Verify agent span has tool information in input - assert "toolsets" in agent_span["input"], "Tool information should be captured in agent input" - toolsets = agent_span["input"]["toolsets"] + assert "toolsets" in sync_span["input"], "Tool information should be captured in agent input" + toolsets = sync_span["input"]["toolsets"] agent_toolset = next((ts for ts in toolsets if ts.get("id") == ""), None) assert agent_toolset is not None, "Agent toolset should be in input" @@ -1612,18 +1621,18 @@ def calculate(operation: str, a: float, b: float) -> float: tool_names = [t["name"] for t in tools if isinstance(t, dict)] assert "calculate" in tool_names, f"calculate tool should be in toolset, got: {tool_names}" + tool_call_count = sum( + isinstance(part, ToolCallPart) for message in result.new_messages() for part in message.parts + ) tool_spans = [s for s in spans if s["span_attributes"].get("type") == SpanTypeAttribute.TOOL] - assert len(tool_spans) >= 1, f"Expected at least 1 TOOL span, got {len(tool_spans)}" - - calc_tool_span = next((s for s in tool_spans if s["span_attributes"]["name"] == "calculate"), None) - assert calc_tool_span is not None, "calculate TOOL span not found" + assert len(tool_spans) == tool_call_count >= 1, ( + f"Expected one TOOL span per tool call ({tool_call_count}), got {len(tool_spans)}" + ) + calc_tool_span = _single_tool_span(tool_spans, "calculate") assert calc_tool_span["input"] is not None, "tool span should have input" assert calc_tool_span["output"] is not None, "tool span should have output" - # Verify tool span is nested within the agent span tree - all_span_ids = {s["span_id"] for s in spans} - assert calc_tool_span["span_parents"][0] in all_span_ids, ( - "tool span should be nested under a span in the agent tree" - ) + assert agent_span["span_parents"] == [sync_span["span_id"]] + assert calc_tool_span["span_parents"] == [agent_span["span_id"]] def _single_tool_span(spans, name): diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index c4ee2c145..955a1e1a9 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -6,6 +6,7 @@ import time from collections.abc import Mapping from contextlib import AbstractAsyncContextManager +from dataclasses import dataclass from typing import Any from braintrust.integrations.utils import _materialize_attachment, _timing_metrics @@ -28,7 +29,19 @@ def start_span(*args, **kwargs): logger = logging.getLogger(__name__) -_tool_trace_state: contextvars.ContextVar[list[int] | None] = contextvars.ContextVar( + + +@dataclass +class _ToolTraceState: + emitted: bool = False + + +@dataclass +class _SyncToolTraceState(_ToolTraceState): + traced_run_result: Any = None + + +_tool_trace_state: contextvars.ContextVar[_ToolTraceState | None] = contextvars.ContextVar( "braintrust_pydantic_ai_tool_trace_state", default=None ) @@ -67,7 +80,7 @@ def _direct_prepare_model_wrapper(wrapped: Any, instance: Any, args: Any, kwargs def _start_tool_trace_capture() -> Any: - return _tool_trace_state.set([0]) + return _tool_trace_state.set(_ToolTraceState()) def _reset_tool_trace_capture(token: Any) -> None: @@ -77,12 +90,12 @@ def _reset_tool_trace_capture(token: Any) -> None: def _mark_tool_span_emitted() -> None: state = _tool_trace_state.get() if state is not None: - state[0] += 1 + state.emitted = True def _maybe_create_tool_spans_from_messages(result: Any) -> None: state = _tool_trace_state.get() - if state is not None and state[0] > 0: + if state is not None and state.emitted: return _create_tool_spans_from_messages(result) @@ -141,6 +154,9 @@ async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any _shape_result_output(result), _wrapper_span_metrics(start_time, end_time), ) + parent_capture = tool_trace_token.old_value + if isinstance(parent_capture, _SyncToolTraceState): + parent_capture.traced_run_result = result return result finally: _reset_tool_trace_capture(tool_trace_token) @@ -155,13 +171,16 @@ def _agent_run_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any) input=input_data if input_data else None, metadata=metadata, ) as agent_span: - tool_trace_token = _start_tool_trace_capture() + tool_trace_state = _SyncToolTraceState() + tool_trace_token = _tool_trace_state.set(tool_trace_state) try: start_time = time.time() result = wrapped(*args, **kwargs) end_time = time.time() - _maybe_create_tool_spans_from_messages(result) + # run_sync normally delegates to run, which already traced this result. + if result is not tool_trace_state.traced_run_result: + _maybe_create_tool_spans_from_messages(result) _log_agent_result( agent_span,