From 4fb71dcdb4e702320a3f32a4767c1ae8cf2994d7 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 12:24:37 -0500 Subject: [PATCH] fix(pydantic-ai): preserve caller-inferred agent names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Pydantic AI infers an unnamed agent's name from its caller. Braintrust's wrappers could cause that lookup to find an internal variable named `instance`, while the outer span still had a generic name. Run Pydantic AI's lookup in the caller's scope before starting the span. Keep explicit names and `infer_name=False`, and defer inference for `run_stream()` until the caller enters the stream. For `my_agent.run_sync("hi")`, the outer and inner span names become: ```text Before agent_run_sync -> agent_run [instance] After agent_run_sync [my_agent] -> agent_run [my_agent] ``` Fixes #1139 Refs: #1141 --- .../test_auto_pydantic_ai.py | 30 ++++--- .../test_pydantic_ai_integration.py | 14 +++- .../integrations/pydantic_ai/tracing.py | 79 +++++++++++++++++-- 3 files changed, 104 insertions(+), 19 deletions(-) diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py index f385dcb9c..adbd0d4f2 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pydantic_ai.py @@ -16,6 +16,7 @@ with autoinstrument_test_context("test_auto_pydantic_ai", integration="pydantic_ai") as memory_logger: from pydantic_ai import Agent from pydantic_ai.models.openai import OpenAIChatModel + from pydantic_ai.models.test import TestModel from pydantic_ai.settings import ModelSettings agent = Agent( @@ -23,15 +24,26 @@ model_settings=ModelSettings(max_tokens=100, openai_reasoning_effort="none"), ) - import asyncio - - result = asyncio.run(agent.run("Say hi")) + result = agent.run_sync("Say hi") assert result.output - - spans = memory_logger.pop() - assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}" - # Find the agent_run span - agent_spans = [s for s in spans if "agent_run" in s["span_attributes"]["name"]] - assert len(agent_spans) >= 1, f"Expected agent_run span, got {[s['span_attributes']['name'] for s in spans]}" + assert agent.name == "agent" + + span_names = {span["span_attributes"]["name"] for span in memory_logger.pop()} + assert {"agent_run_sync [agent]", "agent_run [agent]"} <= span_names + + agents = [Agent(TestModel())] + agents[0].run_sync("Say hi") + assert agents[0].name is None + + opt_out_agent = Agent(TestModel()) + opt_out_agent.run_sync("Say hi", infer_name=False) + assert opt_out_agent.name is None + span_names = [span["span_attributes"]["name"] for span in memory_logger.pop()] + assert sorted(name for name in span_names if name.startswith("agent_run")) == [ + "agent_run", + "agent_run", + "agent_run_sync", + "agent_run_sync", + ] print("SUCCESS") diff --git a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py index e7b575f8a..80e9e619d 100644 --- a/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py +++ b/py/src/braintrust/integrations/pydantic_ai/test_pydantic_ai_integration.py @@ -255,8 +255,9 @@ async def test_agent_run_async(memory_logger, agent_kind): chat_span = next(span for span in spans if span["span_attributes"]["type"] == SpanTypeAttribute.LLM) assert agent_span is not None assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK - if agent_kind == "wrapper": - assert agent_span["span_attributes"]["name"] == "agent_run [wrapped-agent]" + # Unnamed agents must get the caller's variable name, not the wrapper's `instance` argument. + expected_name = "wrapped-agent" if agent_kind == "wrapper" else "agent" + assert agent_span["span_attributes"]["name"] == f"agent_run [{expected_name}]" assert agent_span["input"]["user_prompt"] == TEST_PROMPT assert "4" in str(agent_span["output"]) assert chat_span["span_parents"] == [agent_span["span_id"]] @@ -301,6 +302,8 @@ def test_agent_run_sync(memory_logger): assert chat_span is not None, "chat span not found" # Check agent span + assert agent.name == "agent" + assert agent_sync_span["span_attributes"]["name"] == "agent_run_sync [agent]" assert agent_sync_span["span_attributes"]["type"] == SpanTypeAttribute.TASK assert agent_sync_span["metadata"]["model"] == "gpt-6-luna" assert agent_sync_span["metadata"]["provider"] == "openai" @@ -472,7 +475,9 @@ async def test_agent_run_stream(memory_logger): start = time.time() full_text = "" - async with agent.run_stream("Count from 1 to 5") as result: + stream = agent.run_stream("Count from 1 to 5") + assert agent.name is None + async with stream as result: async for text in result.stream_text(delta=True): full_text += text end = time.time() @@ -494,6 +499,7 @@ async def test_agent_run_stream(memory_logger): # Check agent span assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert agent_span["span_attributes"]["name"] == "agent_run_stream [agent]" assert agent_span["metadata"]["model"] == "gpt-6-luna" assert "Count from 1 to 5" in str(agent_span["input"]) _assert_metrics_are_valid(agent_span["metrics"], start, end) @@ -937,6 +943,7 @@ def test_agent_run_stream_sync(memory_logger): # Check agent span assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert agent_span["span_attributes"]["name"] == "agent_run_stream_sync [agent]" assert agent_span["metadata"]["model"] == "gpt-6-luna" assert "Count from 1 to 3" in str(agent_span["input"]) _assert_metrics_are_valid(agent_span["metrics"], start, end) @@ -1011,6 +1018,7 @@ async def test_agent_run_stream_events(memory_logger): # Check agent span has basic structure assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK + assert agent_span["span_attributes"]["name"] == "agent_run_stream_events [agent]" assert agent_span["metadata"]["model"] == "gpt-6-luna" assert "5+5" in str(agent_span["input"]) or "What" in str(agent_span["input"]) assert agent_span["metrics"]["event_count"] == event_count diff --git a/py/src/braintrust/integrations/pydantic_ai/tracing.py b/py/src/braintrust/integrations/pydantic_ai/tracing.py index c4ee2c145..6010f63e1 100644 --- a/py/src/braintrust/integrations/pydantic_ai/tracing.py +++ b/py/src/braintrust/integrations/pydantic_ai/tracing.py @@ -14,6 +14,7 @@ _INSTRUMENTATION = "pydantic-ai-auto" +_AGENT_WRAPPER_MODULES = (__name__, "braintrust.integrations.base", "wrapt.wrappers") def start_span(*args, **kwargs): @@ -117,7 +118,50 @@ def _log_agent_result(span: Any, input_data: dict[str, Any], result: Any, output span.log(input=resolved_input, output=output, metrics=metrics, metadata=metadata) +def _should_infer_agent_name(wrapped: Any, instance: Any, kwargs: dict[str, Any]) -> bool: + try: + if not kwargs.get("infer_name", True) or not callable(getattr(instance, "_infer_name", None)): + return False + + # Manual wrapping can also target overrides with a different signature. + param = inspect.signature(wrapped).parameters.get("infer_name") + return ( + param is not None + and param.kind == inspect.Parameter.KEYWORD_ONLY + and ("infer_name" in kwargs or param.default is True) + ) + except Exception: + logger.debug("Failed to inspect Pydantic AI agent name inference", exc_info=True) + return False + + +def _infer_name_from_caller(instance: Any) -> None: + # Pydantic AI looks one frame back from the frame we pass it. Skip Braintrust + # and wrapt so its normal locals/globals lookup runs in the caller's frame. + frame = inspect.currentframe() + try: + while frame is not None and (caller := frame.f_back) is not None: + if caller.f_globals.get("__name__") not in _AGENT_WRAPPER_MODULES: + instance._infer_name(frame) + return + frame = caller + except Exception: + logger.debug("Failed to infer Pydantic AI agent name", exc_info=True) + finally: + del frame + + +def _infer_agent_name(wrapped: Any, instance: Any, kwargs: dict[str, Any]) -> dict[str, Any]: + if getattr(instance, "name", "") is None and _should_infer_agent_name(wrapped, instance, kwargs): + _infer_name_from_caller(instance) + if instance.name is None: + # With no caller binding, a second lookup would see the wrapper's locals. + return {**kwargs, "infer_name": False} + return kwargs + + async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): + kwargs = _infer_agent_name(wrapped, instance, kwargs) input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance) with start_span( @@ -147,6 +191,7 @@ async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any def _agent_run_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): + kwargs = _infer_agent_name(wrapped, instance, kwargs) input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance) with start_span( @@ -195,18 +240,21 @@ def _agent_to_cli_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: A def _agent_run_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance) - agent_name = instance.name if hasattr(instance, "name") else None - span_name = f"agent_run_stream [{agent_name}]" if agent_name else "agent_run_stream" + # Pydantic AI infers the name on stream entry, in the frame that enters it. + infer_name = _should_infer_agent_name(wrapped, instance, kwargs) + stream_kwargs = {**kwargs, "infer_name": False} if infer_name else kwargs return _AgentStreamWrapper( - wrapped(*args, **kwargs), - span_name, + wrapped(*args, **stream_kwargs), + instance, + infer_name, input_data, metadata, ) def _agent_run_stream_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): + kwargs = _infer_agent_name(wrapped, instance, kwargs) input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance) agent_name = instance.name if hasattr(instance, "name") else None span_name = f"agent_run_stream_sync [{agent_name}]" if agent_name else "agent_run_stream_sync" @@ -241,6 +289,7 @@ def _agent_run_stream_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwarg def _agent_run_stream_events_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any): + kwargs = _infer_agent_name(wrapped, instance, kwargs) input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance) agent_name = instance.name if hasattr(instance, "name") else None @@ -656,9 +705,17 @@ def _extract_stream_response(stream: Any) -> Any: class _AgentStreamWrapper(AbstractAsyncContextManager): - def __init__(self, stream_cm: Any, span_name: str, input_data: Any, metadata: Any): + def __init__( + self, + stream_cm: Any, + agent: Any, + infer_name: bool, + input_data: Any, + metadata: Any, + ): self.stream_cm = stream_cm - self.span_name = span_name + self.agent = agent + self.infer_name = infer_name self.input_data = input_data self.metadata = metadata self.span_cm = None @@ -671,8 +728,16 @@ def __init__(self, stream_cm: Any, span_name: str, input_data: Any, metadata: An async def __aenter__(self): self._enter_task = asyncio.current_task() + if self.infer_name and self.agent.name is None: + _infer_name_from_caller(self.agent) + agent_name = getattr(self.agent, "name", None) + if agent_name is not None: + self.metadata["agent_name"] = agent_name + else: + self.metadata.pop("agent_name", None) + self.span_cm = start_span( - name=self.span_name, + name=f"agent_run_stream [{agent_name}]" if agent_name else "agent_run_stream", type=SpanTypeAttribute.TASK, input=self.input_data if self.input_data else None, metadata=self.metadata,