Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -16,22 +16,34 @@
with autoinstrument_test_context("test_auto_pydantic_ai", integration="pydantic_ai") as memory_logger:
from pydantic_ai import Agent
from pydantic_ai.models.openai import OpenAIChatModel
from pydantic_ai.models.test import TestModel
from pydantic_ai.settings import ModelSettings

agent = Agent(
OpenAIChatModel("gpt-6-luna"),
model_settings=ModelSettings(max_tokens=100, openai_reasoning_effort="none"),
)

import asyncio

result = asyncio.run(agent.run("Say hi"))
result = agent.run_sync("Say hi")
assert result.output

spans = memory_logger.pop()
assert len(spans) >= 1, f"Expected at least 1 span, got {len(spans)}"
# Find the agent_run span
agent_spans = [s for s in spans if "agent_run" in s["span_attributes"]["name"]]
assert len(agent_spans) >= 1, f"Expected agent_run span, got {[s['span_attributes']['name'] for s in spans]}"
assert agent.name == "agent"

span_names = {span["span_attributes"]["name"] for span in memory_logger.pop()}
assert {"agent_run_sync [agent]", "agent_run [agent]"} <= span_names

agents = [Agent(TestModel())]
agents[0].run_sync("Say hi")
assert agents[0].name is None

opt_out_agent = Agent(TestModel())
opt_out_agent.run_sync("Say hi", infer_name=False)
assert opt_out_agent.name is None
span_names = [span["span_attributes"]["name"] for span in memory_logger.pop()]
assert sorted(name for name in span_names if name.startswith("agent_run")) == [
"agent_run",
"agent_run",
"agent_run_sync",
"agent_run_sync",
]

print("SUCCESS")
Original file line number Diff line number Diff line change
Expand Up @@ -255,8 +255,9 @@ async def test_agent_run_async(memory_logger, agent_kind):
chat_span = next(span for span in spans if span["span_attributes"]["type"] == SpanTypeAttribute.LLM)
assert agent_span is not None
assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK
if agent_kind == "wrapper":
assert agent_span["span_attributes"]["name"] == "agent_run [wrapped-agent]"
# Unnamed agents must get the caller's variable name, not the wrapper's `instance` argument.
expected_name = "wrapped-agent" if agent_kind == "wrapper" else "agent"
assert agent_span["span_attributes"]["name"] == f"agent_run [{expected_name}]"
assert agent_span["input"]["user_prompt"] == TEST_PROMPT
assert "4" in str(agent_span["output"])
assert chat_span["span_parents"] == [agent_span["span_id"]]
Expand Down Expand Up @@ -301,6 +302,8 @@ def test_agent_run_sync(memory_logger):
assert chat_span is not None, "chat span not found"

# Check agent span
assert agent.name == "agent"
assert agent_sync_span["span_attributes"]["name"] == "agent_run_sync [agent]"
assert agent_sync_span["span_attributes"]["type"] == SpanTypeAttribute.TASK
assert agent_sync_span["metadata"]["model"] == "gpt-6-luna"
assert agent_sync_span["metadata"]["provider"] == "openai"
Expand Down Expand Up @@ -472,7 +475,9 @@ async def test_agent_run_stream(memory_logger):

start = time.time()
full_text = ""
async with agent.run_stream("Count from 1 to 5") as result:
stream = agent.run_stream("Count from 1 to 5")
assert agent.name is None
async with stream as result:
async for text in result.stream_text(delta=True):
full_text += text
end = time.time()
Expand All @@ -494,6 +499,7 @@ async def test_agent_run_stream(memory_logger):

# Check agent span
assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK
assert agent_span["span_attributes"]["name"] == "agent_run_stream [agent]"
assert agent_span["metadata"]["model"] == "gpt-6-luna"
assert "Count from 1 to 5" in str(agent_span["input"])
_assert_metrics_are_valid(agent_span["metrics"], start, end)
Expand Down Expand Up @@ -937,6 +943,7 @@ def test_agent_run_stream_sync(memory_logger):

# Check agent span
assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK
assert agent_span["span_attributes"]["name"] == "agent_run_stream_sync [agent]"
assert agent_span["metadata"]["model"] == "gpt-6-luna"
assert "Count from 1 to 3" in str(agent_span["input"])
_assert_metrics_are_valid(agent_span["metrics"], start, end)
Expand Down Expand Up @@ -1011,6 +1018,7 @@ async def test_agent_run_stream_events(memory_logger):

# Check agent span has basic structure
assert agent_span["span_attributes"]["type"] == SpanTypeAttribute.TASK
assert agent_span["span_attributes"]["name"] == "agent_run_stream_events [agent]"
assert agent_span["metadata"]["model"] == "gpt-6-luna"
assert "5+5" in str(agent_span["input"]) or "What" in str(agent_span["input"])
assert agent_span["metrics"]["event_count"] == event_count
Expand Down
79 changes: 72 additions & 7 deletions py/src/braintrust/integrations/pydantic_ai/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@


_INSTRUMENTATION = "pydantic-ai-auto"
_AGENT_WRAPPER_MODULES = (__name__, "braintrust.integrations.base", "wrapt.wrappers")


def start_span(*args, **kwargs):
Expand Down Expand Up @@ -117,7 +118,50 @@ def _log_agent_result(span: Any, input_data: dict[str, Any], result: Any, output
span.log(input=resolved_input, output=output, metrics=metrics, metadata=metadata)


def _should_infer_agent_name(wrapped: Any, instance: Any, kwargs: dict[str, Any]) -> bool:
try:
if not kwargs.get("infer_name", True) or not callable(getattr(instance, "_infer_name", None)):
return False

# Manual wrapping can also target overrides with a different signature.
param = inspect.signature(wrapped).parameters.get("infer_name")
return (
param is not None
and param.kind == inspect.Parameter.KEYWORD_ONLY
and ("infer_name" in kwargs or param.default is True)
)
except Exception:
logger.debug("Failed to inspect Pydantic AI agent name inference", exc_info=True)
return False


def _infer_name_from_caller(instance: Any) -> None:
# Pydantic AI looks one frame back from the frame we pass it. Skip Braintrust
# and wrapt so its normal locals/globals lookup runs in the caller's frame.
frame = inspect.currentframe()
try:
while frame is not None and (caller := frame.f_back) is not None:
if caller.f_globals.get("__name__") not in _AGENT_WRAPPER_MODULES:
instance._infer_name(frame)
return
frame = caller
except Exception:
logger.debug("Failed to infer Pydantic AI agent name", exc_info=True)
finally:
del frame


def _infer_agent_name(wrapped: Any, instance: Any, kwargs: dict[str, Any]) -> dict[str, Any]:
if getattr(instance, "name", "") is None and _should_infer_agent_name(wrapped, instance, kwargs):
_infer_name_from_caller(instance)
if instance.name is None:
# With no caller binding, a second lookup would see the wrapper's locals.
return {**kwargs, "infer_name": False}
return kwargs


async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
kwargs = _infer_agent_name(wrapped, instance, kwargs)
input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance)

with start_span(
Expand Down Expand Up @@ -147,6 +191,7 @@ async def _agent_run_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any


def _agent_run_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
kwargs = _infer_agent_name(wrapped, instance, kwargs)
input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance)

with start_span(
Expand Down Expand Up @@ -195,18 +240,21 @@ def _agent_to_cli_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: A

def _agent_run_stream_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance)
agent_name = instance.name if hasattr(instance, "name") else None
span_name = f"agent_run_stream [{agent_name}]" if agent_name else "agent_run_stream"
# Pydantic AI infers the name on stream entry, in the frame that enters it.
infer_name = _should_infer_agent_name(wrapped, instance, kwargs)
stream_kwargs = {**kwargs, "infer_name": False} if infer_name else kwargs

return _AgentStreamWrapper(
wrapped(*args, **kwargs),
span_name,
wrapped(*args, **stream_kwargs),
instance,
infer_name,
input_data,
metadata,
)


def _agent_run_stream_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
kwargs = _infer_agent_name(wrapped, instance, kwargs)
input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance)
agent_name = instance.name if hasattr(instance, "name") else None
span_name = f"agent_run_stream_sync [{agent_name}]" if agent_name else "agent_run_stream_sync"
Expand Down Expand Up @@ -241,6 +289,7 @@ def _agent_run_stream_sync_wrapper(wrapped: Any, instance: Any, args: Any, kwarg


def _agent_run_stream_events_wrapper(wrapped: Any, instance: Any, args: Any, kwargs: Any):
kwargs = _infer_agent_name(wrapped, instance, kwargs)
input_data, metadata = _build_agent_input_and_metadata(args, kwargs, instance)

agent_name = instance.name if hasattr(instance, "name") else None
Expand Down Expand Up @@ -656,9 +705,17 @@ def _extract_stream_response(stream: Any) -> Any:


class _AgentStreamWrapper(AbstractAsyncContextManager):
def __init__(self, stream_cm: Any, span_name: str, input_data: Any, metadata: Any):
def __init__(
self,
stream_cm: Any,
agent: Any,
infer_name: bool,
input_data: Any,
metadata: Any,
):
self.stream_cm = stream_cm
self.span_name = span_name
self.agent = agent
self.infer_name = infer_name
self.input_data = input_data
self.metadata = metadata
self.span_cm = None
Expand All @@ -671,8 +728,16 @@ def __init__(self, stream_cm: Any, span_name: str, input_data: Any, metadata: An
async def __aenter__(self):
self._enter_task = asyncio.current_task()

if self.infer_name and self.agent.name is None:
_infer_name_from_caller(self.agent)
agent_name = getattr(self.agent, "name", None)
if agent_name is not None:
self.metadata["agent_name"] = agent_name
else:
self.metadata.pop("agent_name", None)

self.span_cm = start_span(
name=self.span_name,
name=f"agent_run_stream [{agent_name}]" if agent_name else "agent_run_stream",
type=SpanTypeAttribute.TASK,
input=self.input_data if self.input_data else None,
metadata=self.metadata,
Expand Down
Loading