From 0788b35b2b71492c7e428e0c6e591b10a8b88a78 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 18 Nov 2026 18:46:31 -0500 Subject: [PATCH 1/3] feat(audio): report schema version 3.1.0 --- py/src/braintrust/_audio/schema.py | 2 +- py/src/braintrust/integrations/livekit_agents/test_cascade.py | 2 +- .../integrations/livekit_agents/test_livekit_agents.py | 4 ++-- .../braintrust/integrations/livekit_agents/test_realtime.py | 2 +- py/src/braintrust/integrations/pipecat/test_pipecat.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/py/src/braintrust/_audio/schema.py b/py/src/braintrust/_audio/schema.py index c6756c63..4db5ccc7 100644 --- a/py/src/braintrust/_audio/schema.py +++ b/py/src/braintrust/_audio/schema.py @@ -9,7 +9,7 @@ from braintrust.logger import NOOP_SPAN, Span -SCHEMA_VERSION = "3.0.0" +SCHEMA_VERSION = "3.1.0" # A turn records its own ID; every operation within it records the turn's ID. TURN_ID = "turn.id" diff --git a/py/src/braintrust/integrations/livekit_agents/test_cascade.py b/py/src/braintrust/integrations/livekit_agents/test_cascade.py index bbf1cf68..64d6c750 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_cascade.py +++ b/py/src/braintrust/integrations/livekit_agents/test_cascade.py @@ -178,7 +178,7 @@ def conversation(event): assert any(row["metadata"].get("turn.reply_to") == user["span_id"] for row in named(rows, "agent_turn")) root = named(rows, "session")[0] assert root["span_parents"] == [outer.span_id] - assert root["metadata"]["audio.schema.version"] == "3.0.0" + assert root["metadata"]["audio.schema.version"] == "3.1.0" assert isinstance(root["metadata"]["audio.timeline"]["origin_unix_ms"], int) for row in rows: if row["span_id"] != root["span_id"]: diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index 07d27696..9643b585 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -64,7 +64,7 @@ async def lookup_weather(city: str) -> str: assert len(named(rows, "session")) == 1 root = named(rows, "session")[0] assert root["span_parents"] == [outer.span_id] - assert root["metadata"]["audio.schema.version"] == "3.0.0" + assert root["metadata"]["audio.schema.version"] == "3.1.0" assert isinstance(root["metadata"]["audio.timeline"]["origin_unix_ms"], int) for row in rows: if row["span_id"] != root["span_id"]: @@ -153,7 +153,7 @@ async def test_livekit_agents_agent_speaking_e2e(memory_logger): outer, root = named(rows, "outer")[0], named(rows, "session")[0] assert root["span_parents"] == [outer["span_id"]] assert named(rows, "llm_response")[0]["output"][0]["content"] == "hi there" - assert root["metadata"]["audio.schema.version"] == "3.0.0" + assert root["metadata"]["audio.schema.version"] == "3.1.0" assert "audio.segments" not in root["metadata"] user = named(rows, "user_turn")[0] assert named(rows, "agent_turn")[0]["metadata"]["turn.reply_to"] == user["span_id"] diff --git a/py/src/braintrust/integrations/livekit_agents/test_realtime.py b/py/src/braintrust/integrations/livekit_agents/test_realtime.py index d41dbaf0..078ed883 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_realtime.py +++ b/py/src/braintrust/integrations/livekit_agents/test_realtime.py @@ -459,7 +459,7 @@ async def generated(): parent = by_id[speech["span_parents"][0]] assert speech["metadata"]["turn.id"] == parent["span_id"] root = named(rows, "session")[0] - assert root["metadata"]["audio.schema.version"] == "3.0.0" + assert root["metadata"]["audio.schema.version"] == "3.1.0" assert all("audio.schema.version" not in row.get("metadata", {}) for row in rows if row is not root) assert not controller.listeners and not controller.inputs and not controller.responses assert controller.instructions is None and not controller.response_options diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 8d9fd2e5..992dc34e 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -391,7 +391,7 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play logs = memory_logger.pop() root = _single_span(logs, "session") - assert root["metadata"]["audio.schema.version"] == "3.0.0" + assert root["metadata"]["audio.schema.version"] == "3.1.0" assert root["metadata"]["contrib.pipecat.terminal_frame"] == "EndFrame" assert root["metadata"]["contrib.pipecat.reason"] == "pipeline complete" user_turn = _single_span(logs, "user_turn") From ea526c88536201470afa420fb158eaa0d6d2cf19 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 18 Nov 2026 18:58:59 -0500 Subject: [PATCH 2/3] feat(livekit): trace end-of-utterance wait lifecycles --- .../test_auto_livekit_agents.py | 110 ++++++- .../test_auto_livekit_agents_before_import.py | 5 + .../integrations/livekit_agents/eou.py | 238 ++++++++++++++ .../livekit_agents/integration.py | 4 +- .../integrations/livekit_agents/patchers.py | 49 ++- .../livekit_agents/test_cascade.py | 81 ++++- .../integrations/livekit_agents/test_eou.py | 307 ++++++++++++++++++ .../livekit_agents/test_livekit_agents.py | 6 +- .../integrations/livekit_agents/tracing.py | 71 +++- 9 files changed, 828 insertions(+), 43 deletions(-) create mode 100644 py/src/braintrust/integrations/livekit_agents/eou.py create mode 100644 py/src/braintrust/integrations/livekit_agents/test_eou.py diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py index 33a1a43b..d828ddda 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents.py @@ -1,6 +1,9 @@ """Auto setup uses the same framework-level cascade hooks after imports.""" +import asyncio import inspect +import os +import uuid from braintrust.auto import auto_instrument from livekit.agents import AgentSession @@ -9,12 +12,101 @@ from wrapt import FunctionWrapper -assert not isinstance(inspect.getattr_static(AgentSession, "start"), FunctionWrapper) -assert auto_instrument().get("livekit_agents") is True -assert isinstance(inspect.getattr_static(AgentSession, "start"), FunctionWrapper) -assert isinstance(inspect.getattr_static(AgentSession, "_aclose_impl"), FunctionWrapper) -assert isinstance(inspect.getattr_static(AgentActivity, "push_audio"), FunctionWrapper) -assert isinstance(generation._llm_inference_task, FunctionWrapper) -assert isinstance(generation._tts_inference_task, FunctionWrapper) -assert auto_instrument().get("livekit_agents") is True -print("SUCCESS") +async def assert_exported_eou_wait(): + """Drive normal RTC/VAD/STT turn completion using the existing cascade tape.""" + from braintrust.integrations.test_utils import autoinstrument_test_context + from livekit import api, rtc + from livekit.agents import Agent + from livekit.plugins import openai, silero + + with autoinstrument_test_context( + "test_livekit_agents_e2e_parenting_under_custom_span", integration="livekit_agents" + ) as memory_logger: + room_name = f"braintrust-auto-eou-{uuid.uuid4()}" + agent_room, caller_room = rtc.Room(), rtc.Room() + source = rtc.AudioSource(24000, 1) + try: + for room, identity in ((agent_room, "agent"), (caller_room, "caller")): + token = ( + api.AccessToken(os.getenv("LIVEKIT_API_KEY", "devkey"), os.getenv("LIVEKIT_API_SECRET", "secret")) + .with_identity(identity) + .with_grants(api.VideoGrants(room_join=True, room=room_name)) + .to_jwt() + ) + await asyncio.wait_for(room.connect(os.getenv("LIVEKIT_URL", "ws://localhost:7880"), token), 15) + track = rtc.LocalAudioTrack.create_audio_track("caller-microphone", source) + await caller_room.local_participant.publish_track( + track, rtc.TrackPublishOptions(source=rtc.TrackSource.SOURCE_MICROPHONE) + ) + replied = asyncio.Event() + eou_metrics = [] + async with ( + openai.LLM(model="gpt-4.1-mini", max_completion_tokens=8) as llm, + openai.TTS(model="gpt-4o-mini-tts", voice="ash") as tts, + openai.STT(model="gpt-4o-mini-transcribe") as stt, + AgentSession(vad=silero.VAD.load(), stt=stt, tts=tts, llm=llm) as session, + ): + speech = [chunk.frame async for chunk in tts.synthesize("hello there")] + + def conversation(event): + if event.item.type == "message" and event.item.role == "assistant": + replied.set() + + session.on("conversation_item_added", conversation) + session.on( + "metrics_collected", + lambda event: eou_metrics.append(event.metrics) if event.metrics.type == "eou_metrics" else None, + ) + await session.start( + Agent(instructions="Reply with exactly: hi there", llm=llm), room=agent_room, record=False + ) + silence = rtc.AudioFrame( + data=bytes(2400 * 2), sample_rate=24000, num_channels=1, samples_per_channel=2400 + ) + for frame in [silence] * 5 + speech + [silence] * 15: + await source.capture_frame(frame) + await asyncio.wait_for(replied.wait(), 15) + await session.drain() + finally: + await source.aclose() + await asyncio.gather(agent_room.disconnect(), caller_room.disconnect()) + + rows = memory_logger.pop() + waits = [row for row in rows if row.get("span_attributes", {}).get("name") == "eou_wait"] + assert len(waits) == 1, f"Expected one exported eou_wait, got {len(waits)}" + wait = waits[0] + user = next( + row for row in rows if row.get("span_attributes", {}).get("name") == "user_turn" and row.get("input") + ) + speech = next(row for row in rows if row.get("span_attributes", {}).get("name") == "user_speech") + assert wait["span_attributes"]["type"] == "task" + assert wait["span_parents"] == [user["span_id"]] + assert wait["metadata"]["turn.id"] == user["span_id"] + assert wait["metadata"]["eou.outcome"] == "committed" + assert wait["metadata"]["eou.speech_end_source"] == "vad" + (reported,) = eou_metrics + assert speech["metrics"]["start"] <= wait["metrics"]["start"] + # LiveKit sets its speech-end anchor after the speech callback returns; + # that callback can take arbitrary time. Compare with native EOU timing, + # rather than imposing a scheduling budget against user_speech.end. + assert wait["metrics"]["start"] + reported.end_of_utterance_delay <= wait["metrics"]["end"] + assert wait["metrics"]["end"] <= reported.timestamp + assert user["metadata"]["contrib.livekit.end_of_turn_delay"] == reported.end_of_utterance_delay + assert wait["metrics"]["start"] < wait["metrics"]["end"] <= user["metrics"]["end"] + assert wait["context"]["span_origin"]["instrumentation"]["name"] == "livekit-agents-auto" + assert "eou.end_reason" not in wait["metadata"] + assert "eou.detection_methods" not in wait["metadata"] + assert not any(row.get("span_attributes", {}).get("name") == "livekit.eou_wait" for row in rows) + + +if __name__ == "__main__": + assert not isinstance(inspect.getattr_static(AgentSession, "start"), FunctionWrapper) + assert auto_instrument().get("livekit_agents") is True + assert isinstance(inspect.getattr_static(AgentSession, "start"), FunctionWrapper) + assert isinstance(inspect.getattr_static(AgentSession, "_aclose_impl"), FunctionWrapper) + assert isinstance(inspect.getattr_static(AgentActivity, "push_audio"), FunctionWrapper) + assert isinstance(generation._llm_inference_task, FunctionWrapper) + assert isinstance(generation._tts_inference_task, FunctionWrapper) + assert auto_instrument().get("livekit_agents") is True + asyncio.run(assert_exported_eou_wait()) + print("SUCCESS") diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents_before_import.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents_before_import.py index e8ba6842..a2d41cda 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents_before_import.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_livekit_agents_before_import.py @@ -1,5 +1,6 @@ """Auto setup before customer imports reaches the same cascade hooks.""" +import asyncio import inspect from braintrust.auto import auto_instrument @@ -13,4 +14,8 @@ assert isinstance(inspect.getattr_static(AgentSession, "start"), FunctionWrapper) assert isinstance(generation._llm_inference_task, FunctionWrapper) +from test_auto_livekit_agents import assert_exported_eou_wait # noqa: E402 + + +asyncio.run(assert_exported_eou_wait()) print("SUCCESS") diff --git a/py/src/braintrust/integrations/livekit_agents/eou.py b/py/src/braintrust/integrations/livekit_agents/eou.py new file mode 100644 index 00000000..a4666c76 --- /dev/null +++ b/py/src/braintrust/integrations/livekit_agents/eou.py @@ -0,0 +1,238 @@ +"""Observe native EOU lifetimes without turning evaluation triggers into gates. + +Private hooks are qualified against 1.8.9 and 1.8.12. Native detector cleanup +can outlive resumed speech, so its execution is a separate framework operation. +""" + +import contextlib +import math +import time +from contextvars import ContextVar +from dataclasses import dataclass, field +from typing import Any + +from braintrust.logger import Span + +from .compat import arguments, state_for + + +SNAPSHOT_WAIT: ContextVar["Wait | None"] = ContextVar("braintrust_livekit_snapshot_wait", default=None) + + +def observed_arguments(wrapped, args, kwargs): + try: + return arguments(wrapped, args, kwargs) + except (TypeError, ValueError): + return {} + + +def timestamp(value): + return type(value) in (int, float) and math.isfinite(value) and value > 0 + + +def native_metadata(native): + return { + f"contrib.livekit.{key}": value + for key, value in (getattr(native, "attributes", None) or {}).items() + if key.startswith("lk.eou.") + } + + +@dataclass +class Wait: + recognition: Any + parent: Span + start: float + span: Span + native_ids: set[tuple[Any, int]] = field(default_factory=set) + committed_at: float | None = None + snapshot: bool = False + + +def begin(owner, recognition, parent, anchor, source): + wait = owner.eou_active.get(recognition) + if wait is None: + from .tracing import _new + + span = _new( + parent, + "eou_wait", + start_time=anchor, + metadata={**owner.source_metadata("session-input"), "eou.speech_end_source": source}, + ) + wait = Wait(recognition, parent, anchor, span) + owner.eou_active[recognition] = wait + elif anchor != wait.start: + wait.start = anchor + wait.span.log(metrics={"start": anchor}, metadata={"eou.speech_end_source": source}) + return wait + + +def finish(owner, wait, end, outcome=None, native=None): + if not timestamp(end) or end < wait.start: + # A backwards terminal stamp cannot describe a canonical interval. + # Close at observation time, without asserting an outcome at that stamp. + end, outcome = max(wait.start, time.time()), None + metadata = native_metadata(native) + if outcome in ("committed", "user_resumed", "dropped"): + metadata.update({"eou.outcome": outcome, "contrib.livekit.outcome": outcome}) + if timestamp(getattr(native, "end_time", None)): + metadata["contrib.livekit.native_end_time_ns"] = native.end_time + wait.span.log(metadata=metadata) + wait.span.end(end) + for key in wait.native_ids: + owner.eou_waits.pop(key, None) + if owner.eou_active.get(wait.recognition) is wait: + owner.eou_active.pop(wait.recognition) + + +def ensure(wrapped, instance, args, kwargs): + values = observed_arguments(wrapped, args, kwargs) + native = wrapped(*args, **kwargs) + owner = state_for(instance) + if owner is None or owner.closed: + return native + anchor = values.get("last_speaking_time") + observed = owner.speech_ends.get(instance) + # Native falls back to now, and STT sometimes substitutes receipt time. + # Neither is evidence of speech end. Only use a separately observed anchor. + if not timestamp(anchor) or anchor > time.time() or observed is None or anchor != observed[0]: + return native + parent = owner.native_users.get((instance, values.get("user_turn_span"))) or owner.span + wait = owner.eou_waits.get((instance, id(native))) or begin(owner, instance, parent, anchor, observed[1]) + if anchor != wait.start and not wait.snapshot: + wait.start = anchor + wait.span.log(metrics={"start": anchor}, metadata={"eou.speech_end_source": observed[1]}) + wait.native_ids.add((instance, id(native))) + owner.eou_waits[(instance, id(native))] = wait + wait.span.log( + metadata={ + **native_metadata(native), + "contrib.livekit.trigger": values.get("trigger"), + "contrib.livekit.endpointing_delay": values.get("endpointing_delay"), + } + ) + return native + + +def snapshot(wrapped, instance, args, kwargs): + result = wrapped(*args, **kwargs) + owner = state_for(instance) + wait = owner.eou_waits.get((instance, id(result.eou_wait_span))) if owner is not None else None + if wait is not None: + # A manual commit owns this wait independently of the current buffer. + # Retain it for the eventual snapshot terminal callback. + wait.snapshot = True + # No-op OTel tracers can return the same native span for every turn. + # Snapshot identity remains unique even when native span identity does not. + for key in wait.native_ids: + owner.eou_waits.pop(key, None) + wait.native_ids = {(instance, id(result))} + owner.eou_waits[(instance, id(result))] = wait + SNAPSHOT_WAIT.set(wait) + if wait.parent is not owner.span: + owner.pending_users.add(wait.parent) + if owner.eou_active.get(instance) is wait: + owner.eou_active.pop(instance) + return result + + +def run_detection(wrapped, instance, args, kwargs): + # Native creates the manual snapshot and its delivery task synchronously. + # Only that task inherits the owning wait, including with native OTel off. + token = SNAPSHOT_WAIT.set(None) + try: + return wrapped(*args, **kwargs) + finally: + SNAPSHOT_WAIT.reset(token) + + +def end(wrapped, instance, args, kwargs): + values = observed_arguments(wrapped, args, kwargs) + snapshot = values.get("turn_snapshot") + native = ( + getattr(snapshot, "eou_wait_span", None) if snapshot is not None else getattr(instance, "_eou_wait_span", None) + ) + requested = values.get("end_time") + ended = time.time() if requested is None else requested + result = wrapped(*args, **kwargs) + owner = state_for(instance) + key = (instance, id(snapshot) if snapshot is not None else id(native)) + wait = owner.eou_waits.get(key) if owner is not None else None + if wait is None and owner is not None and snapshot is None: + wait = owner.eou_active.get(instance) + # Only the pre-transcript wait has no native span yet. A detached + # snapshot must never be finalized by a reset of the current buffer. + if wait is not None and wait.native_ids: + wait = None + if owner is not None and wait is not None: + outcome = values.get("outcome") + # Native labels teardown as dropped; shutdown itself proves no outcome. + if owner.closing or ((closing := getattr(instance, "_closing", None)) is not None and closing.is_set()): + outcome = None + finish( + owner, + wait, + wait.committed_at if outcome == "committed" and wait.committed_at is not None else ended, + outcome, + native, + ) + return result + + +async def vad_event(wrapped, instance, args, kwargs): + event = args[0] if args else kwargs["ev"] + result = await wrapped(*args, **kwargs) + owner = state_for(instance) + if owner is not None and not owner.closed and not owner.closing: + kind = event.type.value + if kind == "end_of_speech" and timestamp(instance._last_speaking_time): + owner.speech_ends[instance] = (instance._last_speaking_time, "vad") + # In these native branches EOS enters _run_eou_detection even when + # it cannot schedule a detector yet because STT has no transcript. + # That is an observed completion wait too, not just STT execution. + if instance._vad_base_turn_detection or ( + instance._turn_detection_mode == "stt" and instance._user_turn_committed + ): + parent = owner.native_users.get((instance, instance._user_turn_span)) or owner.span + begin(owner, instance, parent, instance._last_speaking_time, "vad") + elif kind == "start_of_speech": + owner.speech_ends.pop(instance, None) + return result + + +def stt_anchor(instance, event, owner): + if event.type.value == "start_of_speech": + owner.speech_ends.pop(instance, None) + return + anchor = instance._last_speaking_time + reported = getattr(event, "speech_end_time", None) + if event.type.value == "end_of_speech" and timestamp(reported) and anchor == reported: + owner.speech_ends[instance] = (anchor, "stt") + elif event.alternatives and timestamp(instance._input_started_at): + offset = event.alternatives[0].end_time + if timestamp(offset) and anchor == instance._input_started_at + offset: + owner.speech_ends[instance] = (anchor, "stt") + + +def detection(wrapped, instance, args, kwargs): + native = args[0] if args else kwargs["span"] + owner = state_for(instance) + wait = owner.eou_waits.get((instance, id(instance._eou_wait_span))) if owner is not None else None + + @contextlib.contextmanager + def context(): + try: + with wrapped(*args, **kwargs): + yield + finally: + start, ended = getattr(native, "start_time", None), getattr(native, "end_time", None) + if wait is not None and timestamp(start) and timestamp(ended) and ended >= start: + from .tracing import _new + + operation = _new( + wait.parent, "livekit.eou_detection", start_time=start / 1e9, metadata=native_metadata(native) + ) + operation.end(ended / 1e9) + + return context() diff --git a/py/src/braintrust/integrations/livekit_agents/integration.py b/py/src/braintrust/integrations/livekit_agents/integration.py index 5e33502b..631212f3 100644 --- a/py/src/braintrust/integrations/livekit_agents/integration.py +++ b/py/src/braintrust/integrations/livekit_agents/integration.py @@ -1,6 +1,6 @@ from braintrust.integrations.base import BaseIntegration -from .patchers import AgentSessionPatcher +from .patchers import AgentSessionPatcher, EOUWaitPatcher class LiveKitAgentsIntegration(BaseIntegration): @@ -8,4 +8,4 @@ class LiveKitAgentsIntegration(BaseIntegration): import_names = ("livekit.agents",) distribution_names = ("livekit-agents",) min_version = "1.8.9" - patchers = (AgentSessionPatcher,) + patchers = (AgentSessionPatcher, EOUWaitPatcher) diff --git a/py/src/braintrust/integrations/livekit_agents/patchers.py b/py/src/braintrust/integrations/livekit_agents/patchers.py index 004a9256..73cbb26c 100644 --- a/py/src/braintrust/integrations/livekit_agents/patchers.py +++ b/py/src/braintrust/integrations/livekit_agents/patchers.py @@ -1,8 +1,9 @@ +import inspect from typing import Any from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher -from . import realtime, tracing +from . import eou, realtime, tracing def _patcher(name, module, path, wrapper): @@ -18,6 +19,52 @@ def _patcher(name, module, path, wrapper): ) +class EOUWaitPatcher(CompositeFunctionWrapperPatcher): + name = "eou_wait" + sub_patchers = ( + _patcher("eou_ensure", "voice.audio_recognition", "AudioRecognition._ensure_eou_wait_span", eou.ensure), + _patcher("eou_end", "voice.audio_recognition", "AudioRecognition._end_eou_wait_span", eou.end), + _patcher("eou_snapshot", "voice.audio_recognition", "AudioRecognition._take_turn_snapshot", eou.snapshot), + _patcher( + "eou_run_detection", "voice.audio_recognition", "AudioRecognition._run_eou_detection", eou.run_detection + ), + _patcher("eou_vad", "voice.audio_recognition", "AudioRecognition._on_vad_event", eou.vad_event), + _patcher( + "eou_detection", "voice.audio_recognition", "AudioRecognition._track_eou_detection_span", eou.detection + ), + ) + + @classmethod + def applies(cls, module, version, *, target=None): + end = cls.sub_patchers[1].resolve_target(module, version, target=target) + try: + snapshot_required = "turn_snapshot" in inspect.signature(end).parameters + except (TypeError, ValueError): + return False + contracts = ( + (False, {"user_turn_span": None, "trigger": "vad", "last_speaking_time": None, "endpointing_delay": 0}), + ( + False, + {"outcome": "committed", "end_time": None, **({"turn_snapshot": None} if snapshot_required else {})}, + ), + (False, {}), + (False, {"chat_ctx": None, "trigger": "vad", "skip_reply": False}), + (True, {"ev": None}), + (False, {"span": None}), + ) + for index, (patcher, (asynchronous, arguments)) in enumerate(zip(cls.sub_patchers, contracts)): + method = patcher.resolve_target(module, version, target=target) + if index == 2 and method is None and not snapshot_required: + continue # LiveKit 1.8.9 predates manual snapshots. + if not callable(method) or inspect.iscoroutinefunction(method) != asynchronous: + return False + try: + inspect.signature(method).bind(None, **arguments) + except (TypeError, ValueError): + return False + return super().applies(module, version, target=target) + + class AgentSessionPatcher(CompositeFunctionWrapperPatcher): name = "livekit_cascade_session" sub_patchers = ( diff --git a/py/src/braintrust/integrations/livekit_agents/test_cascade.py b/py/src/braintrust/integrations/livekit_agents/test_cascade.py index 64d6c750..56f3c8a2 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_cascade.py +++ b/py/src/braintrust/integrations/livekit_agents/test_cascade.py @@ -15,22 +15,29 @@ def named(rows, name): @pytest.mark.asyncio @pytest.mark.parametrize( - "preemptive,directions,provider_mode", + "preemptive,directions,provider_mode,manual", [ - (False, "both", "enabled"), - (True, "both", "enabled"), - (False, "user", "enabled"), - (False, "both", "disabled"), + (False, "both", "enabled", False), + (True, "both", "enabled", False), + (False, "user", "enabled", False), + (False, "both", "disabled", False), + (False, "both", "enabled", True), ], ) @pytest.mark.vcr(cassette_name="test_livekit_agents_e2e_parenting_under_custom_span") async def test_consumed_input_drives_real_cascade( - rooms, memory_logger, monkeypatch, preemptive, directions, provider_mode + rooms, memory_logger, monkeypatch, preemptive, directions, provider_mode, manual ): from livekit import rtc from livekit.agents import Agent, AgentSession from livekit.plugins import openai, silero + if manual: + from livekit.agents.voice.audio_recognition import AudioRecognition + + if not hasattr(AudioRecognition, "_take_turn_snapshot"): + pytest.skip("Pending manual snapshots require LiveKit 1.8.12") + monkeypatch.setenv("BRAINTRUST_LEGACY_IDS", "false") logger._state._reset_id_generator() monkeypatch.setenv("BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS", "true") @@ -72,7 +79,13 @@ def upload(attachment): ) with logger.start_span(name="voice_call") as outer: - async with llm, tts, stt, AgentSession(vad=vad, stt=stt, tts=tts, preemptive_generation=preemptive) as session: + options = {"turn_detection": "manual", "min_endpointing_delay": 0.8} if manual else {} + async with ( + llm, + tts, + stt, + AgentSession(vad=vad, stt=stt, tts=tts, preemptive_generation=preemptive, **options) as session, + ): frames = [chunk.frame async for chunk in tts.synthesize("hello there")] assert frames @@ -89,6 +102,30 @@ def conversation(event): Agent(instructions="Reply with exactly: hi there", llm=llm), room=rooms[0], record=False ) owner = getattr(session, STATE_ATTR) + committed = None + if manual: + paused = asyncio.Event() + transcribed = asyncio.Event() + + def user_state(event): + if event.new_state == "listening": + paused.set() + + session.on("user_state_changed", user_state) + + def transcript(event): + if event.is_final: + transcribed.set() + + session.on("user_input_transcribed", transcript) + + async def commit_and_clear(): + await paused.wait() + await transcribed.wait() + await session.commit_user_turn(transcript_timeout=0, stt_flush_duration=0) + session.clear_user_turn() + + committed = asyncio.create_task(commit_and_clear()) try: silence = rtc.AudioFrame( data=bytes(2400 * 2), sample_rate=24000, num_channels=1, samples_per_channel=2400 @@ -100,9 +137,14 @@ def conversation(event): for _ in range(15): await input_source.capture_frame(silence) await asyncio.wait_for(heard.wait(), 15) + if committed is not None: + await committed await session.drain() assert owner.input_end is not None finally: + if committed is not None and not committed.done(): + committed.cancel() + await asyncio.gather(committed, return_exceptions=True) await input_source.aclose() assert logger.current_span() is outer memory_logger.flush() @@ -152,7 +194,12 @@ def conversation(event): assert relationships[0]["attributes"] == {"braintrust.relationship.type": "generated_by"} user = next(row for row in named(rows, "user_turn") if row.get("input")) assert user["input"][0]["content"] == "Hello there." - assert user["metadata"]["audio.selections"] + if manual: + # clear_user_turn detaches the buffer before the pending snapshot commits. + # Speech retains its audio selection; the detached turn has no buffer selection. + assert all(row["metadata"]["audio.selections"] for row in named(rows, "user_speech")) + else: + assert user["metadata"]["audio.selections"] if not preemptive: assert len(eou_metrics) == 1 if eou_metrics: @@ -162,11 +209,21 @@ def conversation(event): assert ( user["metadata"]["contrib.livekit.on_user_turn_completed_delay"] == reported.on_user_turn_completed_delay ) - wait = named(rows, "livekit.eou_wait")[0] + (wait,) = named(rows, "eou_wait") assert wait["span_parents"] == [user["span_id"]] assert wait["span_attributes"]["type"] == "task" + assert wait["metadata"]["eou.outcome"] == "committed" + assert wait["metadata"]["eou.speech_end_source"] == "vad" + assert wait["metadata"]["turn.id"] == user["span_id"] + assert not named(rows, "livekit.eou_wait") + if manual: + assert wait["metrics"]["end"] - wait["metrics"]["start"] >= 0.8 + if preemptive: + assert any(model["metrics"]["start"] < wait["metrics"]["end"] for model in models) + assert "eou.end_reason" not in wait["metadata"] + assert "eou.detection_methods" not in wait["metadata"] assert wait["metrics"]["end"] - wait["metrics"]["start"] == pytest.approx( - user["metadata"]["contrib.livekit.end_of_turn_delay"], abs=1e-6 + user["metadata"]["contrib.livekit.end_of_turn_delay"], abs=0.02 ) for speech in named(rows, "user_speech"): assert speech["metadata"]["contrib.livekit.vad_silence_duration"] >= 0 @@ -189,6 +246,10 @@ def conversation(event): for segment in root["metadata"]["audio.segments"].values() ) assert owner.native is None + assert not owner.eou_waits and not owner.eou_active + assert not owner.native_users and not owner.speech_ends + assert not owner.pending_users and not owner.requests and not owner.tool_results + assert owner.user is owner.user_speech is owner.stt is None @pytest.mark.asyncio diff --git a/py/src/braintrust/integrations/livekit_agents/test_eou.py b/py/src/braintrust/integrations/livekit_agents/test_eou.py new file mode 100644 index 00000000..11da59ad --- /dev/null +++ b/py/src/braintrust/integrations/livekit_agents/test_eou.py @@ -0,0 +1,307 @@ +"""Synthetic lifecycle events on real LiveKit sessions, supplemental to VCR cascade. + +No provider response is fabricated. Explicit native hooks let us exercise +rearms, corrections and teardown independently of audio/scheduler timing. +""" + +import asyncio +import inspect +import time + +import pytest +from braintrust.integrations.livekit_agents import setup_livekit_agents +from braintrust.integrations.livekit_agents.compat import STATE_ATTR + + +@pytest.mark.parametrize("hook", range(6)) +@pytest.mark.parametrize("change", ["missing", "signature", "async"]) +def test_eou_optional_hooks_are_gated_together(monkeypatch, hook, change): + """Pure patch routing: incompatible native layouts must disable the feature.""" + import importlib + + from livekit.agents.voice.audio_recognition import AudioRecognition + + from .patchers import EOUWaitPatcher + + patcher = EOUWaitPatcher.sub_patchers[hook] + module = importlib.import_module(patcher.target_module) + parent, name = patcher.target_path.rsplit(".", 1) + target = getattr(module, parent) + method = getattr(target, name, None) + if method is None: + pytest.skip("Manual snapshots are unavailable in this framework version") + assert EOUWaitPatcher.applies(None, None) + replacement = None if change == "missing" else dict.get + if change == "async" and not inspect.iscoroutinefunction(method): + replacement = AudioRecognition._on_vad_event + monkeypatch.setattr(target, name, replacement) + assert not EOUWaitPatcher.applies(None, None) + + +async def vad(recognition, kind, *, speech=0.0, silence=0.0): + from livekit.agents.vad import VADEvent, VADEventType + + await recognition._on_vad_event( + VADEvent( + type=VADEventType(kind), + samples_index=0, + timestamp=time.time(), + speech_duration=speech, + silence_duration=silence, + ) + ) + + +def waits(rows): + return sorted( + (r for r in rows if r.get("span_attributes", {}).get("name") == "eou_wait"), + key=lambda r: r["metrics"]["start"], + ) + + +def ensure(recognition, anchor=None): + return recognition._ensure_eou_wait_span( + recognition._ensure_user_turn_span(), + trigger="vad", + last_speaking_time=recognition._last_speaking_time if anchor is None else anchor, + endpointing_delay=0.5, + ) + + +@pytest.fixture(params=[False, True]) +def native_tracing(request): + from livekit.agents.telemetry import traces + from opentelemetry.sdk.trace import TracerProvider + from opentelemetry.trace import NoOpTracerProvider + + original = traces.tracer._tracer_provider + provider = TracerProvider() if request.param else NoOpTracerProvider() + traces.tracer.set_provider(provider) + try: + yield request.param + finally: + traces.tracer.set_provider(original) + if request.param: + provider.shutdown() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("terminal", ["committed", "dropped", "shutdown"]) +async def test_eou_pause_resume_rearm_and_terminal(memory_logger, terminal): + from livekit.agents import Agent, AgentSession + from livekit.agents.telemetry import traces + from opentelemetry.sdk.trace import TracerProvider + + setup_livekit_agents() + # Real OTel spans are needed to inspect LiveKit's cleanup floor and attrs. + original = traces.tracer._tracer_provider + provider = TracerProvider() + traces.tracer.set_provider(provider) + try: + async with AgentSession(turn_detection="manual") as session: + await session.start(Agent(instructions="lifecycle test")) + owner = getattr(session, STATE_ATTR) + recognition = session._activity._audio_recognition + await asyncio.sleep(0.02) + await vad(recognition, "start_of_speech", speech=0.01) + await vad(recognition, "end_of_speech", silence=0.005) + first_start = recognition._last_speaking_time + first = ensure(recognition) + ensure(recognition) + # LiveKit wraps OTel's contextmanager decorator. + with traces.tracer.start_as_current_span("eou_detection", end_on_exit=False) as detection: # pylint: disable=not-context-manager + with recognition._track_eou_detection_span(detection): + # Detector cleanup runs after the actual reported resume. + await asyncio.sleep(0.01) + await vad(recognition, "start_of_speech", speech=0.005) + first_native_end = first.end_time / 1e9 + resume = recognition._speech_start_time + assert first_native_end > resume + await asyncio.sleep(0.01) + await vad(recognition, "end_of_speech", silence=0.005) + last_start = recognition._last_speaking_time + last = ensure(recognition) + ensure(recognition) + if terminal != "shutdown": + if terminal == "dropped": + recognition._clear_user_turn() + else: + recognition._end_eou_wait_span(terminal) + recognition._end_eou_wait_span(terminal) # idempotent + # Shutdown must not turn LiveKit's cleanup label into abandonment. + assert not owner.eou_waits + finally: + traces.tracer.set_provider(original) + provider.shutdown() + rows = memory_logger.pop() + emitted = waits(rows) + assert len(emitted) == 2 + first_row, last_row = emitted + assert first_row["metrics"]["start"] == first_start + assert first_row["metrics"]["end"] == resume + assert first_row["metadata"]["eou.outcome"] == "user_resumed" + assert first_row["metadata"]["contrib.livekit.lk.eou.outcome"] == "user_resumed" + assert first_row["metadata"]["contrib.livekit.native_end_time_ns"] == first.end_time + assert last_row["metrics"]["start"] == last_start + if terminal == "shutdown": + assert "eou.outcome" not in last_row["metadata"] + assert last_row["metadata"]["contrib.livekit.lk.eou.outcome"] == "dropped" + else: + assert last_row["metadata"]["eou.outcome"] == terminal + assert last_row["metadata"]["contrib.livekit.outcome"] == terminal + for row in emitted: + assert row["metadata"]["eou.speech_end_source"] == "vad" + assert "eou.end_reason" not in row["metadata"] + assert "eou.detection_methods" not in row["metadata"] + assert row["span_parents"] == [row["metadata"]["turn.id"]] + assert row["context"]["span_origin"]["instrumentation"]["name"] == "livekit-agents-auto" + framework = next(r for r in rows if r.get("span_attributes", {}).get("name") == "livekit.eou_detection") + assert framework["metrics"]["end"] > first_row["metrics"]["end"] + assert all("end" in row["metrics"] for row in rows) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("anchor", [None, 0, -1, True, float("inf")]) +async def test_eou_requires_observed_speech_anchor(memory_logger, anchor): + from livekit.agents import Agent, AgentSession + + setup_livekit_agents() + async with AgentSession(turn_detection="manual") as session: + await session.start(Agent(instructions="lifecycle test")) + recognition = session._activity._audio_recognition + # No speech-end observation, even when native uses a plausible 'now'. + ensure(recognition, anchor) + assert not waits(memory_logger.pop()) + + +@pytest.mark.asyncio +async def test_eou_unknown_turn_membership_and_corrected_anchor(memory_logger): + from livekit.agents import Agent, AgentSession + from livekit.agents.telemetry import traces + from opentelemetry.sdk.trace import TracerProvider + + original = traces.tracer._tracer_provider + provider = TracerProvider() + traces.tracer.set_provider(provider) + setup_livekit_agents() + async with AgentSession(turn_detection="manual") as session: + await session.start(Agent(instructions="lifecycle test")) + owner = getattr(session, STATE_ATTR) + recognition = session._activity._audio_recognition + await vad(recognition, "end_of_speech", silence=0.2) + # A native span with no observed user-turn association stays at session. + with traces.tracer.start_as_current_span("unassociated turn") as native_user: # pylint: disable=not-context-manager + first = recognition._last_speaking_time + recognition._ensure_eou_wait_span( + native_user, trigger="vad", last_speaking_time=first, endpointing_delay=0.5 + ) + await vad(recognition, "end_of_speech", silence=0.1) + corrected = recognition._last_speaking_time + recognition._ensure_eou_wait_span( + native_user, trigger="stt", last_speaking_time=corrected, endpointing_delay=0.5 + ) + recognition._end_eou_wait_span("committed") + traces.tracer.set_provider(original) + provider.shutdown() + (row,) = waits(memory_logger.pop()) + assert row["span_parents"] == [owner.span.span_id] + assert "turn.id" not in row["metadata"] + assert row["metrics"]["start"] == corrected + # An STT evaluation trigger does not change the timestamp's VAD origin. + assert row["metadata"]["eou.speech_end_source"] == "vad" + + +@pytest.mark.asyncio +async def test_eou_waits_for_transcript_before_native_detector_exists(memory_logger): + from livekit.agents import Agent, AgentSession + from livekit.plugins import openai, silero + + setup_livekit_agents() + # Real segmented STT and VAD; no PCM is submitted, so no provider request + # is made. Synthetic native VAD events expose the pending-transcript path. + async with openai.STT() as stt, AgentSession(vad=silero.VAD.load(), stt=stt, turn_detection="vad") as session: + await session.start(Agent(instructions="lifecycle test")) + owner = getattr(session, STATE_ATTR) + recognition = session._activity._audio_recognition + await asyncio.sleep(0.02) + await vad(recognition, "start_of_speech", speech=0.01) + await vad(recognition, "end_of_speech", silence=0.005) + assert recognition._eou_wait_span is None + assert recognition in owner.eou_active + await asyncio.sleep(0.01) + await vad(recognition, "start_of_speech", speech=0.005) + await asyncio.sleep(0.01) + await vad(recognition, "end_of_speech", silence=0.005) + assert not owner.eou_active and not owner.eou_waits + first, second = waits(memory_logger.pop()) + assert first["metadata"]["eou.outcome"] == "user_resumed" + assert "eou.outcome" not in second["metadata"] + assert first["metrics"]["end"] < second["metrics"]["start"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("second_turn", [False, True]) +async def test_eou_pending_manual_snapshot_survives_buffer_clear(memory_logger, second_turn, native_tracing): + from livekit.agents import Agent, AgentSession, stt + from livekit.agents.voice.audio_recognition import AudioRecognition + from livekit.plugins import openai, silero + + if not hasattr(AudioRecognition, "_take_turn_snapshot"): + pytest.skip("Native manual snapshots require LiveKit 1.8.12") + setup_livekit_agents() + async with ( + openai.STT() as service, + AgentSession( + stt=service, vad=silero.VAD.load(), turn_detection="manual", min_endpointing_delay=0.4 + ) as session, + ): + await session.start(Agent(instructions="manual snapshot regression")) + recognition = session._activity._audio_recognition + owner = getattr(session, STATE_ATTR) + await vad(recognition, "start_of_speech") + recognition._process_stt_event( + stt.SpeechEvent( + type=stt.SpeechEventType.FINAL_TRANSCRIPT, + alternatives=[stt.SpeechData(language="en", text="Keep this turn")], + ) + ) + await vad(recognition, "end_of_speech") + anchor = recognition._last_speaking_time + first_turn = owner.user + await session.commit_user_turn(transcript_timeout=0, stt_flush_duration=0, skip_reply=True) + pending = recognition._end_of_turn_task + assert pending is not None and not pending.done() + session.clear_user_turn() + if second_turn: + await vad(recognition, "start_of_speech") + recognition._process_stt_event( + stt.SpeechEvent( + type=stt.SpeechEventType.FINAL_TRANSCRIPT, + alternatives=[stt.SpeechData(language="en", text="A different turn")], + ) + ) + await vad(recognition, "end_of_speech") + next_turn = owner.user + await asyncio.wait_for(asyncio.shield(pending), 2) + await asyncio.sleep(0) # native snapshot done callback + if second_turn: + assert owner.user is next_turn + await session.commit_user_turn(transcript_timeout=0, stt_flush_duration=0, skip_reply=True) + await asyncio.wait_for(asyncio.shield(recognition._end_of_turn_task), 2) + await asyncio.sleep(0) + assert not owner.eou_waits and not owner.eou_active + rows = memory_logger.pop() + emitted = waits(rows) + assert len(emitted) == (2 if second_turn else 1) + row = emitted[0] + assert row["metrics"]["start"] == anchor + assert row["metadata"]["eou.outcome"] == "committed" + if native_tracing: + assert row["metadata"]["contrib.livekit.lk.eou.outcome"] == "committed" + assert row["metrics"]["end"] - anchor >= 0.35 + assert row["span_parents"] == [first_turn.span_id] + assert row["metadata"]["turn.id"] == first_turn.span_id + if second_turn: + assert emitted[1]["span_parents"] == [next_turn.span_id] + assert emitted[1]["metadata"]["eou.outcome"] == "committed" + assert all("end" in span["metrics"] for span in rows) diff --git a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py index 9643b585..ef61d6ed 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py +++ b/py/src/braintrust/integrations/livekit_agents/test_livekit_agents.py @@ -211,6 +211,6 @@ def test_explicit_setup_survives_automatic_setup(): tracing._OPTIONS = original -def test_auto_instrument_livekit_agents_subprocess(): - verify_autoinstrument_script("test_auto_livekit_agents.py") - verify_autoinstrument_script("test_auto_livekit_agents_before_import.py") +def test_auto_instrument_livekit_agents_subprocess(livekit_server): + verify_autoinstrument_script("test_auto_livekit_agents.py", timeout=60) + verify_autoinstrument_script("test_auto_livekit_agents_before_import.py", timeout=60) diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index 4d72ee7a..d8f0fc83 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -137,6 +137,10 @@ def __init__(self, native, record): self.user = None self.user_started = None self.pending_users = set() + self.eou_waits = {} + self.eou_active = {} + self.speech_ends = {} + self.native_users = {} self.user_speech = None self.user_selection = None self.user_start = None @@ -316,6 +320,12 @@ async def _finish(self): return self.closed = True end = time.time() + from .eou import finish + + for wait in list({id(wait): wait for wait in (*self.eou_waits.values(), *self.eou_active.values())}.values()): + finish(self, wait, end) + self.speech_ends.clear() + self.native_users.clear() for controller in self.realtime.values(): controller.close() self.realtime.clear() @@ -500,7 +510,10 @@ def continue_turn(wrapped, instance, args, kwargs): def ensure_user(wrapped, instance, args, kwargs): if (owner := state_for(instance)) is not None: owner.ensure_user(arguments(wrapped, args, kwargs).get("start_time")) - return wrapped(*args, **kwargs) + native = wrapped(*args, **kwargs) + if owner is not None: + owner.native_users[(instance, native)] = owner.user + return native def user_speech_start(wrapped, instance, args, kwargs): @@ -581,18 +594,36 @@ def process_stt(wrapped, instance, args, kwargs): owner.log(span, output=[{"role": "user", "content": result.text}]) if not getattr(result, "_braintrust_streaming_recognition", False): span.end(interval[1] if interval else received_at) - return wrapped(*args, **kwargs) + result = wrapped(*args, **kwargs) + if owner is not None: + from .eou import stt_anchor + + stt_anchor(instance, event, owner) + return result def end_of_turn(wrapped, instance, args, kwargs): owner = state_for(instance) info = args[0] if args else kwargs["info"] if owner is not None: - user = owner.ensure_user() + from .eou import SNAPSHOT_WAIT + + snapshot_wait = SNAPSHOT_WAIT.get() + native_user = getattr(info, "user_turn_span", None) + user = ( + snapshot_wait.parent + if snapshot_wait is not None + and state_for(snapshot_wait.recognition) is owner + and snapshot_wait.parent is not owner.span + else None + ) + if user is None: + user = next((span for (_, native), span in owner.native_users.items() if native is native_user), None) + user = user or owner.ensure_user() setattr(info, "_braintrust_turn", user) setattr(info, "_braintrust_owner", owner) owner.log(user, input=[{"role": "user", "content": info.new_transcript}]) - if owner.user_start is not None and owner.user_end is not None: + if user is owner.user and owner.user_start is not None and owner.user_end is not None: user.log( metadata={ "audio.selections": { @@ -608,27 +639,23 @@ def end_of_turn(wrapped, instance, args, kwargs): try: committed = wrapped(*args, **kwargs) if committed: + committed_at = time.time() + if snapshot_wait is not None and state_for(snapshot_wait.recognition) is owner: + snapshot_wait.committed_at = committed_at + for wait in owner.eou_waits.values(): + if wait.parent is user: + wait.committed_at = committed_at timing = info.metrics _log_native_diagnostics( user, end_of_turn_delay=timing.end_of_turn_delay, transcription_delay=timing.transcription_delay ) - stopped, delay = timing.stopped_speaking_at, timing.end_of_turn_delay - if ( - type(stopped) in (int, float) - and type(delay) in (int, float) - and math.isfinite(stopped) - and math.isfinite(delay) - and stopped >= 0 - and delay >= 0 - ): - wait = _new( - user, "livekit.eou_wait", start_time=stopped, metadata={"contrib.livekit.outcome": "committed"} - ) - wait.end(stopped + delay) - owner.user = None + owner.native_users = {key: span for key, span in owner.native_users.items() if span is not user} + if owner.user is user: + owner.user = None if info.user_turn_span_adopted: owner.pending_users.add(user) else: + owner.pending_users.discard(user) user.end() return committed finally: @@ -672,9 +699,17 @@ def recognition_end(wrapped, instance, args, kwargs): return wrapped(*args, **kwargs) finally: if owner is not None: + owner.speech_ends.pop(instance, None) + snapshots = {wait.parent for wait in owner.eou_waits.values() if wait.snapshot} + owner.native_users = { + key: span for key, span in owner.native_users.items() if key[0] is not instance or span in snapshots + } for field in ("stt", "user_speech", "user"): span = getattr(owner, field) if span is not None: + if field == "user" and span in snapshots: + owner.user = None + continue span.log( metadata={"turn.incomplete": True} if field == "user" else {"contrib.livekit.incomplete": True} ) From e64e049a879b6b6641f5c3c8b360b1885ee875b7 Mon Sep 17 00:00:00 2001 From: David Elner Date: Wed, 18 Nov 2026 18:58:59 -0500 Subject: [PATCH 3/3] feat(pipecat): trace end-of-utterance wait lifecycles --- .../auto_test_scripts/test_auto_pipecat.py | 71 ++- .../integrations/pipecat/integration.py | 2 + .../integrations/pipecat/patchers.py | 126 +++++ .../integrations/pipecat/test_pipecat.py | 433 +++++++++++++++++- .../integrations/pipecat/tracing/_common.py | 4 +- .../integrations/pipecat/tracing/_context.py | 10 + .../integrations/pipecat/tracing/_eou.py | 155 +++++++ .../integrations/pipecat/tracing/_user.py | 28 +- .../integrations/pipecat/tracing/observer.py | 49 +- 9 files changed, 849 insertions(+), 29 deletions(-) create mode 100644 py/src/braintrust/integrations/pipecat/tracing/_eou.py diff --git a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py index 2d8f3501..cc064b83 100644 --- a/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py +++ b/py/src/braintrust/integrations/auto_test_scripts/test_auto_pipecat.py @@ -53,8 +53,13 @@ def _runner_kwargs(**overrides): async def main(): with autoinstrument_test_context("test_auto_pipecat", integration="pipecat") as memory_logger: _ensure_nltk_punkt_tab() - results = auto_instrument() - assert results.get("pipecat") is True + if os.environ.get("BRAINTRUST_TEST_PIPECAT_ENTRYPOINT") == "wrap": + from braintrust.integrations.pipecat import wrap_pipeline_worker + + wrap_pipeline_worker(_import("pipecat.pipeline.worker.PipelineWorker")) + else: + results = auto_instrument() + assert results.get("pipecat") is True EndFrame = _import("pipecat.frames.frames.EndFrame") LLMContextFrame = _import("pipecat.frames.frames.LLMContextFrame") @@ -100,6 +105,68 @@ async def on_pipeline_started(_worker, _frame): names = {log.get("span_attributes", {}).get("name") for log in logs} assert "session" in names assert "llm_response" in names + await check_local_turn_wait(memory_logger) + + +async def check_local_turn_wait(memory_logger): + """Real pipeline and timers with synthetic speech-boundary frames, no IO.""" + frames = importlib.import_module("pipecat.frames.frames") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + Worker = _import("pipecat.pipeline.worker.PipelineWorker") + Runner = _import("pipecat.workers.runner.WorkerRunner") + Identity = _import("pipecat.processors.filters.identity_filter.IdentityFilter") + Context = _import("pipecat.processors.aggregators.llm_context.LLMContext") + Pair = _import("pipecat.processors.aggregators.llm_response_universal.LLMContextAggregatorPair") + Params = _import("pipecat.processors.aggregators.llm_response_universal.LLMUserAggregatorParams") + Strategies = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") + VADStart = _import("pipecat.turns.user_start.VADUserTurnStartStrategy") + Timeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + pair = Pair( + Context(), + user_params=Params( + user_turn_strategies=Strategies( + start=[VADStart(enable_user_speaking_frames=False)], + stop=[Timeout(user_speech_timeout=0.03, enable_user_speaking_frames=False)], + ) + ), + ) + worker = Worker(Pipeline([Identity(), pair.user()]), **_worker_kwargs()) + ready, committed = asyncio.Event(), asyncio.Event() + + @worker.event_handler("on_pipeline_started") + async def started(*_args): + ready.set() + + @pair.user().event_handler("on_user_turn_stopped") + async def stopped(*_args): + committed.set() + + runner = Runner(**_runner_kwargs()) + await runner.add_workers(worker) + running = asyncio.create_task(runner.run()) + try: + await asyncio.wait_for(ready.wait(), 5) + await worker.queue_frame(frames.VADUserStartedSpeakingFrame()) + await asyncio.sleep(0.03) + await worker.queue_frames( + [ + frames.VADUserStoppedSpeakingFrame(stop_secs=0.01), + frames.TranscriptionFrame("Hello", "caller", "0", finalized=True), + ] + ) + await asyncio.wait_for(committed.wait(), 5) + await worker.queue_frame(frames.EndFrame()) + await asyncio.wait_for(running, 10) + finally: + if not running.done(): + await worker.cancel() + await running + logs = memory_logger.pop() + waits = [row for row in logs if row.get("span_attributes", {}).get("name") == "eou_wait"] + assert len(waits) == 1 + assert waits[0]["metadata"]["eou.outcome"] == "committed" + assert waits[0]["span_parents"] == [waits[0]["metadata"]["turn.id"]] + assert all("end" in row["metrics"] for row in logs) if __name__ == "__main__": diff --git a/py/src/braintrust/integrations/pipecat/integration.py b/py/src/braintrust/integrations/pipecat/integration.py index dacd0d75..f8e7d570 100644 --- a/py/src/braintrust/integrations/pipecat/integration.py +++ b/py/src/braintrust/integrations/pipecat/integration.py @@ -3,6 +3,7 @@ from braintrust.integrations.base import BaseIntegration from .patchers import ( + EOUWaitPatcher, LLMServicePushFramePatcher, OpenAILivePatcher, PipelineWorkerPatcher, @@ -19,6 +20,7 @@ class PipecatIntegration(BaseIntegration): distribution_names = ("pipecat-ai",) min_version = "1.3.0" patchers = ( + EOUWaitPatcher, PipelineWorkerPatcher, LLMServicePushFramePatcher, STTServiceProcessGeneratorPatcher, diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index ffc2da7f..d193c9ef 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -115,6 +115,129 @@ async def traced_stt_process_generator( return await wrapped(*args, **kwargs) +def _controller_wait(instance): + observer = _ACTIVE_OBSERVER.get() + return observer._controller(instance) if observer is not None else None + + +def traced_observer_proxy(wrapped, instance, args, kwargs): + proxy = wrapped(*args, **kwargs) + if isinstance(proxy.observer, BraintrustPipecatObserver): + proxy.observer._controller_queue = proxy.queue + return proxy + + +class WorkerObserverProxyPatcher(FunctionWrapperPatcher): + name = "worker_observer_proxy" + target_module = "pipecat.pipeline.worker_observer" + target_path = "WorkerObserver._create_proxy" + wrapper = staticmethod(traced_observer_proxy) + + +async def traced_controller_frame(wrapped, instance, args, kwargs): + from .tracing._eou import CONDITION, stop_strategies + + wait = None + frame = args[0] if args else kwargs["frame"] + condition = None + try: + wait = _controller_wait(instance) + if wait is not None: + wait.observe(frame) + strategies = stop_strategies(instance) + if type(frame).__name__ == "TranscriptionFrame" and frame.finalized and len(strategies) == 1: + strategy = strategies[0] + if ( + type(strategy).__module__ == "pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy" + and type(strategy).__name__ == "SpeechTimeoutUserTurnStopStrategy" + and strategy._user_speech_wait_done + and not strategy._stt_wait_done + ): + condition = (strategy, "transcription_finalized") + except Exception: + from .tracing._common import LOG + + LOG.warning("Unable to observe Pipecat turn wait", exc_info=True) + token = CONDITION.set(condition) + try: + return await wrapped(*args, **kwargs) + finally: + CONDITION.reset(token) + + +async def traced_controller_event(wrapped, instance, args, kwargs): + name = args[0] if args else kwargs.get("event_name") + try: + if name in ("on_user_turn_started", "on_user_turn_stopped"): + wait = _controller_wait(instance) + if wait is not None: + wait.event(name, args[1:]) + except Exception: + from .tracing._common import LOG + + LOG.warning("Unable to observe Pipecat turn decision", exc_info=True) + return await wrapped(*args, **kwargs) + + +async def traced_speech_timeout(wrapped, instance, args, kwargs): + from .tracing._eou import CONDITION + + token = CONDITION.set((instance, "silence_timeout")) + try: + return await wrapped(*args, **kwargs) + finally: + CONDITION.reset(token) + + +class UserTurnControllerFramePatcher(FunctionWrapperPatcher): + name = "user_turn_controller_frame" + target_module = "pipecat.turns.user_turn_controller" + target_path = "UserTurnController.process_frame" + wrapper = staticmethod(traced_controller_frame) + + +class UserTurnControllerEventPatcher(FunctionWrapperPatcher): + name = "user_turn_controller_event" + target_module = "pipecat.turns.user_turn_controller" + target_path = "UserTurnController._call_event_handler" + wrapper = staticmethod(traced_controller_event) + + +class SpeechTimeoutPatcher(FunctionWrapperPatcher): + name = "speech_timeout_eou_condition" + target_module = "pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy" + target_path = "SpeechTimeoutUserTurnStopStrategy._user_speech_timeout_handler" + wrapper = staticmethod(traced_speech_timeout) + + +class EOUWaitPatcher(CompositeFunctionWrapperPatcher): + """Install queue routing, lifecycle and causal observations together.""" + + name = "eou_wait" + sub_patchers = ( + WorkerObserverProxyPatcher, + UserTurnControllerFramePatcher, + UserTurnControllerEventPatcher, + SpeechTimeoutPatcher, + ) + + @classmethod + def applies(cls, module, version, *, target=None): + for patcher, asynchronous, arguments in zip( + cls.sub_patchers, + (False, True, True, True), + ({"observer": None}, {"frame": None}, {"event_name": "on_user_turn_stopped"}, {"timeout": 0}), + ): + method = patcher.resolve_target(module, version, target=target) + if not callable(method) or inspect.iscoroutinefunction(method) != asynchronous: + return False + try: + inspect.signature(method).bind(None, **arguments) + except (TypeError, ValueError): + return False + return super().applies(module, version, target=target) + + class PipelineWorkerInitPatcher(FunctionWrapperPatcher): """Patch ``PipelineWorker.__init__`` to add a Braintrust observer.""" @@ -254,6 +377,9 @@ def wrap_pipeline_worker( from pipecat.services.llm_service import LLMService # pylint: disable=import-error from pipecat.services.stt_service import SegmentedSTTService, STTService # pylint: disable=import-error + # The same gate and composite own global markers for every entrypoint. + if EOUWaitPatcher.applies(None, None): + EOUWaitPatcher.patch(None, None) LLMServicePushFramePatcher.wrap_target(LLMService) STTServiceProcessGeneratorPatcher.wrap_target(STTService) SegmentedSTTPushSegmentFramesPatcher.wrap_target(SegmentedSTTService) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 992dc34e..b8033452 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -17,19 +17,27 @@ import os import socket import tempfile +import time import wave from contextlib import asynccontextmanager from dataclasses import dataclass, field from pathlib import Path +from typing import TYPE_CHECKING from urllib.parse import urlsplit, urlunsplit import pytest +import pytest_asyncio from braintrust import SpanCustomizer, logger, set_span_customizers from braintrust.integrations.pipecat import setup_pipecat, wrap_pipeline_worker from braintrust.integrations.test_utils import verify_autoinstrument_script from braintrust.test_helpers import init_test_logger +if TYPE_CHECKING: + from .tracing._eou import ControllerWait + from .tracing.observer import BraintrustPipecatObserver, _Session + + TEST_MODEL = "gpt-6-luna" PROJECT = "test-project-pipecat-py-tracing" SPEECH = Path(__file__).parents[2] / "fixtures/test_speech.wav" @@ -145,7 +153,7 @@ def _make_worker(pipeline, **overrides): @asynccontextmanager -async def _running(worker): +async def _running(worker, *, cancel=False): """Run a worker; on exit, end it with an ``EndFrame`` and wait for shutdown.""" WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") kwargs = {"handle_sigint": False} @@ -156,7 +164,10 @@ async def _running(worker): running = asyncio.create_task(runner.run()) try: yield - await worker.queue_frame(_frames().EndFrame(reason="pipeline complete")) + if cancel: + await worker.cancel() + else: + await worker.queue_frame(_frames().EndFrame(reason="pipeline complete")) await asyncio.wait_for(running, 20) finally: if not running.done(): @@ -225,6 +236,14 @@ class _VoiceSession: system: str max_tokens: int = 20 + speaking_frames: bool = True + inference_gate: bool = False + smart_turn: bool = False + delay_observer: bool = False + cancel: bool = False + observer: "BraintrustPipecatObserver | None" = field(init=False, default=None) + tracker: "_Session | None" = field(init=False, default=None) + controllers: "tuple[ControllerWait, ...]" = field(init=False, default=()) assistant_turns: list = field(default_factory=list) played: bytearray = field(default_factory=bytearray) @@ -238,6 +257,7 @@ async def run(self, *, utterances=1, barge_in=False): LLMUserAggregatorParams = _import(f"{universal}.LLMUserAggregatorParams") UserTurnStrategies = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") SpeechTimeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + VADStart = _import("pipecat.turns.user_start.VADUserTurnStartStrategy") SileroVADAnalyzer = _import("pipecat.audio.vad.silero.SileroVADAnalyzer") VADParams = _import("pipecat.audio.vad.vad_analyzer.VADParams") OpenAISTTService = _import("pipecat.services.openai.stt.OpenAISTTService") @@ -264,11 +284,22 @@ async def run(self, *, utterances=1, barge_in=False): stt = OpenAISTTService( api_key=os.environ["OPENAI_API_KEY"], settings=OpenAISTTService.Settings(model="whisper-1") ) + stop = [SpeechTimeout(user_speech_timeout=0.5, enable_user_speaking_frames=self.speaking_frames)] + if self.smart_turn: + Analyzer = _import("pipecat.audio.turn.smart_turn.local_smart_turn_v3.LocalSmartTurnAnalyzerV3") + AnalyzerStrategy = _import("pipecat.turns.user_stop.TurnAnalyzerUserTurnStopStrategy") + stop = [AnalyzerStrategy(turn_analyzer=Analyzer())] + if self.inference_gate: + Gate = _import("pipecat.turns.user_stop.LLMTurnCompletionUserTurnStopStrategy") + stop.append(Gate()) aggregators = LLMContextAggregatorPair( LLMContext(messages=[{"role": "system", "content": self.system}]), user_params=LLMUserAggregatorParams( vad_analyzer=SileroVADAnalyzer(params=VADParams(stop_secs=0.5)), - user_turn_strategies=UserTurnStrategies(stop=[SpeechTimeout(user_speech_timeout=0.5)]), + user_turn_strategies=UserTurnStrategies( + start=None if self.speaking_frames else [VADStart(enable_user_speaking_frames=False)], + stop=stop, + ), ), ) worker = _make_worker( @@ -286,6 +317,28 @@ async def run(self, *, utterances=1, barge_in=False): params=PipelineParams(enable_metrics=True, enable_usage_metrics=True), ) ready, turn_stopped = asyncio.Event(), asyncio.Event() + self.observer = getattr(worker, "_braintrust_pipecat_observer") + release_observer = asyncio.Event() + if self.delay_observer: + on_push = self.observer.on_push_frame + started_turns = 0 + + @aggregators.user().event_handler("on_user_turn_started") + async def on_user_started(*_args): + nonlocal started_turns + started_turns += 1 + # Hold first-turn observations through commitment and the next + # native turn's start, reproducing the ownership race through + # the real audio pipeline without changing turn-taking. + if started_turns == 2: + release_observer.set() + + async def delayed_push(data): + if isinstance(data.frame, frames.VADUserStartedSpeakingFrame): + await release_observer.wait() + await on_push(data) + + self.observer.on_push_frame = delayed_push @transport.event_handler("on_websocket_ready") async def on_ready(_transport): @@ -300,8 +353,9 @@ async def on_stopped(_aggregator, message, *_args): assert audio.getframerate() == INPUT_RATE pcm = audio.readframes(audio.getnframes()) - async with _running(worker): + async with _running(worker, cancel=self.cancel): await asyncio.wait_for(ready.wait(), 10) + self.tracker = self.observer._session async with connect(f"ws://127.0.0.1:{port}") as websocket: agent_audio = asyncio.Event() @@ -321,6 +375,10 @@ async def receive(): await self._wait_for_turns(turn_stopped, index) await self._speak(websocket, serializer, pcm) await self._wait_for_turns(turn_stopped, utterances) + release_observer.set() + if self.observer._controller_queue is not None: + await asyncio.wait_for(self.observer._controller_queue.join(), 5) + self.controllers = tuple(self.observer._controllers.values()) reading.cancel() async def _wait_for_turns(self, turn_stopped, count): @@ -345,6 +403,24 @@ async def _speak(websocket, serializer, pcm): # Setup +@pytest.mark.parametrize("hook", range(4)) +@pytest.mark.parametrize("change", ["missing", "signature", "async"]) +def test_eou_optional_hooks_are_gated_together(monkeypatch, hook, change): + """Pure patch routing, independent of provider response payloads.""" + from .patchers import EOUWaitPatcher + + patcher = EOUWaitPatcher.sub_patchers[hook] + parent, name = patcher.target_path.rsplit(".", 1) + target = _import(f"{patcher.target_module}.{parent}") + method = getattr(target, name) + assert EOUWaitPatcher.applies(None, None) + replacement = None if change == "missing" else dict.get + if change == "async" and not inspect.iscoroutinefunction(method): + replacement = _import("pipecat.turns.user_turn_controller.UserTurnController").process_frame + monkeypatch.setattr(target, name, replacement) + assert not EOUWaitPatcher.applies(None, None) + + def test_setup_and_wrap_pipeline_worker_are_idempotent(memory_logger): PipelineWorker = _import("pipecat.pipeline.worker.PipelineWorker") Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") @@ -367,38 +443,84 @@ async def run(): @pytest.mark.vcr @pytest.mark.skipif(__import__("sys").version_info < (3, 11), reason="Pipecat AI 1.x requires Python 3.11+") -def test_auto_instrument_pipecat_subprocess(): +@pytest.mark.parametrize("entrypoint", ["auto", "wrap"]) +def test_auto_instrument_pipecat_subprocess(monkeypatch, entrypoint): pytest.importorskip("pipecat") + monkeypatch.setenv("BRAINTRUST_TEST_PIPECAT_ENTRYPOINT", entrypoint) verify_autoinstrument_script("test_auto_pipecat.py") # Voice sessions -@pytest.mark.vcr +@pytest.mark.vcr(cassette_name="test_voice_session_traces_turns_speech_recognition_generation_and_playback") @pytest.mark.asyncio +@pytest.mark.parametrize( + "speaking_frames,inference_gate,smart_turn,cancel", + [ + (True, False, False, False), + (False, False, False, False), + (True, True, False, False), + (True, False, True, False), + (True, False, False, True), + ], +) async def test_voice_session_traces_turns_speech_recognition_generation_and_playback( - memory_logger, attachment_uploads + memory_logger, attachment_uploads, speaking_frames, inference_gate, smart_turn, cancel ): from braintrust._audio.config import AudioEncoding, AudioRecordingOptions + from braintrust.integrations.versioning import detect_module_version, version_satisfies + pipecat = pytest.importorskip("pipecat") + if inference_gate and not version_satisfies(detect_module_version(pipecat, ("pipecat",)), ">=1.16.0"): + pytest.skip("ALL stop-strategy composition is qualified on Pipecat 1.16.0") _instrument( capture_audio_attachments=True, audio_recording_options=AudioRecordingOptions(encoding=AudioEncoding(codec="pcm", container="wav")), ) - session = _VoiceSession(system="Reply with exactly: Hi.") + session = _VoiceSession( + system="Reply with exactly: Hi.", + speaking_frames=speaking_frames, + inference_gate=inference_gate, + smart_turn=smart_turn, + cancel=cancel, + ) await session.run() logs = memory_logger.pop() root = _single_span(logs, "session") assert root["metadata"]["audio.schema.version"] == "3.1.0" - assert root["metadata"]["contrib.pipecat.terminal_frame"] == "EndFrame" - assert root["metadata"]["contrib.pipecat.reason"] == "pipeline complete" + assert root["metadata"]["contrib.pipecat.terminal_frame"] == ("CancelFrame" if cancel else "EndFrame") + if not cancel: + assert root["metadata"]["contrib.pipecat.reason"] == "pipeline complete" + assert session.observer._controllers == {} + assert session.observer._controller_queue is None + assert session.tracker.user._last_speech is None + assert session.tracker.user._last_turn is None + assert not session.tracker.user._recognitions + assert not session.tracker.user._recognized_until + assert not session.tracker.linker.messages + assert not session.tracker.linker._submitted + assert not session.tracker.linker._transcripts + assert not session.tracker.linker._tool_results + for controller in session.controllers: + assert controller.span is None and controller.turn is None + assert controller.session is None and controller.controller is None user_turn = _single_span(logs, "user_turn") agent_turn = _single_span(logs, "agent_turn") speech = _single_span(logs, "user_speech") + wait = _single_span(logs, "eou_wait") + assert wait["metadata"]["eou.outcome"] == "committed" + assert wait["metadata"]["eou.speech_end_source"] == "vad" + assert wait["span_parents"] == [user_turn["span_id"]] + assert wait["metadata"]["turn.id"] == user_turn["span_id"] + assert wait["metrics"]["start"] == pytest.approx(speech["metrics"]["end"], abs=0.03) + assert "eou.detection_methods" not in wait["metadata"] recognition = _single_span(logs, "stt") model = _single_span(logs, "llm_response") + if inference_gate: + assert wait["metrics"]["start"] < model["metrics"]["start"] < wait["metrics"]["end"] + assert "eou.end_reason" not in wait["metadata"] synthesis = _single_span(logs, "tts") playback = _single_span(logs, "agent_speech") @@ -417,7 +539,10 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play assert user_turn["input"] == recognition["output"] # Agent side: the reply turn owns generation, synthesis and playback. - assert agent_turn["metadata"]["turn.reply_to"] == user_turn["span_id"] + if inference_gate: + assert "turn.reply_to" not in agent_turn["metadata"] + else: + assert agent_turn["metadata"]["turn.reply_to"] == user_turn["span_id"] assert agent_turn["metadata"]["turn.incomplete"] is False for operation in (model, synthesis, playback): assert operation["span_parents"] == [agent_turn["span_id"]] @@ -427,7 +552,10 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play assert model["metadata"]["model"] == TEST_MODEL assert model["metadata"]["provider"] == "openai" assert model["input"][-1] == recognition["output"][0] - assert model["input"][0] == {"role": "system", "content": session.system} + if inference_gate: + assert any(message.get("content") == session.system for message in model["input"]) + else: + assert model["input"][0] == {"role": "system", "content": session.system} assert model["output"][0]["role"] == "assistant" assert model["output"][0]["content"].strip() == "Hi." assert set(model["output"][0]) == {"role", "content"} @@ -461,9 +589,12 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play _assert_all_ended(logs) -@pytest.mark.vcr +@pytest.mark.vcr(cassette_name="test_consecutive_voice_turns_reply_to_and_link_only_their_own_speech") @pytest.mark.asyncio -async def test_consecutive_voice_turns_reply_to_and_link_only_their_own_speech(memory_logger, attachment_uploads): +@pytest.mark.parametrize("delayed", [False, True]) +async def test_consecutive_voice_turns_reply_to_and_link_only_their_own_speech( + memory_logger, attachment_uploads, delayed +): from braintrust._audio.config import AudioEncoding, AudioRecordingOptions # User audio only, configured when wrapping: agent playback is not recorded. @@ -474,7 +605,7 @@ async def test_consecutive_voice_turns_reply_to_and_link_only_their_own_speech(m capture_agent_audio_attachments=False, audio_recording_options=AudioRecordingOptions(encoding=AudioEncoding(codec="pcm", container="wav")), ) is _import("pipecat.pipeline.worker.PipelineWorker") - await _VoiceSession(system="Reply with exactly one word: Okay.").run(utterances=2) + await _VoiceSession(system="Reply with exactly one word: Okay.", delay_observer=delayed).run(utterances=2) logs = memory_logger.pop() user_turns = _spans_named(logs, "user_turn") @@ -482,6 +613,11 @@ async def test_consecutive_voice_turns_reply_to_and_link_only_their_own_speech(m speeches = _spans_named(logs, "user_speech") recognitions = _spans_named(logs, "stt") models = _spans_named(logs, "llm_response") + waits = _spans_named(logs, "eou_wait") + assert len(waits) == 2 + for wait, turn in zip(waits, user_turns): + assert wait["span_parents"] == [turn["span_id"]] + assert wait["metadata"]["turn.id"] == turn["span_id"] for synthesis in _spans_named(logs, "tts"): assert synthesis["metrics"]["characters"] == len(synthesis["input"]["text"]) assert len(user_turns) == len(agent_turns) == len(speeches) == len(recognitions) == len(models) == 2 @@ -1131,3 +1267,270 @@ async def test_provider_transcriptions_nest_under_their_stt_spans(memory_logger, for response, call in zip(_spans_named(logs, "llm_response"), _spans_named(logs, "Chat Completion")): assert call["span_parents"] == [response["span_id"]] assert all(row["metrics"].get("end") is not None for row in logs) + + +@pytest_asyncio.fixture +async def tracker_observer(memory_logger): + """Supplemental timestamp/layout cases; recorded pipelines cover behavior.""" + _frames() + from braintrust.integrations.pipecat.tracing.observer import BraintrustPipecatObserver + + observer = BraintrustPipecatObserver(capture_audio_attachments=False) + yield observer + await observer.cleanup() + + +@asynccontextmanager +async def _native_eou_controller(observer, strategies, *, observe_start=True, begin_turn=True): + """Synthetic native frames + real controller/strategies/timers, no provider IO.""" + from braintrust.integrations.pipecat.patchers import _ACTIVE_OBSERVER + + Controller = _import("pipecat.turns.user_turn_controller.UserTurnController") + TaskManager = _import("pipecat.utils.asyncio.task_manager.TaskManager") + TaskManagerParams = _import("pipecat.utils.asyncio.task_manager.TaskManagerParams") + controller = Controller(user_turn_strategies=strategies, user_turn_stop_timeout=30) + manager = TaskManager() + manager.setup(TaskManagerParams(loop=asyncio.get_running_loop())) + if "task_manager" in inspect.signature(controller.setup).parameters: + await controller.setup(manager) + else: + Setup = _import("pipecat.processors.frame_processor.FrameProcessorSetup") + Clock = _import("pipecat.clocks.system_clock.SystemClock") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + Identity = _import("pipecat.processors.filters.identity_filter.IdentityFilter") + await controller.setup( + Setup(clock=Clock(), task_manager=manager, pipeline_worker=_make_worker(Pipeline([Identity()]))) + ) + token = _ACTIVE_OBSERVER.set(observer if observe_start else None) + try: + if begin_turn: + await controller.process_frame(_frames().VADUserStartedSpeakingFrame()) + await asyncio.sleep(0.02) + if not observe_start: + _ACTIVE_OBSERVER.reset(token) + token = _ACTIVE_OBSERVER.set(observer) + yield controller + finally: + await controller.cleanup() + _ACTIVE_OBSERVER.reset(token) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "mode", + [ + "committed", + "resumed", + "transcript_late", + "shutdown", + "unassociated", + "multiple", + "audio_only", + "text_late", + "gate_changed", + ], +) +async def test_eou_native_controller_lifecycle(memory_logger, tracker_observer, mode): + + _instrument() + frames = _frames() + Strategies = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") + SpeechTimeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + options = {} + if mode in ("audio_only", "gate_changed"): + if "wait_for_transcript" not in inspect.signature(SpeechTimeout).parameters: + pytest.skip("wait_for_transcript=False requires current Pipecat") + options["wait_for_transcript"] = False + strategies = [SpeechTimeout(user_speech_timeout=0.03, **options)] + if mode == "multiple": + strategies.append(SpeechTimeout(user_speech_timeout=0.06)) + stopped = asyncio.Event() + async with _native_eou_controller( + tracker_observer, Strategies(stop=strategies), observe_start=mode != "unassociated" + ) as controller: + + @controller.event_handler("on_user_turn_stopped") + async def on_stopped(*_args): + stopped.set() + + await controller.process_frame( + frames.STTMetadataFrame( + service_name="local-test", + ttfs_p99_latency=0 if mode in ("audio_only", "text_late", "gate_changed") else 1.0, + ) + ) + pause = frames.VADUserStoppedSpeakingFrame(stop_secs=0.01) + await controller.process_frame(pause) + if mode == "gate_changed": + strategies[0].wait_for_transcript = True + await asyncio.sleep(0.06) + assert not stopped.is_set(), "The controller must still require text" + if mode == "resumed": + resume = frames.VADUserStartedSpeakingFrame(start_secs=0.001) + await controller.process_frame(resume) + await asyncio.sleep(0.02) + pause = frames.VADUserStoppedSpeakingFrame(stop_secs=0.01) + await controller.process_frame(pause) + # Multiple transcriptions/evaluations during this pause must not split it. + await controller.process_frame(frames.InterimTranscriptionFrame("Hi", "caller", "0")) + await controller.process_frame(pause) # Re-evaluate/re-arm this same pause. + if mode in ("transcript_late", "text_late"): + await asyncio.sleep(0.06) + if mode != "shutdown": + if mode != "audio_only": + await controller.process_frame(frames.TranscriptionFrame("Hi", "caller", "0", finalized=True)) + await asyncio.wait_for(stopped.wait(), 2) + else: + assert not stopped.is_set() + await tracker_observer.cleanup() + logs = memory_logger.pop() + emitted = _spans_named(logs, "eou_wait") + assert len(emitted) == (2 if mode == "resumed" else 1) + final = emitted[-1] + assert final["metrics"]["start"] == pause.timestamp - pause.stop_secs + assert final["metadata"]["contrib.pipecat.vad_timestamp"] == pause.timestamp + assert final["metadata"]["contrib.pipecat.vad_stop_secs"] == pause.stop_secs + assert len(final["metadata"]["contrib.pipecat.configured_stop_strategies"]) == len(strategies) + if mode == "shutdown": + assert "eou.outcome" not in final["metadata"] + assert "eou.end_reason" not in final["metadata"] + else: + assert final["metadata"]["eou.outcome"] == "committed" + if mode == "transcript_late": + assert final["metadata"]["eou.end_reason"] == "transcription_finalized" + if mode == "text_late": + assert "eou.end_reason" not in final["metadata"] + if mode == "resumed": + assert emitted[0]["metadata"]["eou.outcome"] == "user_resumed" + assert emitted[0]["metrics"]["end"] == resume.timestamp - resume.start_secs + assert "eou.end_reason" not in emitted[0]["metadata"] + if mode == "unassociated": + assert "turn.id" not in final["metadata"] + assert final["span_parents"] == [_single_span(logs, "session")["span_id"]] + else: + assert final["span_parents"] == [final["metadata"]["turn.id"]] + for wait in emitted: + assert wait["metadata"]["eou.speech_end_source"] == "vad" + if mode == "audio_only": + assert wait["metadata"]["eou.end_reason"] == "silence_timeout" + assert "eou.detection_methods" not in wait["metadata"] + assert wait["context"]["span_origin"]["instrumentation"]["name"] == "pipecat-auto" + _assert_all_ended(logs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("delayed", [False, True]) +async def test_eou_queued_observations_keep_turn_ownership(memory_logger, tracker_observer, monkeypatch, delayed): + """Real WorkerObserver queue falls behind two real native controller turns.""" + _instrument() + WorkerObserver = _import("pipecat.pipeline.worker_observer.WorkerObserver") + FramePushed = _import("pipecat.observers.base_observer.FramePushed") + Direction = _import("pipecat.processors.frame_processor.FrameDirection") + Identity = _import("pipecat.processors.filters.identity_filter.IdentityFilter") + Manager = _import("pipecat.utils.asyncio.task_manager.TaskManager") + Params = _import("pipecat.utils.asyncio.task_manager.TaskManagerParams") + Strategies = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") + Timeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + manager = Manager() + manager.setup(Params(loop=asyncio.get_running_loop())) + worker = WorkerObserver(observers=[tracker_observer]) + release = asyncio.Event() + entered = asyncio.Event() + original = tracker_observer.on_push_frame + + async def on_push(data): + if delayed and not entered.is_set(): + entered.set() + await release.wait() + await original(data) + + monkeypatch.setattr(tracker_observer, "on_push_frame", on_push) + await worker.setup(manager) + if hasattr(worker, "start"): + await worker.start() + source, destination = Identity(), Identity() + session = tracker_observer._start_session() + with session.clock.event(time.monotonic_ns()): + pass + + async def push(frame): + await worker.on_push_frame(FramePushed(source, destination, frame, Direction.DOWNSTREAM, time.monotonic_ns())) + + try: + async with _native_eou_controller( + tracker_observer, Strategies(stop=[Timeout(user_speech_timeout=0.03)]), begin_turn=False + ) as controller: + committed = asyncio.Queue() + committed_at = [] + + @controller.event_handler("on_user_turn_stopped") + async def on_stopped(*_args): + committed_at.append(time.time()) + committed.put_nowait(True) + + async def process(frame): + await push(frame) + await controller.process_frame(frame) + + for index in range(2): + await process(_frames().VADUserStartedSpeakingFrame()) + await asyncio.sleep(0.005) # Speech must outlast the stop correction. + await process(_frames().VADUserStoppedSpeakingFrame(stop_secs=0.001)) + await process(_frames().TranscriptionFrame(f"Turn {index}", "caller", "0", finalized=True)) + await asyncio.wait_for(committed.get(), 2) + if delayed: + await asyncio.wait_for(entered.wait(), 2) + release.set() + await asyncio.wait_for(worker._proxies[tracker_observer].queue.join(), 2) + finally: + release.set() + if hasattr(worker, "stop"): + await worker.stop() + await worker.cleanup() + logs = memory_logger.pop() + turns = _spans_named(logs, "user_turn") + assert len(turns) == 2 + for name in ("user_speech", "transcript_received", "eou_wait"): + rows = _spans_named(logs, name) + assert len(rows) == 2 + for row, turn in zip(rows, turns): + assert row["span_parents"] == [turn["span_id"]], (name, row, turn) + assert row["metadata"]["turn.id"] == turn["span_id"] + for wait, turn, accepted_at in zip(_spans_named(logs, "eou_wait"), turns, committed_at): + assert wait["metrics"]["end"] <= accepted_at + assert turn["metrics"]["end"] == pytest.approx(wait["metrics"]["end"], abs=1e-6, rel=0) + _assert_all_ended(logs) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "stamp,correction", [(0, 0), (-1, 0), (float("nan"), 0), (float("inf"), 0), (None, 0), (None, -1)] +) +async def test_eou_omits_invalid_speech_timestamps(memory_logger, tracker_observer, stamp, correction): + _instrument() + Strategies = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") + SpeechTimeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + async with _native_eou_controller( + tracker_observer, Strategies(stop=[SpeechTimeout(user_speech_timeout=1)]) + ) as controller: + frame = _frames().VADUserStoppedSpeakingFrame(timestamp=stamp, stop_secs=correction) + await controller.process_frame(frame) + await tracker_observer.cleanup() + assert not _spans_named(memory_logger.pop(), "eou_wait") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["external", "inactive"]) +async def test_eou_external_turns_do_not_make_a_wait_from_vad(memory_logger, tracker_observer, mode): + _instrument() + Strategies = _import("pipecat.turns.user_turn_strategies.ExternalUserTurnStrategies") + if mode == "external": + strategies = Strategies() + else: + Local = _import("pipecat.turns.user_turn_strategies.UserTurnStrategies") + Timeout = _import("pipecat.turns.user_stop.SpeechTimeoutUserTurnStopStrategy") + strategies = Local(stop=[Timeout()]) + async with _native_eou_controller(tracker_observer, strategies, begin_turn=mode != "inactive") as controller: + await controller.process_frame(_frames().VADUserStoppedSpeakingFrame()) + await tracker_observer.cleanup() + assert not _spans_named(memory_logger.pop(), "eou_wait") diff --git a/py/src/braintrust/integrations/pipecat/tracing/_common.py b/py/src/braintrust/integrations/pipecat/tracing/_common.py index 5a0daca1..14d70fc4 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_common.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_common.py @@ -85,8 +85,8 @@ def start( **kwargs, ) - def turn(self, name: str) -> Span: - span = self.start(name) + def turn(self, name: str, *, start_ms: float | None = None) -> Span: + span = self.start(name, start_ms=start_ms) span.log(metadata={TURN_ID: span_id(span)}) return span diff --git a/py/src/braintrust/integrations/pipecat/tracing/_context.py b/py/src/braintrust/integrations/pipecat/tracing/_context.py index 62577ccf..7c7e7c99 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_context.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_context.py @@ -62,6 +62,16 @@ def add_transcript(self, text: str, span: Span) -> None: self._warn("transcript") self._transcripts.append((text, span)) + def clear(self) -> None: + """Release routing payloads after all session work has been exported.""" + self.messages = None + self.metadata.clear() + self._submitted = () + self._consumed = 0 + self._transcripts.clear() + self._tool_results.clear() + self._warned.clear() + def add_tool_result(self, tool_call_id: str, span: Span) -> None: self._tool_results[tool_call_id] = span if len(self._tool_results) > MAX_PENDING: diff --git a/py/src/braintrust/integrations/pipecat/tracing/_eou.py b/py/src/braintrust/integrations/pipecat/tracing/_eou.py new file mode 100644 index 00000000..7b7d166f --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/tracing/_eou.py @@ -0,0 +1,155 @@ +"""Waits observed at the local turn controller, rather than inferred from VAD. + +Qualified built-in stop strategies consume the pause and start a real decision +wait. Other modes (notably externally managed realtime turns) are omitted. +""" + +import math +import time +from collections.abc import Callable +from contextvars import ContextVar +from dataclasses import dataclass +from typing import Any + +from braintrust._audio.schema import PARTICIPANT_IDS, SOURCE_ID, TURN_ID + +from ._common import INSTRUMENTATION + + +CONDITION: ContextVar[tuple[Any, str] | None] = ContextVar("braintrust_pipecat_eou_condition", default=None) +_LOCAL_STRATEGIES = { + ("pipecat.turns.user_stop.speech_timeout_user_turn_stop_strategy", "SpeechTimeoutUserTurnStopStrategy"), + ("pipecat.turns.user_stop.turn_analyzer_user_turn_stop_strategy", "TurnAnalyzerUserTurnStopStrategy"), +} + +_AUXILIARY_STRATEGIES = { + ("pipecat.turns.user_stop.llm_turn_completion_user_turn_stop_strategy", "LLMTurnCompletionUserTurnStopStrategy"), +} + + +def stop_strategies(controller): + configured = getattr(controller, "user_turn_strategies", None) + if configured is None: # Pipecat 1.3.0 has no public property. + configured = getattr(controller, "_user_turn_strategies", None) + return tuple(getattr(configured, "stop", None) or ()) + + +def speech_time(frame, correction): + stamp, delay = getattr(frame, "timestamp", None), getattr(frame, correction, None) + if not all(type(v) in (int, float) and math.isfinite(v) for v in (stamp, delay)): + return None + if stamp <= 0 or delay < 0 or stamp - delay <= 0 or stamp > time.time(): + return None + return stamp - delay + + +@dataclass +class ControllerWait: + controller: Any + session: Any + enqueue: Callable[[Callable[[], None]], None] | None + turn: Any = None + span: Any = None + start: float | None = None + # A resumed event can arrive through both the pipeline and controller. + last_stop: float | None = None + + def observe(self, frame): + # Capture framework state before it changes, then update all trace + # trackers in the native observer queue's FIFO order. + if self.enqueue is None: + return + kind = type(frame).__name__ + if kind not in ("VADUserStartedSpeakingFrame", "VADUserStoppedSpeakingFrame"): + return + speech_at = speech_time(frame, "start_secs" if kind == "VADUserStartedSpeakingFrame" else "stop_secs") + strategies = stop_strategies(self.controller) + active = bool(getattr(self.controller, "_user_turn", False)) + self.enqueue(lambda: self._observe(frame, strategies, active, speech_at)) + + def _observe(self, frame, strategies, active, speech_at): + kind = type(frame).__name__ + if kind == "VADUserStartedSpeakingFrame": + resumed = speech_at + self.finish("user_resumed" if resumed is not None else None, resumed) + self.last_stop = None + elif kind == "VADUserStoppedSpeakingFrame": + stopped = speech_at + # Unknown/custom strategies may not consume this frame, or may use + # FIRST/ALL semantics this spike cannot fully establish. Do not + # infer a local completion wait from a configured detector alone. + if ( + stopped is None + or self.span is not None + or stopped == self.last_stop + or not strategies + or not active + or not any((type(s).__module__, type(s).__name__) in _LOCAL_STRATEGIES for s in strategies) + or not all( + (type(s).__module__, type(s).__name__) in _LOCAL_STRATEGIES | _AUXILIARY_STRATEGIES + for s in strategies + ) + ): + return + self.start = self.last_stop = stopped + metadata = { + SOURCE_ID: "session-input", + "eou.speech_end_source": "vad", + "contrib.pipecat.configured_stop_strategies": [type(s).__name__ for s in strategies], + "contrib.pipecat.vad_timestamp": frame.timestamp, + "contrib.pipecat.vad_stop_secs": frame.stop_secs, + } + if self.turn is not None: + metadata[TURN_ID] = self.turn.span_id + participants = self.session.user.speech_participants + if participants: + metadata[PARTICIPANT_IDS] = participants + # Strategy identity/configuration snapshots cannot prove the full + # applied gate history: public settings may change in place and + # change back between observations. Omit detection_methods. + self.span = (self.turn or self.session.span).start_span( + name="eou_wait", + type="task", + start_time=stopped, + metadata=metadata, + internal={"instrumentation": INSTRUMENTATION}, + ) + + def event(self, name, args): + if self.enqueue is None: + return + reason = CONDITION.get() + end_reason = reason[1] if reason is not None and args and args[0] is reason[0] else None + observed_at = time.time() + self.enqueue(lambda: self._event(name, observed_at, end_reason)) + + def _event(self, name, observed_at, end_reason): + if name == "on_user_turn_started": + self.session.user.start_turn(self.session.clock.from_wall_ms(observed_at)) + self.turn = self.session.user.turn_span + elif name == "on_user_turn_stopped": + # The causal condition must belong to the strategy which actually + # completed this controller's turn. No inference-event shortcut. + self.finish("committed", ended=observed_at, end_reason=end_reason) + self.session.user.stop_turn(self.session.clock.from_wall_ms(observed_at)) + self.turn = None + + def finish(self, outcome=None, ended=None, end_reason=None): + span, self.span = self.span, None + if span is None: + return + ended = time.time() if ended is None else ended + metadata = {} + if self.start is not None and ended < self.start: + ended, outcome = time.time(), None + if outcome is not None: + metadata["eou.outcome"] = outcome + if outcome == "committed" and end_reason is not None: + metadata["eou.end_reason"] = end_reason + span.log(metadata=metadata) + span.end(end_time=max(ended, self.start or ended)) + + def close(self, ended): + self.finish(ended=ended) + self.turn = self.controller = self.session = self.enqueue = None + self.start = self.last_stop = None diff --git a/py/src/braintrust/integrations/pipecat/tracing/_user.py b/py/src/braintrust/integrations/pipecat/tracing/_user.py index bcad9f70..c99b4c6d 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_user.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_user.py @@ -80,6 +80,7 @@ def __init__( self._speech: _UserSpeech | None = None self._last_turn: Span | None = None self._last_speech: Span | None = None + self._last_speech_participants: list[str] = [] self._turn_index = 0 self._vad_seen = False self._recognitions: dict[int, _Recognition] = {} @@ -100,21 +101,29 @@ def _turn_span(self) -> Span | None: def _context_turn(self) -> Span | None: return self._turn_span or self._last_turn - def start_turn(self) -> None: + @property + def turn_span(self) -> Span | None: + return self._turn_span + + @property + def speech_participants(self) -> list[str]: + return sorted(self._speech.participants) if self._speech is not None else self._last_speech_participants + + def start_turn(self, start_ms: float | None = None) -> None: if self._turn is None: - self._turn = _UserTurn(self._spans.turn("user_turn")) + self._turn = _UserTurn(self._spans.turn("user_turn", start_ms=start_ms)) if self._speech is not None: self._speech_span(self._speech) self._resolve_recognitions() - def stop_turn(self) -> None: + def stop_turn(self, end_ms: float | None = None) -> None: self._resolve_recognitions() # Recognition without a final transcript must not keep timing through # the agent's reply or into the next user turn. A later transcript can # still be represented as transcript_received. for recognition in list(self._recognitions.values()): - self._retire(recognition) - self._finish_turn(incomplete=False) + self._retire(recognition, end_ms) + self._finish_turn(incomplete=False, end_ms=end_ms) def log_input(self, message: dict[str, Any]) -> None: turn = self._context_turn @@ -124,6 +133,7 @@ def log_input(self, message: dict[str, Any]) -> None: def start_speech(self, start_ms: float) -> None: self._vad_seen = True if self._speech is None: + self._last_speech_participants = [] self._speech = _UserSpeech(start_ms) if self._turn is not None: self._speech_span(self._speech) @@ -139,6 +149,7 @@ def stop_speech(self, end_ms: float | None = None) -> None: span.end(end_time=self._clock.wall(end_ms)) if self._turn is not None: self._turn.selections.append(interval) + self._last_speech_participants = sorted(speech.participants) self._last_speech, self._speech = span, None def identify(self, frame: "Frame", span: Span | None = None) -> None: @@ -219,6 +230,8 @@ def close(self, end_ms: float) -> None: self._recognitions.clear() self._recognized_until.clear() self._finish_turn(incomplete=True, end_ms=end_ms) + self._last_turn = self._last_speech = None + self._last_speech_participants.clear() def _finish_turn(self, *, incomplete: bool, end_ms: float | None = None) -> None: turn = self._turn @@ -231,6 +244,7 @@ def _finish_turn(self, *, incomplete: bool, end_ms: float | None = None) -> None } ) turn.span.end(end_time=self._clock.wall(end_ms)) + self._last_speech_participants = [] self._last_turn, self._turn = turn.span, None self._turn_index += 1 @@ -261,11 +275,11 @@ def _open(self, recognition: _Recognition) -> Span: recognition.phase = _RecognitionPhase.ACTIVE return recognition.span - def _retire(self, recognition: _Recognition) -> None: + def _retire(self, recognition: _Recognition, end_ms: float | None = None) -> None: self._recognitions.pop(id(recognition.processor), None) if recognition.span is not None: recognition.span.log(metadata={"contrib.pipecat.incomplete": True}) - recognition.span.end(end_time=self._clock.wall()) + recognition.span.end(end_time=self._clock.wall(end_ms)) def _emit(self, frame: "TranscriptionFrame", recognition: _Recognition | None, end_ms: float) -> None: speech: Span | None diff --git a/py/src/braintrust/integrations/pipecat/tracing/observer.py b/py/src/braintrust/integrations/pipecat/tracing/observer.py index 03e134c0..56414c71 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/observer.py +++ b/py/src/braintrust/integrations/pipecat/tracing/observer.py @@ -34,7 +34,7 @@ try: import pipecat from pipecat.frames import frames - from pipecat.observers.base_observer import BaseObserver + from pipecat.observers.base_observer import BaseObserver, FrameProcessed except ImportError: # pragma: no cover - Pipecat is an optional dependency. _USES_NATIVE_FRAME_DEDUPLICATION = False @@ -44,6 +44,9 @@ def __init__(self, **_kwargs: Any) -> None: async def cleanup(self) -> None: pass + + class FrameProcessed: + pass else: _USES_NATIVE_FRAME_DEDUPLICATION = version_satisfies(detect_module_version(pipecat, ("pipecat",)), ">=1.12.0") @@ -116,6 +119,18 @@ def clear(self) -> None: ] +class _ControllerObservation(FrameProcessed): + """Instrumentation-only queue item; never submitted to the pipeline. + + Both supported WorkerObservers route FrameProcessed items to the same + observer task as frames. No pipeline operation waits for this callback. + """ + + def __init__(self, action: Callable[[], None]) -> None: + super().__init__(processor=None, frame=None, direction=None, timestamp=0) + self.action = action + + class BraintrustPipecatObserver(BaseObserver): """Observe cascade sessions, native turns, speech, STT, LLM, tools and TTS. @@ -149,6 +164,8 @@ def __init__( self._phase = _SessionPhase.NEW self._session: _Session | None = None self._finish_task: asyncio.Task[RecordingResult] | None = None + self._controllers: dict[Any, Any] = {} + self._controller_queue: asyncio.Queue | None = None # Response start frame ID -> its llm_response span, for the service. self._response_waiters: dict[int, asyncio.Future[Span | None]] = {} @@ -162,6 +179,9 @@ async def on_process_frame(self, data: "FrameProcessed") -> None: if self._phase is _SessionPhase.CLOSED: return try: + if isinstance(data, _ControllerObservation): + data.action() + return frame, processor = data.frame, data.processor session = self._start_session() kind = processor_kind(type(processor)) @@ -304,6 +324,21 @@ def _current_session(self) -> Iterator[None]: finally: span.unset_current() + def _controller(self, controller): + from ._eou import ControllerWait + + if self._phase is _SessionPhase.CLOSED: + return None + if controller not in self._controllers: + self._controllers[controller] = ControllerWait(controller, self._start_session(), self._enqueue_controller) + return self._controllers[controller] + + def _enqueue_controller(self, action: Callable[[], None]) -> None: + if self._controller_queue is None: + action() # Directly used observer, without a native worker proxy. + else: + self._controller_queue.put_nowait(_ControllerObservation(action)) + # Session lifecycle def _start_session(self) -> _Session: @@ -352,10 +387,16 @@ async def _finish(self) -> None: if session is None: return end_ms = session.clock.now_ms() + for controller in self._controllers.values(): + # Use the same observed shutdown boundary as the session. + controller.close(ended=session.clock.wall(end_ms)) + self._controllers.clear() + self._controller_queue = None closers: tuple[Callable[[], None], ...] = ( lambda: session.live.close(end_ms) if session.live is not None else None, lambda: session.user.close(end_ms), lambda: session.agent.interrupt(interrupted=False, end_ms=end_ms), + session.linker.clear, lambda: session.span.end(end_time=session.clock.wall(end_ms)), ) for close in closers: @@ -471,10 +512,12 @@ def _on_error( return self._finish() if frame.fatal else None def _on_user_turn_started(self, s: _Session, _f: "Frame", _p: "FrameProcessor | None", _played: bool) -> None: - s.user.start_turn() + if not self._controllers: + s.user.start_turn() def _on_user_turn_stopped(self, s: _Session, _f: "Frame", _p: "FrameProcessor | None", _played: bool) -> None: - s.user.stop_turn() + if not self._controllers: + s.user.stop_turn() def _on_vad_started(self, s: _Session, frame: "Frame", _p: "FrameProcessor | None", _played: bool) -> None: s.user.start_speech(s.clock.from_wall_ms(frame.timestamp - getattr(frame, "start_secs", 0)))