diff --git a/examples/livekit/README.md b/examples/livekit/README.md index df2ea6b5..21d2a6ec 100644 --- a/examples/livekit/README.md +++ b/examples/livekit/README.md @@ -89,3 +89,13 @@ separate from the segmented session recording. Duration-only STT usage reports do not create token counts. TTS model children record the text after LiveKit's configured text transforms. Media model inputs identify their `operation` as `transcribe` or `speech`. + + +LiveKit turn diagnostics are included without an OpenTelemetry exporter. User +turns expose `contrib.livekit.end_of_turn_delay`, +`contrib.livekit.transcription_delay`, and +`contrib.livekit.on_user_turn_completed_delay` in seconds when available. +`livekit.eou_wait` shows the observed interval from speech end to an accepted +turn decision. User speech spans retain native `contrib.livekit.vad_silence_duration` +and `contrib.livekit.vad_inference_duration`, also in seconds. Missing native +timing stays absent. Model timing and usage continue to belong to model children. diff --git a/py/src/braintrust/integrations/livekit_agents/test_cascade.py b/py/src/braintrust/integrations/livekit_agents/test_cascade.py index c1142a47..bbf1cf68 100644 --- a/py/src/braintrust/integrations/livekit_agents/test_cascade.py +++ b/py/src/braintrust/integrations/livekit_agents/test_cascade.py @@ -64,6 +64,7 @@ def upload(attachment): llm.on("metrics_collected", model_metrics.append) vad = silero.VAD.load() heard = asyncio.Event() + eou_metrics = [] input_source = rtc.AudioSource(24000, 1) track = rtc.LocalAudioTrack.create_audio_track("caller-microphone", input_source) await rooms[1].local_participant.publish_track( @@ -80,6 +81,10 @@ def conversation(event): heard.set() session.on("conversation_item_added", conversation) + session.on( + "metrics_collected", + lambda event: eou_metrics.append(event.metrics) if event.metrics.type == "eou_metrics" else None, + ) await session.start( Agent(instructions="Reply with exactly: hi there", llm=llm), room=rooms[0], record=False ) @@ -148,6 +153,24 @@ def conversation(event): user = next(row for row in named(rows, "user_turn") if row.get("input")) assert user["input"][0]["content"] == "Hello there." assert user["metadata"]["audio.selections"] + if not preemptive: + assert len(eou_metrics) == 1 + if eou_metrics: + reported = eou_metrics[0] + assert user["metadata"]["contrib.livekit.end_of_turn_delay"] == pytest.approx(reported.end_of_utterance_delay) + assert user["metadata"]["contrib.livekit.transcription_delay"] == pytest.approx(reported.transcription_delay) + assert ( + user["metadata"]["contrib.livekit.on_user_turn_completed_delay"] == reported.on_user_turn_completed_delay + ) + wait = named(rows, "livekit.eou_wait")[0] + assert wait["span_parents"] == [user["span_id"]] + assert wait["span_attributes"]["type"] == "task" + assert wait["metrics"]["end"] - wait["metrics"]["start"] == pytest.approx( + user["metadata"]["contrib.livekit.end_of_turn_delay"], abs=1e-6 + ) + for speech in named(rows, "user_speech"): + assert speech["metadata"]["contrib.livekit.vad_silence_duration"] >= 0 + assert speech["metadata"]["contrib.livekit.vad_inference_duration"] >= 0 by_id = {row["span_id"]: row for row in rows} for speech in named(rows, "user_speech"): parent = by_id[speech["span_parents"][0]] diff --git a/py/src/braintrust/integrations/livekit_agents/tracing.py b/py/src/braintrust/integrations/livekit_agents/tracing.py index b0ba2484..4d72ee7a 100644 --- a/py/src/braintrust/integrations/livekit_agents/tracing.py +++ b/py/src/braintrust/integrations/livekit_agents/tracing.py @@ -4,6 +4,7 @@ import contextlib import inspect import logging +import math import re import time import weakref @@ -85,6 +86,18 @@ def _operation_context(span: Span): span.unset_current() +def _log_native_diagnostics(span, **fields): + # Call sites explicitly select native fields. Do not invent zeroes when + # LiveKit lacks a reliable timing anchor, or promote these to model metrics. + metadata = { + f"contrib.livekit.{name}": value + for name, value in fields.items() + if type(value) in (int, float) and math.isfinite(value) and value >= 0 + } + if metadata: + span.log(metadata=metadata) + + def link(target, relationship): global _warned_links trace_id, span_id = getattr(target, "root_span_id", ""), getattr(target, "span_id", "") @@ -137,6 +150,7 @@ def __init__(self, native, record): self.closed = False self.closing = False self._finishing = None + native.on("metrics_collected", self.metrics_collected) if self.privacy.job is not None: reference = weakref.ref(self) @@ -152,6 +166,13 @@ async def shutdown(): self.privacy.job.add_shutdown_callback(shutdown) + def metrics_collected(self, event): + # The native callback report is emitted in _user_turn_completed_impl, + # where our existing context identifies the exact originating user turn. + user = _REPLY_TO.get() + if event.metrics.type == "eou_metrics" and user is not None: + _log_native_diagnostics(user, on_user_turn_completed_delay=event.metrics.on_user_turn_completed_delay) + def permits(self): allowed = self.privacy.allows_content() if not allowed: @@ -318,6 +339,8 @@ async def _finish(self): try: await self.audio.finish() finally: + if self.native is not None: + self.native.off("metrics_collected", self.metrics_collected) self.native = None self.privacy.session = None self.privacy.job = None @@ -500,6 +523,12 @@ def user_speech_end(wrapped, instance, args, kwargs): owner = state_for(instance) event = arguments(wrapped, args, kwargs).get("ev") if owner is not None and owner.user_speech is not None: + if event is not None: + _log_native_diagnostics( + owner.user_speech, + vad_silence_duration=event.silence_duration, + vad_inference_duration=event.inference_duration, + ) end = time.time() - event.silence_duration - event.inference_duration if event is not None else time.time() if event is not None: end_ns = owner.audio.from_wall(end) @@ -579,6 +608,23 @@ def end_of_turn(wrapped, instance, args, kwargs): try: committed = wrapped(*args, **kwargs) if committed: + timing = info.metrics + _log_native_diagnostics( + user, end_of_turn_delay=timing.end_of_turn_delay, transcription_delay=timing.transcription_delay + ) + stopped, delay = timing.stopped_speaking_at, timing.end_of_turn_delay + if ( + type(stopped) in (int, float) + and type(delay) in (int, float) + and math.isfinite(stopped) + and math.isfinite(delay) + and stopped >= 0 + and delay >= 0 + ): + wait = _new( + user, "livekit.eou_wait", start_time=stopped, metadata={"contrib.livekit.outcome": "committed"} + ) + wait.end(stopped + delay) owner.user = None if info.user_turn_span_adopted: owner.pending_users.add(user)