Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions examples/livekit/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,3 +89,13 @@ separate from the segmented session recording. Duration-only STT usage reports
do not create token counts. TTS model children record the text after LiveKit's
configured text transforms. Media model inputs identify their `operation` as
`transcribe` or `speech`.


LiveKit turn diagnostics are included without an OpenTelemetry exporter. User
turns expose `contrib.livekit.end_of_turn_delay`,
`contrib.livekit.transcription_delay`, and
`contrib.livekit.on_user_turn_completed_delay` in seconds when available.
`livekit.eou_wait` shows the observed interval from speech end to an accepted
turn decision. User speech spans retain native `contrib.livekit.vad_silence_duration`
and `contrib.livekit.vad_inference_duration`, also in seconds. Missing native
timing stays absent. Model timing and usage continue to belong to model children.
23 changes: 23 additions & 0 deletions py/src/braintrust/integrations/livekit_agents/test_cascade.py
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ def upload(attachment):
llm.on("metrics_collected", model_metrics.append)
vad = silero.VAD.load()
heard = asyncio.Event()
eou_metrics = []
input_source = rtc.AudioSource(24000, 1)
track = rtc.LocalAudioTrack.create_audio_track("caller-microphone", input_source)
await rooms[1].local_participant.publish_track(
Expand All @@ -80,6 +81,10 @@ def conversation(event):
heard.set()

session.on("conversation_item_added", conversation)
session.on(
"metrics_collected",
lambda event: eou_metrics.append(event.metrics) if event.metrics.type == "eou_metrics" else None,
)
await session.start(
Agent(instructions="Reply with exactly: hi there", llm=llm), room=rooms[0], record=False
)
Expand Down Expand Up @@ -148,6 +153,24 @@ def conversation(event):
user = next(row for row in named(rows, "user_turn") if row.get("input"))
assert user["input"][0]["content"] == "Hello there."
assert user["metadata"]["audio.selections"]
if not preemptive:
assert len(eou_metrics) == 1
if eou_metrics:
reported = eou_metrics[0]
assert user["metadata"]["contrib.livekit.end_of_turn_delay"] == pytest.approx(reported.end_of_utterance_delay)
assert user["metadata"]["contrib.livekit.transcription_delay"] == pytest.approx(reported.transcription_delay)
assert (
user["metadata"]["contrib.livekit.on_user_turn_completed_delay"] == reported.on_user_turn_completed_delay
)
wait = named(rows, "livekit.eou_wait")[0]
assert wait["span_parents"] == [user["span_id"]]
assert wait["span_attributes"]["type"] == "task"
assert wait["metrics"]["end"] - wait["metrics"]["start"] == pytest.approx(
user["metadata"]["contrib.livekit.end_of_turn_delay"], abs=1e-6
)
for speech in named(rows, "user_speech"):
assert speech["metadata"]["contrib.livekit.vad_silence_duration"] >= 0
assert speech["metadata"]["contrib.livekit.vad_inference_duration"] >= 0
by_id = {row["span_id"]: row for row in rows}
for speech in named(rows, "user_speech"):
parent = by_id[speech["span_parents"][0]]
Expand Down
46 changes: 46 additions & 0 deletions py/src/braintrust/integrations/livekit_agents/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import contextlib
import inspect
import logging
import math
import re
import time
import weakref
Expand Down Expand Up @@ -85,6 +86,18 @@ def _operation_context(span: Span):
span.unset_current()


def _log_native_diagnostics(span, **fields):
# Call sites explicitly select native fields. Do not invent zeroes when
# LiveKit lacks a reliable timing anchor, or promote these to model metrics.
metadata = {
f"contrib.livekit.{name}": value
for name, value in fields.items()
if type(value) in (int, float) and math.isfinite(value) and value >= 0
}
if metadata:
span.log(metadata=metadata)


def link(target, relationship):
global _warned_links
trace_id, span_id = getattr(target, "root_span_id", ""), getattr(target, "span_id", "")
Expand Down Expand Up @@ -137,6 +150,7 @@ def __init__(self, native, record):
self.closed = False
self.closing = False
self._finishing = None
native.on("metrics_collected", self.metrics_collected)
if self.privacy.job is not None:
reference = weakref.ref(self)

Expand All @@ -152,6 +166,13 @@ async def shutdown():

self.privacy.job.add_shutdown_callback(shutdown)

def metrics_collected(self, event):
# The native callback report is emitted in _user_turn_completed_impl,
# where our existing context identifies the exact originating user turn.
user = _REPLY_TO.get()
if event.metrics.type == "eou_metrics" and user is not None:
_log_native_diagnostics(user, on_user_turn_completed_delay=event.metrics.on_user_turn_completed_delay)

def permits(self):
allowed = self.privacy.allows_content()
if not allowed:
Expand Down Expand Up @@ -318,6 +339,8 @@ async def _finish(self):
try:
await self.audio.finish()
finally:
if self.native is not None:
self.native.off("metrics_collected", self.metrics_collected)
self.native = None
self.privacy.session = None
self.privacy.job = None
Expand Down Expand Up @@ -500,6 +523,12 @@ def user_speech_end(wrapped, instance, args, kwargs):
owner = state_for(instance)
event = arguments(wrapped, args, kwargs).get("ev")
if owner is not None and owner.user_speech is not None:
if event is not None:
_log_native_diagnostics(
owner.user_speech,
vad_silence_duration=event.silence_duration,
vad_inference_duration=event.inference_duration,
)
end = time.time() - event.silence_duration - event.inference_duration if event is not None else time.time()
if event is not None:
end_ns = owner.audio.from_wall(end)
Expand Down Expand Up @@ -579,6 +608,23 @@ def end_of_turn(wrapped, instance, args, kwargs):
try:
committed = wrapped(*args, **kwargs)
if committed:
timing = info.metrics
_log_native_diagnostics(
user, end_of_turn_delay=timing.end_of_turn_delay, transcription_delay=timing.transcription_delay
)
stopped, delay = timing.stopped_speaking_at, timing.end_of_turn_delay
if (
type(stopped) in (int, float)
and type(delay) in (int, float)
and math.isfinite(stopped)
and math.isfinite(delay)
and stopped >= 0
and delay >= 0
):
wait = _new(
user, "livekit.eou_wait", start_time=stopped, metadata={"contrib.livekit.outcome": "committed"}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this feels like a more generic span we should spec out, eou_wait will apply to basically every other framework out there as well. But I'm fine with this shipping in current state to unblock as well.

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think that's a fine idea. We can migrate it to our schema later.

)
wait.end(stopped + delay)
owner.user = None
if info.user_turn_span_adopted:
owner.pending_users.add(user)
Expand Down
Loading