From 6951ad85d583409bf27b25f495be85ff42ef3acd Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 17 Nov 2026 22:47:02 -0500 Subject: [PATCH] feat(pipecat): trace gpt-live calls and record session audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Pipecat connects directly to GPT-Live, so OpenAI SDK instrumentation cannot capture its model execution. Add transcripts, recorded audio, usage, backend responses, and tool execution to the Pipecat trace: ```text session ├─ pipecat.live llm ├─ user_turn / agent_turn │ └─ transcript_received ├─ user_speech / agent_speech audio selections └─ llm_response task ├─ pipecat.live.backend_response llm └─ tool execution ``` Enable tracing and recording with `setup_pipecat(capture_audio_attachments=True)`. Framework frames define transcript groups; observed IDs define tool relationships. Preserve partial output on cancellation and keep model usage on its owning LLM span. Supports one Live session per pipeline with hosted Responses delegation. --- .../integrations/pipecat/__init__.py | 25 +- .../latest/test_live_observations.jsonl | 64 +++ .../integrations/pipecat/integration.py | 2 + .../integrations/pipecat/patchers.py | 86 +++- .../integrations/pipecat/test_pipecat.py | 294 +++++++++++- .../integrations/pipecat/tracing/_agent.py | 32 +- .../integrations/pipecat/tracing/_live.py | 444 ++++++++++++++++++ .../integrations/pipecat/tracing/observer.py | 75 ++- 8 files changed, 974 insertions(+), 48 deletions(-) create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl create mode 100644 py/src/braintrust/integrations/pipecat/tracing/_live.py diff --git a/py/src/braintrust/integrations/pipecat/__init__.py b/py/src/braintrust/integrations/pipecat/__init__.py index 6b077daa..c7c31422 100644 --- a/py/src/braintrust/integrations/pipecat/__init__.py +++ b/py/src/braintrust/integrations/pipecat/__init__.py @@ -24,25 +24,16 @@ def setup_pipecat( capture_agent_audio_attachments: bool | None = None, audio_recording_options: AudioRecordingOptions | None = None, ) -> bool: - """Set up Braintrust tracing for Pipecat ``PipelineWorker`` instances. + """Enable Braintrust tracing for newly created Pipecat ``PipelineWorker`` instances. - Setup instruments ``PipelineWorker`` so every newly constructed worker is - traced, the same as ``braintrust.auto_instrument()`` or - ``wrap_pipeline_worker()``. Pass ``capture_audio_attachments=True`` to attach - both user and agent audio, or configure them independently with + Set ``capture_audio_attachments=True`` to record both sides, or use ``capture_user_audio_attachments`` and ``capture_agent_audio_attachments``. - When arguments are omitted, the generic - environment variables ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` and - ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` apply. Explicit arguments - take precedence. - - Emits the audio v3 schema: a ``session`` with shared recordings, native - ``user_turn`` / ``agent_turn`` groups, speech, STT, LLM, tools and TTS. - Turn containers follow Pipecat's native boundaries. Agent recordings - contain transport-confirmed playback. - ``audio_recording_options`` configures segmentation, encoding and shutdown - using ``braintrust._audio.config.AudioRecordingOptions``. Recordings default - to Opus/Ogg with ``braintrust[audio]`` installed, otherwise raw PCM. + Omitted capture options use ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` + and ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` environment defaults. + Explicit arguments take precedence. + + Use ``audio_recording_options`` to configure recording format, segmentation, + and shutdown. """ set_default_observer_options( capture_audio_attachments=capture_audio_attachments, diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl new file mode 100644 index 00000000..4cc8b87b --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl @@ -0,0 +1,64 @@ +{"direction":"client","event":{"event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","type":"session.start","session":{"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]},"description":"Look up a demo order."}]}}}}} +{"direction":"server","event":{"event_id":"event_ElPVUiIYrZGMNlqJLFiFG","type":"session.started","client_event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]}}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":600,"end_ms":800,"delta":" Hi","event_id":"event_ElPVWWHlfukMfxwz618Rz"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1000,"end_ms":1200,"delta":", can you","event_id":"event_ElPVWoeOkGkKIKBVgdY5i"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1400,"end_ms":1600,"delta":" check","event_id":"event_ElPVXjuV7mEbPWbmd5P9r"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1600,"end_ms":1800,"delta":" the","event_id":"event_ElPVXFTRoQjDOBP1LlEls"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2000,"end_ms":2200,"delta":" status of","event_id":"event_ElPVXVGsDhgnQQ9OtnYEi"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2400,"end_ms":2600,"delta":" order","event_id":"event_ElPVYt0HmxVEv5CxInigK"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2600,"end_ms":2800,"delta":" demo","event_id":"event_ElPVYbFRHP7XQMPjWJMPF"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3400,"end_ms":3600,"delta":" one-zero-four","event_id":"event_ElPVZ52wTQKYBquK1OXrG"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3800,"end_ms":4000,"delta":"-two","event_id":"event_ElPVZ07JA6vf0lCqsIl9Z"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":4000,"end_ms":4200,"delta":" for me","event_id":"event_ElPVayzVz5Z3qTqYbCPTj"}} +{"direction":"server","event":{"type":"session.delegation.created","offset_ms":4000,"delegation":{"id":"item_ElPValIun0iiFZeK3vH3V","type":"delegation","response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","target":"responses"},"event_id":"event_ElPVaJSXQiPNp0Z0Zbv5t"}} +{"direction":"server","event":{"event_id":"event_ElPVa8w6ciMc8UdVenwku","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVajhc2ndB2mFvEoRo8","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"event_id":"event_ElPVa0BkWN8Ng98YBVgyP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"in_progress","arguments":"","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVaARgg3NqlQfyJzeKv","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"dBYZCdvQTKGArb","output_index":0,"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVaMvD2Y935CoXAYUqC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"order","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"SYKI7ZBw0MU","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVarxoCcxzRUhbjBqvC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"_id","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"XzU0Mg9SVfmc5","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVaFbiZMpGQgu3hZ1hC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"Qm2eKieFwRWrX","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVad1NxeGPNTi1g2tuq","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"demo","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"3I38JeIPwUAe","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVajin91jv8dq70TkDP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":" one","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"yAkBTo0JCQ9L","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVaWWowaT9lnEga5ejm","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-zero","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"jaS6KMVTmrF","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4400,"end_ms":4600,"delta":" Sure,","event_id":"event_ElPVaSQGfzFZCEatLEfSQ"}} +{"direction":"server","event":{"event_id":"event_ElPVaWPYYHSakH5cwvshA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-four","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"r9L1kQGfMCT","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVa5yxCYDy4X1IoZikk","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-two","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"1sc2EYMYNuS8","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPValeQiW62DmkQY8UK3","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"ywrVdquEX49McQ","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVaK6y8l1STM0wDSL2y","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.done","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVacIMXw3PzZyXXrhEB","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"completed","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":14}}} +{"direction":"client","event":{"event_id":"3b642bcb-fe88-4579-954b-ddb9880eaaa6","type":"response.item.create","item":{"type":"function_call_output","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","output":"\"The function `lookup_order` is not currently available.\""}}} +{"direction":"server","event":{"event_id":"event_ElPVaDCYiCRVSg52Cyz9A","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968178,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":813,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":836},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":15}}} +{"direction":"client","event":{"event_id":"7ae94f2a-3c67-44c1-9ee9-634da4bb4552","type":"response.create"}} +{"direction":"server","event":{"event_id":"event_ElPVaoEvAem5AAWJcHecy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVaHJ0doxl5xMgV84ie","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4800,"end_ms":5000,"delta":" one","event_id":"event_ElPVaISi6kc0Tl3WOIXaJ"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5000,"end_ms":5200,"delta":" moment","event_id":"event_ElPVbQgaKdets1lYHX4bV"}} +{"direction":"server","event":{"event_id":"event_ElPVbTgZ5lbR0ApghDO2N","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVbeBwMyUxT9B2GNVEy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.added","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVbVKt6gD7K22l5Hs6P","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"I","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Uin9ACIpZiskhN0","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVbdO25sEOmkCPzRshx","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" can","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"nLPBMZogyfYU","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVbXILZVqfGcZ7Jdfta","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"\u2019t","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Tj9PG9N5o066Nn","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVbzHhIAI7b619Ys1Be","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" look","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"m2YnsBIuTMs","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVb0SyXHpF40QYLPTi2","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" up","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"EUIFn4mTYOm6q","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVbyimQMpEIoINz3oTg","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"V9ftJtijasUo","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"event_id":"event_ElPVblJhPZs8uSQ6UA2rd","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" order","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"gqHyKjmM6T","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVbQfPkMUkCuJ1OeSnA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" right","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"uFjfB2AZ91","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPVbA0VV3QQWMGVtFYax","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" now","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"cLca2VTTDu1u","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVbnEDiWTJmTc6ERtII","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"SAkhNXWgmld9Pfv","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVb5JhvPA2NybeFOV6J","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"output_index":0,"sequence_number":14,"text":"I can\u2019t look up the order right now."}}} +{"direction":"server","event":{"event_id":"event_ElPVblv16txRZfO6LKcBA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."},"sequence_number":15}}} +{"direction":"server","event":{"event_id":"event_ElPVb93iTa5CFzhNrBzDz","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":16}}} +{"direction":"server","event":{"event_id":"event_ElPVbgyrNJux5a9qjkear","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968179,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":858,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":14,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":872},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":17}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5400,"end_ms":5600,"delta":" while I","event_id":"event_ElPVbwgjIJZXs7w85zo58"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5600,"end_ms":5800,"delta":" check","event_id":"event_ElPVblC6bB5SM0w8rf5av"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5800,"end_ms":6000,"delta":" that.","event_id":"event_ElPVbTj7q4QwvBIWSYP39"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":7800,"end_ms":8000,"delta":" Sorry,","event_id":"event_ElPVdeOzdi4jxkH6wKGPj"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8000,"end_ms":8200,"delta":" I","event_id":"event_ElPVe4U4kVNeVckkofX1E"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8200,"end_ms":8400,"delta":" can\u2019t","event_id":"event_ElPVely9H3CKKfrfBnYRx"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8600,"end_ms":8800,"delta":" look","event_id":"event_ElPVemy8xOZC9CX9lIiGL"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8800,"end_ms":9000,"delta":" it up","event_id":"event_ElPVeRGzzIweT3dX1OpJ8"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9000,"end_ms":9200,"delta":" right","event_id":"event_ElPVfryUNgBPSpSqOQ3S3"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9200,"end_ms":9400,"delta":" now.","event_id":"event_ElPVf6o5Pm0ILKBxygnAY"}} +{"direction":"client","event":{"event_id":"735fd94b-b073-4a40-8372-bd7500213598","type":"session.close"}} +{"direction":"server","event":{"event_id":"event_ElPVhGcRa2AcM4UssEkqq","type":"session.closed","reason":"close_requested","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]},"usage":{"seconds":11.0},"client_event_id":"735fd94b-b073-4a40-8372-bd7500213598"}} diff --git a/py/src/braintrust/integrations/pipecat/integration.py b/py/src/braintrust/integrations/pipecat/integration.py index 7b2c1476..dacd0d75 100644 --- a/py/src/braintrust/integrations/pipecat/integration.py +++ b/py/src/braintrust/integrations/pipecat/integration.py @@ -4,6 +4,7 @@ from .patchers import ( LLMServicePushFramePatcher, + OpenAILivePatcher, PipelineWorkerPatcher, SegmentedSTTPushSegmentFramesPatcher, STTServiceProcessGeneratorPatcher, @@ -22,4 +23,5 @@ class PipecatIntegration(BaseIntegration): LLMServicePushFramePatcher, STTServiceProcessGeneratorPatcher, SegmentedSTTPushSegmentFramesPatcher, + OpenAILivePatcher, ) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index ce9d9e75..87ec3ab2 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -1,5 +1,6 @@ """Pipecat integration patchers.""" +import inspect from contextvars import ContextVar from typing import Any @@ -7,6 +8,8 @@ from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher from .tracing import BraintrustPipecatObserver +from .tracing._common import LOG +from .tracing._live import is_live_service _DEFAULT_OBSERVER_OPTIONS: dict[str, Any] = { @@ -74,14 +77,18 @@ def _release_response_span(service: Any) -> None: async def traced_llm_push_frame(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: """Make an LLM service's llm_response span current while it makes the call. - Every LLM service pushes its response start frame, sends the provider - request and pushes its response end frame from the same task. Spans the + Cascade LLM services push their response start frame, send the provider + request and push their response end frame from the same task. Spans the provider integration starts in between nest under llm_response. """ observer = _ACTIVE_OBSERVER.get() frame = args[0] if args else kwargs.get("frame") kind = type(frame).__name__ - if observer is None or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame"): + if ( + observer is None + or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame") + or is_live_service(instance) + ): return await wrapped(*args, **kwargs) _release_response_span(instance) if kind == "LLMFullResponseEndFrame": @@ -166,6 +173,77 @@ class PipelineWorkerPatcher(CompositeFunctionWrapperPatcher): sub_patchers = (PipelineWorkerInitPatcher, PipelineWorkerRunPatcher) +async def traced_live_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + event = args[0] if args else kwargs.get("evt") + if observer is not None: + observer._live_event(instance, event) + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_event_processed(event) + return result + + +async def traced_live_client_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_client_event(args[0] if args else kwargs.get("event")) + return result + + +class OpenAILiveEventPatcher(FunctionWrapperPatcher): + name = "openai_live_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService._handle_server_event" + wrapper = staticmethod(traced_live_event) + + +class OpenAILiveClientEventPatcher(FunctionWrapperPatcher): + name = "openai_live_client_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService.send_client_event" + wrapper = staticmethod(traced_live_client_event) + + +class OpenAILivePatcher(CompositeFunctionWrapperPatcher): + name = "openai_live" + sub_patchers = (OpenAILiveEventPatcher, OpenAILiveClientEventPatcher) + + @classmethod + def applies(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + root = OpenAILiveEventPatcher.resolve_root(module, version, target=target) + return getattr(root, "OpenAILiveLLMService", None) is not None + + @classmethod + def _compatible(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + for patcher, argument in ((OpenAILiveEventPatcher, "evt"), (OpenAILiveClientEventPatcher, "event")): + method = patcher.resolve_target(module, version, target=target) + if not inspect.iscoroutinefunction(method): + return False + try: + inspect.signature(method).bind(None, **{argument: None}) + except (TypeError, ValueError): + return False + return True + + @classmethod + def is_patched(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + return cls._compatible(module, version, target=target) and super().is_patched(module, version, target=target) + + @classmethod + def patch(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + if not cls.applies(module, version, target=target): + return False + if not cls._compatible(module, version, target=target): + LOG.warning( + "GPT-Live model/turn tracing disabled: incompatible Pipecat Live event hooks. " + "Validated with pipecat-ai 1.16.0. Audio capture and other Pipecat tracing remain available." + ) + return False + return super().patch(module, version, target=target) + + def wrap_pipeline_worker( PipelineWorker: Any, *, @@ -199,4 +277,6 @@ def wrap_pipeline_worker( LLMServicePushFramePatcher.wrap_target(LLMService) STTServiceProcessGeneratorPatcher.wrap_target(STTService) SegmentedSTTPushSegmentFramesPatcher.wrap_target(SegmentedSTTService) + # Optional on older Pipecat versions; the patchers feature-detect the module. + OpenAILivePatcher.patch(None, None) return PipelineWorkerPatcher.wrap_target(PipelineWorker) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index bcae36d5..48750f01 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -858,21 +858,32 @@ async def test_tts_and_played_text_preserve_native_spacing(memory_logger, tracke @pytest.mark.asyncio @pytest.mark.parametrize("boundary", ["completed", "interrupted", "shutdown"]) async def test_playback_completion_distinguishes_shutdown_from_interruption(memory_logger, tracker_observer, boundary): + # Local observer bookkeeping: use native public events, without a provider + # or private tracker calls. Provider replay cannot control cleanup latency. frames = _frames() - session = tracker_observer._start_session() - with session.clock.event(0): - session.agent.start_tts(frames.TTSStartedFrame(), _openai_tts()) - session.agent.start_playback() - session.agent.played_audio(frames.OutputAudioRawFrame(bytes(320), 16000, 1)) - with session.clock.event(100_000_000): - # Synthesis can finish while its audio is still playing. - session.agent.stop_tts(frames.TTSStoppedFrame()) - with session.clock.event(200_000_000): - if boundary == "completed": - session.agent.stop_playback() - elif boundary == "interrupted": - session.agent.interrupt() - await tracker_observer._finish() + FramePushed = _import("pipecat.observers.base_observer.FramePushed") + FrameDirection = _import("pipecat.processors.frame_processor.FrameDirection") + FrameProcessor = _import("pipecat.processors.frame_processor.FrameProcessor") + Transport = _import("pipecat.transports.websocket.server.WebsocketServerTransport") + Params = _import("pipecat.transports.websocket.server.WebsocketServerParams") + transport = Transport(params=Params(audio_out_enabled=True)) + destination = FrameProcessor() + + async def push(frame, timestamp): + await tracker_observer.on_push_frame( + FramePushed(transport.output(), destination, frame, FrameDirection.DOWNSTREAM, timestamp) + ) + + await push(frames.BotStartedSpeakingFrame(), 0) + await push(frames.OutputAudioRawFrame(bytes(320), 16000, 1), 0) + if boundary == "completed": + await push(frames.BotStoppedSpeakingFrame(), 200_000_000) + elif boundary == "interrupted": + await push(frames.InterruptionFrame(), 200_000_000) + else: + # Teardown can outlast the final 10 ms of audio that was actually sent. + await asyncio.sleep(0.03) + await tracker_observer.cleanup() logs = memory_logger.pop() playback = _single_span(logs, "agent_speech") @@ -881,10 +892,8 @@ async def test_playback_completion_distinguishes_shutdown_from_interruption(memo assert playback["metadata"]["contrib.pipecat.incomplete"] is True else: assert "contrib.pipecat.incomplete" not in playback["metadata"] - assert playback["metadata"]["audio.selections"]["selection-0"] == { - "start_offset_ms": 0, - "end_offset_ms": 200, - } + window = playback["metadata"]["audio.selections"]["selection-0"] + assert window["end_offset_ms"] - window["start_offset_ms"] == (10 if boundary == "shutdown" else 200) _assert_all_ended(logs) @@ -908,7 +917,254 @@ async def test_recognition_scope_restores_context_after_errors_and_cancellation( assert logger.current_span() is recognition await asyncio.sleep(0) raise failure("operation stopped") - assert logger.current_span() is caller + assert logger.current_span() is caller + + +def test_live_incompatible_hook_reports_disabled_tracing(monkeypatch, caplog, memory_logger): + live = pytest.importorskip("pipecat.services.openai.live.llm") + # Simulate an upstream API removal. Provider cassettes cannot reproduce a + # missing method; assertions concern the public setup diagnostic only. + monkeypatch.delattr(live.OpenAILiveLLMService, "_handle_server_event") + with caplog.at_level(logging.WARNING): + _instrument() + assert "GPT-Live model/turn tracing disabled" in caplog.text + caplog.clear() + wrap_pipeline_worker(_import("pipecat.pipeline.worker.PipelineWorker")) + assert "GPT-Live model/turn tracing disabled" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("boundary", ["late_transcript", "cancel", "cancel_callback"]) +async def test_live_observations(memory_logger, vcr_cassette_dir, boundary): + live = pytest.importorskip("pipecat.services.openai.live.llm") + from pipecat.adapters.llm.schemas.function_schema import FunctionSchema + from pipecat.adapters.llm.schemas.tools_schema import ToolsSchema + from pipecat.pipeline.pipeline import Pipeline + from pipecat.processors.aggregators.llm_context import LLMContext + from pipecat.services.openai.responses.llm import OpenAIResponsesLLMService + from pipecat.workers.runner import WorkerRunner + from websockets.asyncio.server import serve + from websockets.exceptions import ConnectionClosed + + # Native WebSocket capture, including the real built-in missing-tool result. + # Audio payloads are omitted; shared recorder coverage lives in voice tests. + recording = [ + json.loads(line) for line in (Path(vcr_cassette_dir) / "test_live_observations.jsonl").read_text().splitlines() + ] + config = recording[0]["event"]["session"] + backend = config["delegation"]["responses"] + server_events = [r["event"] for r in recording if r["direction"] == "server"] + transcripts = [ + e for e in server_events if e["type"] in ("session.input_transcript.delta", "session.output_transcript.delta") + ] + late = transcripts[-1] + sent, outgoing = [], [] + ready, callback_cancelled = asyncio.Event(), asyncio.Event() + replay_errors = [] + + async def replay(websocket): + delayed_transcripts = [] + try: + for record in recording: + event = record["event"] + if record["direction"] == "client": + actual = json.loads(await websocket.recv()) + outgoing.append(actual) + assert actual["type"] == event["type"] + if actual["type"] == "session.start": + assert actual["session"]["instructions"] == event["session"]["instructions"] + if actual["type"] == "response.item.create": + assert actual["item"] == event["item"] + if actual["type"] == "session.close": + # Deliver a captured fragment late, while the real + # service awaits the server's close acknowledgment. + await websocket.send(json.dumps(late)) + sent.append(late) + else: + if event == late: + ready.set() + continue + if boundary == "cancel_callback" and event["type"] == "session.output_transcript.delta": + # Delay the independent speech stream until backend + # text has arrived, then cancel inside its public callback. + delayed_transcripts.append(event) + continue + if ( + boundary == "cancel_callback" + and event["type"] == "response.event" + and event["event"]["type"] == "response.output_text.done" + ): + await websocket.send(json.dumps(delayed_transcripts[0])) + sent.append(delayed_transcripts[0]) + await websocket.wait_closed() + return + await websocket.send(json.dumps(event)) + sent.append(event) + if boundary != "late_transcript" and event["type"] == "session.output_transcript.delta": + await websocket.wait_closed() + return + except ConnectionClosed: + if boundary == "late_transcript": + raise + except Exception as exc: + replay_errors.append(exc) + ready.set() + + _instrument() + async with serve(replay, "127.0.0.1", 0) as server: + port = server.sockets[0].getsockname()[1] + service = live.OpenAILiveLLMService( + api_key="unused-in-replay", + base_url=f"ws://127.0.0.1:{port}", + settings=live.OpenAILiveLLMService.Settings( + model=config["model"], system_instruction=config["instructions"] + ), + delegation=live.ResponsesDelegation( + settings=OpenAIResponsesLLMService.Settings( + model=backend["model"], system_instruction=backend["instructions"] + ) + ), + ) + context = LLMContext( + messages=[{"role": "system", "content": "Superseded context instructions."}] + if boundary == "cancel_callback" + else [], + tools=ToolsSchema( + standard_tools=[ + FunctionSchema( + name=t["name"], + description=t["description"], + properties=t["parameters"]["properties"], + required=t["parameters"]["required"], + ) + for t in backend["tools"] + ] + ), + ) + + # Pipecat executes its real missing-tool handler. No replacement tool, + # provider client, frame processor, or observer stands in for the SDK. + @service.event_handler("on_after_push_frame") + async def on_frame(_service, frame): + if isinstance(frame, _frames().LLMTextFrame) and boundary != "late_transcript": + ready.set() + if boundary == "cancel_callback": + try: + await asyncio.Event().wait() + finally: + callback_cancelled.set() + + worker = _make_worker(Pipeline([service])) + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + running = asyncio.create_task(runner.run()) + try: + await worker.queue_frame(_frames().LLMContextFrame(context)) + await asyncio.wait_for(ready.wait(), 10) + if replay_errors: + raise replay_errors[0] + if boundary == "late_transcript": + await worker.queue_frame(_frames().EndFrame()) + else: + await worker.cancel() + await asyncio.wait_for(running, 10) + finally: + if not running.done(): + await worker.cancel() + await asyncio.wait_for(running, 10) + assert not replay_errors + if boundary == "cancel_callback": + assert callback_cancelled.is_set() + + logs = memory_logger.pop() + session = _single_span(logs, "session") + model = _single_span(logs, "pipecat.live") + assert model["span_attributes"]["type"] == "llm" + assert model["metadata"]["audio.scope"] == "session" + assert model["span_parents"] == [session["span_id"]] + assert model["metadata"]["contrib.pipecat.live.finalization_confirmed"] is (boundary == "late_transcript") + assert all(row["metrics"].get("end") is not None for row in logs) + turns = _spans_named(logs, "user_turn") + _spans_named(logs, "agent_turn") + by_id = {row["span_id"]: row for row in turns} + received = _spans_named(logs, "transcript_received") + assert turns and len(turns) == len(received) + for role in ("user", "assistant"): + kind = "session.input_transcript.delta" if role == "user" else "session.output_transcript.delta" + observed = "".join(e["delta"] for e in sent if e["type"] == kind) + assert "".join(r["output"][0]["content"] for r in received if r["output"][0]["role"] == role) == observed + for row in received: + turn = by_id[row["metadata"]["turn.id"]] + assert row["span_parents"] == [turn["span_id"]] + assert turn["metadata"]["turn.id"] == turn["span_id"] + assert turn["span_attributes"]["name"] == ("user_turn" if row["output"][0]["role"] == "user" else "agent_turn") + assert turn["metrics"]["start"] <= row["metrics"]["start"] <= row["metrics"]["end"] <= turn["metrics"]["end"] + assert "audio.selections" not in row["metadata"] + assert "audio.selections" not in turn["metadata"] + assert "turn.reply_to" not in turn["metadata"] + # Cancellation, or a group opened by late text after native stop, must not + # be presented as a normally completed framework turn. + assert any(row["metadata"].get("turn.incomplete") for row in turns) + assert all("turn.id" not in row.get("metadata", {}) for row in logs if row not in turns + received) + responses = _spans_named(logs, "llm_response") + assert all(row["span_attributes"]["type"] == "task" and "tokens" not in row["metrics"] for row in responses) + assert all(not _link_targets(row, "consumes_tool_result") for row in responses) + llms = _spans_named(logs, "pipecat.live.backend_response") + if boundary == "late_transcript": + assert len(responses) == 2 + assert ( + responses[1]["metadata"]["contrib.pipecat.live.previous_response_id"] + == responses[0]["metadata"]["contrib.pipecat.live.response_id"] + ) + delegation = next(e for e in server_events if e["type"] == "session.delegation.created") + assert all( + row["metadata"]["contrib.pipecat.live.delegation_offset_ms"] == delegation["offset_ms"] + for row in responses + ) + tool = _single_span(logs, "lookup_order") + assert tool["span_parents"] == [responses[0]["span_id"]] + assert _link_targets(tool, "requested_by") + assert responses[0]["metrics"]["end"] >= tool["metrics"]["end"] + assert any(e["type"] == "response.item.create" for e in outgoing) + completions = [ + e["event"]["response"] + for e in server_events + if e["type"] == "response.event" and e["event"]["type"] == "response.completed" + ] + assert [r["metrics"]["tokens"] for r in llms] == [r["usage"]["total_tokens"] for r in completions] + completed_items = [ + e["event"]["item"] + for e in sent + if e["type"] == "response.event" and e["event"]["type"] == "response.output_item.done" + ] + assert [r["output"] for r in llms] == [[item] for item in completed_items] + closed = next(e for e in server_events if e["type"] == "session.closed") + assert model["metadata"]["contrib.pipecat.live.usage_seconds"] == closed["usage"]["seconds"] + else: + assert responses[-1]["metadata"].get("contrib.pipecat.incomplete") + assert llms[-1]["metadata"].get("contrib.pipecat.incomplete") + events = [e["event"] for e in sent if e["type"] == "response.event"] + item = [e["item"] for e in events if e["type"] == "response.output_item.added"][-1] + assert not any(e["type"] == "response.output_item.done" and e["item"]["id"] == item["id"] for e in events) + if boundary == "cancel": + arguments = "".join(e["delta"] for e in events if e["type"] == "response.function_call_arguments.delta") + assert arguments + expected = {**item, "arguments": arguments} + else: + part = next(e["part"] for e in events if e["type"] == "response.content_part.added") + text = "".join(e["delta"] for e in events if e["type"] == "response.output_text.delta") + assert text + expected = {**item, "content": [{**part, "text": text}]} + assert llms[-1]["output"] == responses[-1]["output"] == [expected] + started = next(e for e in server_events if e["type"] == "session.started") + assert [message for message in model["input"] if message.get("role") == "system"] == [ + {"role": "system", "content": started["session"]["instructions"]} + ] + by_id = {row["span_id"]: row for row in logs} + for row in logs: + for parent_id in row.get("span_parents") or []: + parent = by_id[parent_id] + assert parent["metrics"]["start"] <= row["metrics"]["start"] + assert row["metrics"]["end"] <= parent["metrics"]["end"], row["span_attributes"]["name"] # Keep provider tests last: enabling OpenAI patches the client for the process. diff --git a/py/src/braintrust/integrations/pipecat/tracing/_agent.py b/py/src/braintrust/integrations/pipecat/tracing/_agent.py index ca432979..f7a97c5d 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_agent.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_agent.py @@ -107,6 +107,7 @@ class _Playback: # span and set its start only after the transport confirms an audio frame. span: Span | None = None start_ms: float = 0.0 + audio_end_ms: float = 0.0 text: SpeechText = field(default_factory=SpeechText) generators: dict[str, _Synthesis] = field(default_factory=dict) @@ -133,6 +134,7 @@ class AgentTracker: def __init__(self, spans: SpanFactory, clock: SessionClock, recorder: AudioSessionRecorder) -> None: self._spans, self._clock, self._recorder = spans, clock, recorder + self.report_uninterrupted_playback = True self._turn: _AgentTurn | None = None self._response: _ModelResponse | None = None self._last_response: Span | None = None @@ -250,6 +252,12 @@ def record_metrics(self, frame: "MetricsFrame", processor: "FrameProcessor | Non # Tools + def link_tool_request(self, call_id: str, requester: Span) -> None: + """Associate a native Live backend request with its later tool frame.""" + # Pipecat owns the tool's execution and result submission as part of + # handling this response, even when execution uses another task. + self._requesters[call_id] = requester + def start_tool(self, frame: "FunctionCallInProgressFrame | FunctionCallResultFrame") -> None: key = frame.tool_call_id if key in self._tools or key in self._finished_tools: @@ -405,6 +413,9 @@ def played_audio(self, frame: "AudioRawFrame") -> None: playback = self._playback if playback is None: return + playback.audio_end_ms = max(playback.audio_end_ms, self._clock.now_ms()) + ( + len(frame.audio) * 1000 / (2 * frame.num_channels * frame.sample_rate) + ) span = playback.span if span is None: playback.start_ms = self._clock.now_ms() @@ -470,7 +481,7 @@ def interrupt(self, *, interrupted: bool = True, end_ms: float | None = None) -> if self._playback is not None: if self._playback.span is not None: self._playback.span.log(metadata={status: True}) - self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms) + self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms, shutdown=not interrupted) if self._response is not None: self._response.span.log(metadata={status: True}) self._end_response(self._response, end_ms=end_ms) @@ -529,17 +540,22 @@ def _end_synthesis(self, synthesis: _Synthesis, *, end_ms: float | None = None) synthesis.ended = True synthesis.text.parts.clear() - def _end_playback(self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None) -> None: + def _end_playback( + self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None, shutdown: bool = False + ) -> None: if playback.span is None: self._playback = None return end_ms = self._clock.now_ms() if end_ms is None else end_ms - playback.span.log( - metadata={ - SELECTIONS: {"selection-0": selection(playback.start_ms, end_ms)}, - "contrib.pipecat.interrupted": interrupted, - } - ) + # Cleanup may wait for remote closure after the transport stops sending. + # Keep the observed span lifetime, but don't select teardown as speech. + selection_end = min(end_ms, playback.audio_end_ms) if shutdown else end_ms + metadata: dict[str, Any] = {SELECTIONS: {"selection-0": selection(playback.start_ms, selection_end)}} + # Live handles automatic interruption remotely without notifying + # Pipecat. An explicit local interruption is still observable. + if interrupted or self.report_uninterrupted_playback: + metadata["contrib.pipecat.interrupted"] = interrupted + playback.span.log(metadata=metadata) if playback.text.parts: playback.span.log(output=[{"role": "assistant", "content": str(playback.text)}]) playback.span.end(end_time=self._clock.wall(end_ms)) diff --git a/py/src/braintrust/integrations/pipecat/tracing/_live.py b/py/src/braintrust/integrations/pipecat/tracing/_live.py new file mode 100644 index 00000000..cf9315af --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/tracing/_live.py @@ -0,0 +1,444 @@ +"""Prototype GPT-Live tracing: continuous conversation and hosted Responses. + +The voice service's LLM/TTS frames are transcript grouping notifications, not +provider calls. Public frames own transcript groups; native events own hosted +Responses. The ordinary observer still owns transport audio and speech. This +intentionally supports one Live service per pipeline and hosted Responses +delegation first. +""" + +import math +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any + +from braintrust._audio.clock import SessionClock +from braintrust._audio.schema import SOURCE_ID +from braintrust.logger import Span, SpanTypeAttribute + +from ._common import SpanFactory, metadata_from_processor, tools_schema_to_openai_tools + + +# Prototype pricing bridge until the backend prices duration-based models. +# Public USD/minute rate verified 2026-11-16. Do not price unknown models. +_LIVE_USD_PER_MINUTE = {"gpt-live-1": 0.05} +_LIVE_PRICING_SOURCE = "https://developers.openai.com/api/docs/models/gpt-live-1" + + +def is_live_service(processor: Any) -> bool: + return any(base.__name__ == "OpenAILiveLLMService" for base in type(processor).__mro__) + + +@dataclass +class _Transcript: + span: Span + text: list[str] = field(default_factory=list) + start_ms: int | None = None + end_ms: int | None = None + + +@dataclass +class _Response: + wrapper: Span + llm: Span + start_ms: float + output: dict[str, Any] = field(default_factory=dict) + first_content: bool = False + llm_finished: bool = False + processing_finished: bool = False + pending_calls: set[str] = field(default_factory=set) + pending_tool_spans: set[str] = field(default_factory=set) + + def record_output(self, event: dict[str, Any]) -> None: + kind = event.get("type") + if kind in ("response.output_item.added", "response.output_item.done"): + item = event.get("item") or {} + key = item.get("id") or str(event.get("output_index", len(self.output))) + # Deltas update our snapshot, never the event Pipecat is handling. + self.output[key] = deepcopy(item) + return + item = self.output.get(event.get("item_id") or str(event.get("output_index"))) + if item is None: + return + if kind == "response.function_call_arguments.delta": + item["arguments"] = item.get("arguments", "") + event["delta"] + elif kind == "response.function_call_arguments.done": + item["arguments"] = event["arguments"] + elif kind in ("response.content_part.added", "response.content_part.done"): + content = item.setdefault("content", []) + index = event["content_index"] + if index == len(content): + content.append(deepcopy(event["part"])) + elif 0 <= index < len(content): + content[index] = deepcopy(event["part"]) + elif kind in ("response.output_text.delta", "response.output_text.done"): + content = item.get("content", []) + index = event["content_index"] + if 0 <= index < len(content): + part = content[index] + part["text"] = part.get("text", "") + event["delta"] if kind.endswith(".delta") else event["text"] + + +class LiveTracker: + def __init__(self, session: Any, service: Any) -> None: + self.session, self.service = session, service + session.agent.report_uninterrupted_playback = False + # Native callbacks and frame callbacks run in different tasks. Do not + # share the frame observer's temporarily pinned event clock. + self.clock = SessionClock(session.recorder) + self.spans = SpanFactory(session.span, self.clock) + self.frame_spans = SpanFactory(session.span, session.clock) + self.origin_ms = self.clock.now_ms() + self.turns: dict[str, Span] = {} + self.transcripts: dict[str, _Transcript] = {} + self.responses: dict[str, _Response] = {} + self.delegation_responses: dict[str, str] = {} + self.delegation_offsets: dict[str, int] = {} + self.tool_responses: dict[str, str] = {} + self.closed = False + self.model_closed = False + self.inputs: list[Any] = [] + self.outputs: list[dict[str, str]] = [] + self.first_content = False + self.model_name: str | None = None + self.model = self.spans.start( + "pipecat.live", + type=SpanTypeAttribute.LLM, + metadata={**metadata_from_processor(service), "audio.scope": "session"}, + ) + session.span.log(metadata={"contrib.pipecat.mode": "live"}) + + def event(self, event: Any) -> None: + if self.closed: + return + kind = event.type + if kind == "session.started": + self.origin_ms = self.clock.now_ms() + resource = event.session + self.model_name = resource.model + metadata = { + "provider": "openai", + "model": resource.model, + "contrib.pipecat.live.session_id": resource.id, + } + self.model.log( + metadata={ + **metadata, + "contrib.pipecat.live.input_partial": True, + "contrib.pipecat.live.transcript_grouping": "pipecat_transcript_gap", + } + ) + self.session.span.log(metadata=metadata) + context = getattr(self.service, "_context", None) + self.inputs = list(context.get_messages()) if context is not None else [] + if resource.instructions: + # Pipecat consumes a leading system message as instructions; + # the server reports the effective value after settings override it. + if self.inputs and isinstance(self.inputs[0], dict) and self.inputs[0].get("role") == "system": + self.inputs.pop(0) + self.inputs.insert(0, {"role": "system", "content": resource.instructions}) + self.model.log(input=self.inputs) + elif kind == "session.output_audio.delta": + if event.delta: + self._first_content() + elif kind in ("session.input_transcript.delta", "session.output_transcript.delta"): + if kind == "session.output_transcript.delta" and event.delta: + self._first_content() + elif kind == "session.delegation.created": + delegation = event.delegation + self.delegation_offsets[delegation.id] = event.offset_ms + if delegation.response_id: + self.delegation_responses[delegation.id] = delegation.response_id + elif kind == "response.event": + self._response(event) + elif kind in ("session.usage.updated", "session.closed"): + usage = getattr(event, "usage", None) + if usage is not None and usage.seconds is not None: + self._usage(usage.seconds) + if kind == "session.closed": + self.model.log(metadata={"contrib.pipecat.live.close_reason": event.reason}) + # Frame observations are queued separately. Closing the remote + # model must not discard transcript frames already in flight. + self._end_model(confirmed=True) + elif kind == "error": + self.model.log(error=event.error.message) + + def _first_content(self) -> None: + if not self.first_content: + self.first_content = True + self.model.log(metrics={"time_to_first_token": (self.clock.now_ms() - self.origin_ms) / 1000}) + + def _usage(self, seconds: float) -> None: + if not math.isfinite(seconds) or seconds < 0: + return + # Each provider update is cumulative: replace the estimate, never sum + # updates or substitute local wall time for the provider's billed time. + metadata: dict[str, Any] = {"contrib.pipecat.live.usage_seconds": seconds} + rate = _LIVE_USD_PER_MINUTE.get(self.model_name or "") + if rate is not None: + metadata.update( + { + "contrib.pipecat.live.pricing": { + "usd_per_minute": rate, + "source": _LIVE_PRICING_SOURCE, + "as_of": "2026-11-16", + } + } + ) + self.model.log(metrics={"estimated_cost": seconds * rate / 60}) + self.model.log(metadata=metadata) + + def frame(self, frame: Any) -> None: + if self.closed: + return + kind = type(frame).__name__ + if kind == "ProposedUserStartedSpeakingFrame": + self._open_turn("user") + elif kind == "LLMFullResponseStartFrame": + self._open_turn("assistant") + elif kind in ("InterimTranscriptionFrame", "TranscriptionFrame"): + # User frames contain the accumulated group, not a text delta. + self._transcript("user", frame.text, replace=True, event=getattr(frame, "result", None)) + elif kind == "LLMTextFrame": + self._transcript("assistant", frame.text) + elif kind == "ProposedUserStoppedSpeakingFrame": + self._end_turn("user") + elif kind == "LLMFullResponseEndFrame": + self._end_turn("assistant") + + def _open_turn(self, role: str) -> None: + if role in self.turns: + return + turn = self.turns[role] = self.frame_spans.turn("user_turn" if role == "user" else "agent_turn") + turn.log(metadata={"contrib.pipecat.live.grouping": "pipecat_transcript_gap"}) + + def _transcript(self, role: str, text: str, *, replace: bool = False, event: Any = None) -> None: + if not text or role not in self.turns: + return + transcript = self.transcripts.get(role) + if transcript is None: + transcript = self.transcripts[role] = _Transcript( + self.frame_spans.start( + "transcript_received", + turn=self.turns[role], + metadata={ + SOURCE_ID: "session-input" if role == "user" else "agent-output", + "contrib.pipecat.live.grouping": "pipecat_transcript_gap", + }, + ) + ) + if replace: + transcript.text = [text] + else: + transcript.text.append(text) + # User interim frames carry the original event; assistant text frames + # do not. Retain provider timestamps only when attached to this frame. + start_ms, end_ms = getattr(event, "start_ms", None), getattr(event, "end_ms", None) + if start_ms is not None: + transcript.start_ms = start_ms if transcript.start_ms is None else min(transcript.start_ms, start_ms) + if end_ms is not None: + transcript.end_ms = end_ms if transcript.end_ms is None else max(transcript.end_ms, end_ms) + transcript.span.log(output=[{"role": role, "content": "".join(transcript.text)}]) + if transcript.start_ms is not None and transcript.end_ms is not None: + transcript.span.log( + metadata={ + "contrib.pipecat.live.start_ms": transcript.start_ms, + "contrib.pipecat.live.end_ms": transcript.end_ms, + } + ) + + def _end_turn(self, role: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + end_time = self.session.clock.wall(end_ms) + transcript = self.transcripts.pop(role, None) + if transcript is not None: + message = {"role": role, "content": "".join(transcript.text)} + if role == "assistant": + self.outputs.append(message) + self.model.log(output=self.outputs) + else: + self.inputs.append(message) + self.model.log(input=self.inputs) + transcript.span.end(end_time=end_time) + turn = self.turns.pop(role, None) + if turn is not None: + if incomplete: + turn.log(metadata={"turn.incomplete": True}) + turn.end(end_time=end_time) + + def client_event(self, event: Any) -> None: + if self.closed or event.type != "response.item.create": + return + item = event.item + kind = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + if kind == "function_call_output": + call_id = item.get("call_id") if isinstance(item, dict) else getattr(item, "call_id", None) + # Observed submission, not server acknowledgment: Live doesn't + # acknowledge these items. Never infer which continuation uses it. + self.tool_result_submitted(call_id) + + def _response(self, envelope: Any) -> None: + event = envelope.event + kind = event.get("type", "") + snapshot = event.get("response") or {} + delegation = envelope.delegation_id or "uncorrelated" + response_id = snapshot.get("id") or event.get("response_id") or self.delegation_responses.get(delegation) + if not response_id: + return + self.delegation_responses[delegation] = response_id + response = self.responses.get(response_id) + if response is None: + # Missing an observed start is preferable to inventing one. + if kind != "response.created": + return + settings = getattr(getattr(self.service, "_delegation", None), "settings", None) + metadata = { + "provider": "openai", + "model": snapshot.get("model") or getattr(settings, "model", None), + "contrib.pipecat.live.response_id": response_id, + "contrib.pipecat.live.delegation_id": envelope.delegation_id, + "contrib.pipecat.live.input_partial": True, + } + if snapshot.get("previous_response_id"): + metadata["contrib.pipecat.live.previous_response_id"] = snapshot["previous_response_id"] + if delegation in self.delegation_offsets: + # This locates delegation creation on the provider's session + # timeline, including for subsequent responses in that delegation. + metadata["contrib.pipecat.live.delegation_offset_ms"] = self.delegation_offsets[delegation] + context = getattr(self.service, "_context", None) + tools = tools_schema_to_openai_tools(getattr(context, "tools", None)) + if tools: + metadata["tools"] = tools + instruction = getattr(settings, "system_instruction", None) + inputs = [{"role": "system", "content": instruction}] if isinstance(instruction, str) else None + wrapper = self.spans.start("llm_response", input=inputs, metadata=metadata) + llm = self.spans.start( + "pipecat.live.backend_response", + parent=wrapper, + type=SpanTypeAttribute.LLM, + input=inputs, + metadata=metadata, + ) + response = self.responses[response_id] = _Response(wrapper, llm, self.clock.now_ms()) + response.record_output(event) + if not response.first_content and ( + (kind.endswith(".delta") and event.get("delta")) or kind == "response.output_item.done" + ): + response.first_content = True + response.llm.log(metrics={"time_to_first_token": (self.clock.now_ms() - response.start_ms) / 1000}) + if kind == "response.output_item.done": + item = event.get("item") or {} + if item.get("type") == "function_call" and item.get("status") == "completed" and item.get("call_id"): + call_id = item["call_id"] + response.pending_calls.add(call_id) + response.pending_tool_spans.add(call_id) + self.tool_responses[call_id] = response_id + self.session.agent.link_tool_request(call_id, response.wrapper) + elif kind in ("response.completed", "response.failed", "response.incomplete"): + if snapshot.get("model"): + response.llm.log(metadata={"model": snapshot["model"]}) + response.llm.log(metrics=_usage_metrics(snapshot.get("usage") or {})) + if kind != "response.completed": + error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} + response.llm.log(error=error) + response.wrapper.log(error=error) + # Provider inference ends here. Pipecat may still be executing + # tools and submitting their results as part of this response. + response.llm.log(output=list(response.output.values())) + response.llm.end(end_time=self.clock.wall()) + response.llm_finished = True + + def event_processed(self, envelope: Any) -> None: + if self.closed or envelope.type != "response.event": + return + event = envelope.event + if event.get("type") not in ("response.completed", "response.failed", "response.incomplete"): + return + response_id = (event.get("response") or {}).get("id") or self.delegation_responses.get(envelope.delegation_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.processing_finished = True + self._finish_if_ready(response_id, response) + + def tool_result_submitted(self, call_id: str) -> None: + response_id = self.tool_responses.get(call_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.pending_calls.discard(call_id) + if call_id not in response.pending_tool_spans: + self.tool_responses.pop(call_id, None) + self._finish_if_ready(response_id, response) + + def tool_finished(self, call_id: str) -> None: + response_id = self.tool_responses.get(call_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.pending_tool_spans.discard(call_id) + if call_id not in response.pending_calls: + self.tool_responses.pop(call_id, None) + self._finish_if_ready(response_id, response) + + def _finish_if_ready(self, response_id: str, response: _Response) -> None: + if ( + response.llm_finished + and response.processing_finished + and not response.pending_calls + and not response.pending_tool_spans + ): + self._end_response(response_id) + + def _end_response(self, response_id: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + response = self.responses.pop(response_id) + output = list(response.output.values()) + spans = (response.wrapper,) if response.llm_finished else (response.llm, response.wrapper) + for span in spans: + span.log(output=output) + if incomplete: + span.log(metadata={"contrib.pipecat.incomplete": True}) + span.end(end_time=self.clock.wall(end_ms)) + for call_id in response.pending_calls | response.pending_tool_spans: + self.tool_responses.pop(call_id, None) + + def _end_model(self, *, confirmed: bool = False, end_ms: float | None = None) -> None: + if self.model_closed: + return + self.model_closed = True + self.model.log(metadata={"contrib.pipecat.live.finalization_confirmed": confirmed}) + self.model.end(end_time=self.clock.wall(end_ms)) + + def close(self, end_ms: float) -> None: + if self.closed: + return + self.closed = True + for role in list(self.turns): + self._end_turn(role, incomplete=True, end_ms=end_ms) + for response_id in list(self.responses): + self._end_response(response_id, incomplete=True, end_ms=end_ms) + self._end_model(end_ms=end_ms) + + +def _usage_metrics(usage: dict[str, Any]) -> dict[str, int | float]: + metrics = { + target: usage[source] + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "tokens"), + ) + if isinstance(usage.get(source), (int, float)) + } + for source, key, target in ( + ("input_tokens_details", "cached_tokens", "prompt_cached_tokens"), + ("input_tokens_details", "audio_tokens", "prompt_audio_tokens"), + ("output_tokens_details", "reasoning_tokens", "completion_reasoning_tokens"), + ("output_tokens_details", "audio_tokens", "completion_audio_tokens"), + ): + value = (usage.get(source) or {}).get(key) + if isinstance(value, (int, float)): + metrics[target] = value + return metrics diff --git a/py/src/braintrust/integrations/pipecat/tracing/observer.py b/py/src/braintrust/integrations/pipecat/tracing/observer.py index fdad560d..03e134c0 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/observer.py +++ b/py/src/braintrust/integrations/pipecat/tracing/observer.py @@ -19,6 +19,7 @@ from ._agent import AgentTracker from ._common import LOG, SpanFactory, is_pipeline_sink, processor_kind, start_span from ._context import ContextLinker +from ._live import LiveTracker, is_live_service from ._user import UserTracker @@ -55,6 +56,8 @@ async def cleanup(self) -> None: _UPSTREAM_IGNORED_FRAME_TYPES = { "UserStartedSpeakingFrame", "UserStoppedSpeakingFrame", + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", "VADUserStartedSpeakingFrame", "VADUserStoppedSpeakingFrame", "BotStartedSpeakingFrame", @@ -81,6 +84,8 @@ class _Session: agent: AgentTracker # Without playback, a model response end is the agent completion boundary. has_output_transport: bool = False + live: LiveTracker | None = None + live_mode: bool = False class _FrameAdmission: @@ -204,6 +209,36 @@ async def cleanup(self) -> None: await self._finish() await super().cleanup() + def _live_event(self, service: Any, event: Any) -> None: + if self._phase is _SessionPhase.CLOSED: + return + try: + session = self._start_session() + if session.live is None: + # The model span represents an observed remote session, not + # whichever notification happens to arrive first. + if event.type != "session.started": + return + session.live_mode = True + session.live = LiveTracker(session, service) + session.live.event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live event", exc_info=True) + + def _live_event_processed(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.event_processed(event) + except Exception: + LOG.warning("Unable to finish Pipecat Live response processing", exc_info=True) + + def _live_client_event(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.client_event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live client event", exc_info=True) + async def _response_span(self, frame: "Frame", push: Callable[[], Awaitable[Any]]) -> tuple[Any, Span | None]: """Push a service's response start frame, then return its span. @@ -318,6 +353,7 @@ async def _finish(self) -> None: return end_ms = session.clock.now_ms() closers: tuple[Callable[[], None], ...] = ( + lambda: session.live.close(end_ms) if session.live is not None else None, lambda: session.user.close(end_ms), lambda: session.agent.interrupt(interrupted=False, end_ms=end_ms), lambda: session.span.end(end_time=session.clock.wall(end_ms)), @@ -340,6 +376,37 @@ async def _dispatch( self, session: _Session, frame: "Frame", processor: "FrameProcessor | None", *, played: bool ) -> None: kind = type(frame).__name__ + live_service = is_live_service(processor) + if live_service: + # Even without compatible event hooks, never reinterpret Live's + # compatibility frames as cascade inference/TTS or speech ownership. + session.live_mode = True + session.agent.report_uninterrupted_playback = False + if live_service and kind in { + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", + "InterimTranscriptionFrame", + "LLMFullResponseStartFrame", + "LLMFullResponseEndFrame", + "LLMTextFrame", + "TTSStartedFrame", + "TTSStoppedFrame", + "TTSTextFrame", + "TranscriptionFrame", + "MetricsFrame", + }: + # Live's public frames delimit transcript groups, not backend + # inference or generated TTS. Speech stays transport-owned. + if session.live is not None and self._admission.admit(frame): + session.live.frame(frame) + return + if session.live_mode and kind in { + "UserStartedSpeakingFrame", + "UserStoppedSpeakingFrame", + "LLMContextUserTurnFrame", + "LLMContextAssistantTurnFrame", + }: + return action = self._FRAME_ACTIONS.get(kind) if action is None or (kind in _TERMINAL_FRAME_TYPES and not is_pipeline_sink(processor)): return @@ -353,7 +420,7 @@ def _observe_playback(self, session: _Session, frame: "Frame") -> None: if isinstance(frame, frames.OutputAudioRawFrame): self._record_audio(session, "agent-output", frame) session.agent.played_audio(frame) - elif isinstance(frame, frames.TTSTextFrame): + elif isinstance(frame, frames.TTSTextFrame) and not session.live_mode: session.agent.played_text(frame) def _record_audio(self, session: _Session, source: _AudioSource, frame: "AudioRawFrame") -> None: @@ -470,11 +537,17 @@ def _on_tool_result( self, s: _Session, frame: "frames.FunctionCallResultFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.finish_tool(frame, s.linker) + if s.live is not None and getattr(getattr(frame, "properties", None), "is_final", True): + # Frame observation is queued separately from Live's result + # submission. Both must finish before the parent task closes. + s.live.tool_finished(frame.tool_call_id) def _on_tool_cancelled( self, s: _Session, frame: "frames.FunctionCallCancelFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.cancel_tool(frame.tool_call_id) + if s.live is not None: + s.live.tool_finished(frame.tool_call_id) def _on_tts_started( self, s: _Session, frame: "frames.TTSStartedFrame", processor: "FrameProcessor | None", _played: bool