From 4c220d426993234ded17e4ace7a4efcde33ef633 Mon Sep 17 00:00:00 2001 From: David Elner Date: Tue, 17 Nov 2026 22:47:02 -0500 Subject: [PATCH 1/4] feat(pipecat): trace gpt-live calls and record session audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Pipecat connects directly to GPT-Live, so OpenAI SDK instrumentation cannot capture its model execution. Add transcripts, recorded audio, usage, backend responses, and tool execution to the Pipecat trace: ```text session ├─ pipecat.live llm ├─ user_turn / agent_turn │ └─ transcript_received ├─ user_speech / agent_speech audio selections └─ llm_response task ├─ pipecat.live.backend_response llm └─ tool execution ``` Enable tracing and recording with `setup_pipecat(capture_audio_attachments=True)`. Framework frames define transcript groups; observed IDs define tool relationships. Preserve partial output on cancellation and keep model usage on its owning LLM span. Supports one Live session per pipeline with hosted Responses delegation. --- .../integrations/pipecat/__init__.py | 26 +- .../latest/test_live_observations.jsonl | 64 +++ .../integrations/pipecat/integration.py | 2 + .../integrations/pipecat/patchers.py | 86 +++- .../integrations/pipecat/test_pipecat.py | 301 +++++++++++- .../integrations/pipecat/tracing/_agent.py | 32 +- .../integrations/pipecat/tracing/_live.py | 444 ++++++++++++++++++ .../integrations/pipecat/tracing/observer.py | 75 ++- 8 files changed, 981 insertions(+), 49 deletions(-) create mode 100644 py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl create mode 100644 py/src/braintrust/integrations/pipecat/tracing/_live.py diff --git a/py/src/braintrust/integrations/pipecat/__init__.py b/py/src/braintrust/integrations/pipecat/__init__.py index d11c9265..0bc0ddc0 100644 --- a/py/src/braintrust/integrations/pipecat/__init__.py +++ b/py/src/braintrust/integrations/pipecat/__init__.py @@ -24,26 +24,16 @@ def setup_pipecat( capture_agent_audio_attachments: bool | None = None, audio_recording_options: AudioRecordingOptions | None = None, ) -> bool: - """Set up Braintrust tracing for Pipecat ``PipelineWorker`` instances. + """Enable Braintrust tracing for newly created Pipecat ``PipelineWorker`` instances. - Setup instruments ``PipelineWorker`` so every newly constructed worker is - traced, the same as ``braintrust.auto_instrument()`` or - ``wrap_pipeline_worker()``. Pass ``capture_audio_attachments=True`` to attach - both user and agent audio, or configure them independently with + Set ``capture_audio_attachments=True`` to record both sides, or use ``capture_user_audio_attachments`` and ``capture_agent_audio_attachments``. - When arguments are omitted, the generic - environment variables ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` and - ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` apply. Explicit arguments - take precedence. - - Emits the audio v3 schema: a ``session`` with shared recordings, native - ``user_turn`` / ``agent_turn`` groups, speech, STT, LLM, tools and TTS. - Turn containers follow Pipecat's native boundaries. Agent recordings - contain transport-confirmed playback. - ``audio_recording_options`` configures segmentation, encoding and shutdown - using ``braintrust.audio.AudioRecordingOptions``. Recordings default - to Opus/Ogg when PyAV with libopus is available, otherwise raw PCM. - ``braintrust[audio]`` installs a pinned PyAV on Python 3.12+. + Omitted capture options use ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` + and ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` environment defaults. + Explicit arguments take precedence. + + Use ``braintrust.audio.AudioRecordingOptions`` with ``audio_recording_options`` + to configure recording format, segmentation, and shutdown. """ set_default_observer_options( capture_audio_attachments=capture_audio_attachments, diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl new file mode 100644 index 00000000..4cc8b87b --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl @@ -0,0 +1,64 @@ +{"direction":"client","event":{"event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","type":"session.start","session":{"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]},"description":"Look up a demo order."}]}}}}} +{"direction":"server","event":{"event_id":"event_ElPVUiIYrZGMNlqJLFiFG","type":"session.started","client_event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]}}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":600,"end_ms":800,"delta":" Hi","event_id":"event_ElPVWWHlfukMfxwz618Rz"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1000,"end_ms":1200,"delta":", can you","event_id":"event_ElPVWoeOkGkKIKBVgdY5i"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1400,"end_ms":1600,"delta":" check","event_id":"event_ElPVXjuV7mEbPWbmd5P9r"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1600,"end_ms":1800,"delta":" the","event_id":"event_ElPVXFTRoQjDOBP1LlEls"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2000,"end_ms":2200,"delta":" status of","event_id":"event_ElPVXVGsDhgnQQ9OtnYEi"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2400,"end_ms":2600,"delta":" order","event_id":"event_ElPVYt0HmxVEv5CxInigK"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2600,"end_ms":2800,"delta":" demo","event_id":"event_ElPVYbFRHP7XQMPjWJMPF"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3400,"end_ms":3600,"delta":" one-zero-four","event_id":"event_ElPVZ52wTQKYBquK1OXrG"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3800,"end_ms":4000,"delta":"-two","event_id":"event_ElPVZ07JA6vf0lCqsIl9Z"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":4000,"end_ms":4200,"delta":" for me","event_id":"event_ElPVayzVz5Z3qTqYbCPTj"}} +{"direction":"server","event":{"type":"session.delegation.created","offset_ms":4000,"delegation":{"id":"item_ElPValIun0iiFZeK3vH3V","type":"delegation","response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","target":"responses"},"event_id":"event_ElPVaJSXQiPNp0Z0Zbv5t"}} +{"direction":"server","event":{"event_id":"event_ElPVa8w6ciMc8UdVenwku","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVajhc2ndB2mFvEoRo8","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"event_id":"event_ElPVa0BkWN8Ng98YBVgyP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"in_progress","arguments":"","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVaARgg3NqlQfyJzeKv","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"dBYZCdvQTKGArb","output_index":0,"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVaMvD2Y935CoXAYUqC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"order","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"SYKI7ZBw0MU","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVarxoCcxzRUhbjBqvC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"_id","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"XzU0Mg9SVfmc5","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVaFbiZMpGQgu3hZ1hC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"Qm2eKieFwRWrX","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVad1NxeGPNTi1g2tuq","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"demo","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"3I38JeIPwUAe","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVajin91jv8dq70TkDP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":" one","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"yAkBTo0JCQ9L","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVaWWowaT9lnEga5ejm","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-zero","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"jaS6KMVTmrF","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4400,"end_ms":4600,"delta":" Sure,","event_id":"event_ElPVaSQGfzFZCEatLEfSQ"}} +{"direction":"server","event":{"event_id":"event_ElPVaWPYYHSakH5cwvshA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-four","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"r9L1kQGfMCT","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVa5yxCYDy4X1IoZikk","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-two","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"1sc2EYMYNuS8","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPValeQiW62DmkQY8UK3","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"ywrVdquEX49McQ","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVaK6y8l1STM0wDSL2y","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.done","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVacIMXw3PzZyXXrhEB","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"completed","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":14}}} +{"direction":"client","event":{"event_id":"3b642bcb-fe88-4579-954b-ddb9880eaaa6","type":"response.item.create","item":{"type":"function_call_output","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","output":"\"The function `lookup_order` is not currently available.\""}}} +{"direction":"server","event":{"event_id":"event_ElPVaDCYiCRVSg52Cyz9A","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968178,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":813,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":836},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":15}}} +{"direction":"client","event":{"event_id":"7ae94f2a-3c67-44c1-9ee9-634da4bb4552","type":"response.create"}} +{"direction":"server","event":{"event_id":"event_ElPVaoEvAem5AAWJcHecy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVaHJ0doxl5xMgV84ie","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4800,"end_ms":5000,"delta":" one","event_id":"event_ElPVaISi6kc0Tl3WOIXaJ"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5000,"end_ms":5200,"delta":" moment","event_id":"event_ElPVbQgaKdets1lYHX4bV"}} +{"direction":"server","event":{"event_id":"event_ElPVbTgZ5lbR0ApghDO2N","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVbeBwMyUxT9B2GNVEy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.added","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVbVKt6gD7K22l5Hs6P","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"I","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Uin9ACIpZiskhN0","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVbdO25sEOmkCPzRshx","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" can","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"nLPBMZogyfYU","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVbXILZVqfGcZ7Jdfta","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"\u2019t","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Tj9PG9N5o066Nn","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVbzHhIAI7b619Ys1Be","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" look","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"m2YnsBIuTMs","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVb0SyXHpF40QYLPTi2","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" up","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"EUIFn4mTYOm6q","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVbyimQMpEIoINz3oTg","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"V9ftJtijasUo","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"event_id":"event_ElPVblJhPZs8uSQ6UA2rd","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" order","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"gqHyKjmM6T","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVbQfPkMUkCuJ1OeSnA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" right","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"uFjfB2AZ91","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPVbA0VV3QQWMGVtFYax","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" now","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"cLca2VTTDu1u","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVbnEDiWTJmTc6ERtII","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"SAkhNXWgmld9Pfv","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVb5JhvPA2NybeFOV6J","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"output_index":0,"sequence_number":14,"text":"I can\u2019t look up the order right now."}}} +{"direction":"server","event":{"event_id":"event_ElPVblv16txRZfO6LKcBA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."},"sequence_number":15}}} +{"direction":"server","event":{"event_id":"event_ElPVb93iTa5CFzhNrBzDz","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":16}}} +{"direction":"server","event":{"event_id":"event_ElPVbgyrNJux5a9qjkear","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968179,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":858,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":14,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":872},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":17}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5400,"end_ms":5600,"delta":" while I","event_id":"event_ElPVbwgjIJZXs7w85zo58"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5600,"end_ms":5800,"delta":" check","event_id":"event_ElPVblC6bB5SM0w8rf5av"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5800,"end_ms":6000,"delta":" that.","event_id":"event_ElPVbTj7q4QwvBIWSYP39"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":7800,"end_ms":8000,"delta":" Sorry,","event_id":"event_ElPVdeOzdi4jxkH6wKGPj"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8000,"end_ms":8200,"delta":" I","event_id":"event_ElPVe4U4kVNeVckkofX1E"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8200,"end_ms":8400,"delta":" can\u2019t","event_id":"event_ElPVely9H3CKKfrfBnYRx"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8600,"end_ms":8800,"delta":" look","event_id":"event_ElPVemy8xOZC9CX9lIiGL"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8800,"end_ms":9000,"delta":" it up","event_id":"event_ElPVeRGzzIweT3dX1OpJ8"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9000,"end_ms":9200,"delta":" right","event_id":"event_ElPVfryUNgBPSpSqOQ3S3"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9200,"end_ms":9400,"delta":" now.","event_id":"event_ElPVf6o5Pm0ILKBxygnAY"}} +{"direction":"client","event":{"event_id":"735fd94b-b073-4a40-8372-bd7500213598","type":"session.close"}} +{"direction":"server","event":{"event_id":"event_ElPVhGcRa2AcM4UssEkqq","type":"session.closed","reason":"close_requested","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]},"usage":{"seconds":11.0},"client_event_id":"735fd94b-b073-4a40-8372-bd7500213598"}} diff --git a/py/src/braintrust/integrations/pipecat/integration.py b/py/src/braintrust/integrations/pipecat/integration.py index 7b2c1476..dacd0d75 100644 --- a/py/src/braintrust/integrations/pipecat/integration.py +++ b/py/src/braintrust/integrations/pipecat/integration.py @@ -4,6 +4,7 @@ from .patchers import ( LLMServicePushFramePatcher, + OpenAILivePatcher, PipelineWorkerPatcher, SegmentedSTTPushSegmentFramesPatcher, STTServiceProcessGeneratorPatcher, @@ -22,4 +23,5 @@ class PipecatIntegration(BaseIntegration): LLMServicePushFramePatcher, STTServiceProcessGeneratorPatcher, SegmentedSTTPushSegmentFramesPatcher, + OpenAILivePatcher, ) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index ce9d9e75..87ec3ab2 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -1,5 +1,6 @@ """Pipecat integration patchers.""" +import inspect from contextvars import ContextVar from typing import Any @@ -7,6 +8,8 @@ from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher from .tracing import BraintrustPipecatObserver +from .tracing._common import LOG +from .tracing._live import is_live_service _DEFAULT_OBSERVER_OPTIONS: dict[str, Any] = { @@ -74,14 +77,18 @@ def _release_response_span(service: Any) -> None: async def traced_llm_push_frame(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: """Make an LLM service's llm_response span current while it makes the call. - Every LLM service pushes its response start frame, sends the provider - request and pushes its response end frame from the same task. Spans the + Cascade LLM services push their response start frame, send the provider + request and push their response end frame from the same task. Spans the provider integration starts in between nest under llm_response. """ observer = _ACTIVE_OBSERVER.get() frame = args[0] if args else kwargs.get("frame") kind = type(frame).__name__ - if observer is None or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame"): + if ( + observer is None + or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame") + or is_live_service(instance) + ): return await wrapped(*args, **kwargs) _release_response_span(instance) if kind == "LLMFullResponseEndFrame": @@ -166,6 +173,77 @@ class PipelineWorkerPatcher(CompositeFunctionWrapperPatcher): sub_patchers = (PipelineWorkerInitPatcher, PipelineWorkerRunPatcher) +async def traced_live_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + event = args[0] if args else kwargs.get("evt") + if observer is not None: + observer._live_event(instance, event) + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_event_processed(event) + return result + + +async def traced_live_client_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_client_event(args[0] if args else kwargs.get("event")) + return result + + +class OpenAILiveEventPatcher(FunctionWrapperPatcher): + name = "openai_live_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService._handle_server_event" + wrapper = staticmethod(traced_live_event) + + +class OpenAILiveClientEventPatcher(FunctionWrapperPatcher): + name = "openai_live_client_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService.send_client_event" + wrapper = staticmethod(traced_live_client_event) + + +class OpenAILivePatcher(CompositeFunctionWrapperPatcher): + name = "openai_live" + sub_patchers = (OpenAILiveEventPatcher, OpenAILiveClientEventPatcher) + + @classmethod + def applies(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + root = OpenAILiveEventPatcher.resolve_root(module, version, target=target) + return getattr(root, "OpenAILiveLLMService", None) is not None + + @classmethod + def _compatible(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + for patcher, argument in ((OpenAILiveEventPatcher, "evt"), (OpenAILiveClientEventPatcher, "event")): + method = patcher.resolve_target(module, version, target=target) + if not inspect.iscoroutinefunction(method): + return False + try: + inspect.signature(method).bind(None, **{argument: None}) + except (TypeError, ValueError): + return False + return True + + @classmethod + def is_patched(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + return cls._compatible(module, version, target=target) and super().is_patched(module, version, target=target) + + @classmethod + def patch(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + if not cls.applies(module, version, target=target): + return False + if not cls._compatible(module, version, target=target): + LOG.warning( + "GPT-Live model/turn tracing disabled: incompatible Pipecat Live event hooks. " + "Validated with pipecat-ai 1.16.0. Audio capture and other Pipecat tracing remain available." + ) + return False + return super().patch(module, version, target=target) + + def wrap_pipeline_worker( PipelineWorker: Any, *, @@ -199,4 +277,6 @@ def wrap_pipeline_worker( LLMServicePushFramePatcher.wrap_target(LLMService) STTServiceProcessGeneratorPatcher.wrap_target(STTService) SegmentedSTTPushSegmentFramesPatcher.wrap_target(SegmentedSTTService) + # Optional on older Pipecat versions; the patchers feature-detect the module. + OpenAILivePatcher.patch(None, None) return PipelineWorkerPatcher.wrap_target(PipelineWorker) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index bcae36d5..16e6cdc7 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -13,6 +13,7 @@ import inspect import json import logging +import math import os import socket import tempfile @@ -858,21 +859,32 @@ async def test_tts_and_played_text_preserve_native_spacing(memory_logger, tracke @pytest.mark.asyncio @pytest.mark.parametrize("boundary", ["completed", "interrupted", "shutdown"]) async def test_playback_completion_distinguishes_shutdown_from_interruption(memory_logger, tracker_observer, boundary): + # Local observer bookkeeping: use native public events, without a provider + # or private tracker calls. Provider replay cannot control cleanup latency. frames = _frames() - session = tracker_observer._start_session() - with session.clock.event(0): - session.agent.start_tts(frames.TTSStartedFrame(), _openai_tts()) - session.agent.start_playback() - session.agent.played_audio(frames.OutputAudioRawFrame(bytes(320), 16000, 1)) - with session.clock.event(100_000_000): - # Synthesis can finish while its audio is still playing. - session.agent.stop_tts(frames.TTSStoppedFrame()) - with session.clock.event(200_000_000): - if boundary == "completed": - session.agent.stop_playback() - elif boundary == "interrupted": - session.agent.interrupt() - await tracker_observer._finish() + FramePushed = _import("pipecat.observers.base_observer.FramePushed") + FrameDirection = _import("pipecat.processors.frame_processor.FrameDirection") + FrameProcessor = _import("pipecat.processors.frame_processor.FrameProcessor") + Transport = _import("pipecat.transports.websocket.server.WebsocketServerTransport") + Params = _import("pipecat.transports.websocket.server.WebsocketServerParams") + transport = Transport(params=Params(audio_out_enabled=True)) + destination = FrameProcessor() + + async def push(frame, timestamp): + await tracker_observer.on_push_frame( + FramePushed(transport.output(), destination, frame, FrameDirection.DOWNSTREAM, timestamp) + ) + + await push(frames.BotStartedSpeakingFrame(), 0) + await push(frames.OutputAudioRawFrame(bytes(320), 16000, 1), 0) + if boundary == "completed": + await push(frames.BotStoppedSpeakingFrame(), 200_000_000) + elif boundary == "interrupted": + await push(frames.InterruptionFrame(), 200_000_000) + else: + # Teardown can outlast the final 10 ms of audio that was actually sent. + await asyncio.sleep(0.03) + await tracker_observer.cleanup() logs = memory_logger.pop() playback = _single_span(logs, "agent_speech") @@ -881,10 +893,8 @@ async def test_playback_completion_distinguishes_shutdown_from_interruption(memo assert playback["metadata"]["contrib.pipecat.incomplete"] is True else: assert "contrib.pipecat.incomplete" not in playback["metadata"] - assert playback["metadata"]["audio.selections"]["selection-0"] == { - "start_offset_ms": 0, - "end_offset_ms": 200, - } + window = playback["metadata"]["audio.selections"]["selection-0"] + assert window["end_offset_ms"] - window["start_offset_ms"] == (10 if boundary == "shutdown" else 200) _assert_all_ended(logs) @@ -908,7 +918,260 @@ async def test_recognition_scope_restores_context_after_errors_and_cancellation( assert logger.current_span() is recognition await asyncio.sleep(0) raise failure("operation stopped") - assert logger.current_span() is caller + assert logger.current_span() is caller + + +def test_live_incompatible_hook_reports_disabled_tracing(monkeypatch, caplog, memory_logger): + live = pytest.importorskip("pipecat.services.openai.live.llm") + # Simulate an upstream API removal. Provider cassettes cannot reproduce a + # missing method; assertions concern the public setup diagnostic only. + monkeypatch.delattr(live.OpenAILiveLLMService, "_handle_server_event") + with caplog.at_level(logging.WARNING): + _instrument() + assert "GPT-Live model/turn tracing disabled" in caplog.text + caplog.clear() + wrap_pipeline_worker(_import("pipecat.pipeline.worker.PipelineWorker")) + assert "GPT-Live model/turn tracing disabled" in caplog.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("boundary", ["late_transcript", "cancel", "cancel_callback"]) +async def test_live_observations(memory_logger, vcr_cassette_dir, boundary): + live = pytest.importorskip("pipecat.services.openai.live.llm") + from websockets.asyncio.server import serve + from websockets.exceptions import ConnectionClosed + + FunctionSchema = _import("pipecat.adapters.llm.schemas.function_schema.FunctionSchema") + ToolsSchema = _import("pipecat.adapters.llm.schemas.tools_schema.ToolsSchema") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + LLMContext = _import("pipecat.processors.aggregators.llm_context.LLMContext") + OpenAIResponsesLLMService = _import("pipecat.services.openai.responses.llm.OpenAIResponsesLLMService") + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") + + # Native WebSocket capture, including the real built-in missing-tool result. + # Audio payloads are omitted; shared recorder coverage lives in voice tests. + recording = [ + json.loads(line) for line in (Path(vcr_cassette_dir) / "test_live_observations.jsonl").read_text().splitlines() + ] + config = recording[0]["event"]["session"] + backend = config["delegation"]["responses"] + server_events = [r["event"] for r in recording if r["direction"] == "server"] + transcripts = [ + e for e in server_events if e["type"] in ("session.input_transcript.delta", "session.output_transcript.delta") + ] + late = transcripts[-1] + sent, outgoing = [], [] + ready, callback_cancelled = asyncio.Event(), asyncio.Event() + replay_errors = [] + + async def replay(websocket): + delayed_transcripts = [] + try: + for record in recording: + event = record["event"] + if record["direction"] == "client": + actual = json.loads(await websocket.recv()) + outgoing.append(actual) + assert actual["type"] == event["type"] + if actual["type"] == "session.start": + assert actual["session"]["instructions"] == event["session"]["instructions"] + if actual["type"] == "response.item.create": + assert actual["item"] == event["item"] + if actual["type"] == "session.close": + # Deliver a captured fragment late, while the real + # service awaits the server's close acknowledgment. + await websocket.send(json.dumps(late)) + sent.append(late) + else: + if event == late: + ready.set() + continue + if boundary == "cancel_callback" and event["type"] == "session.output_transcript.delta": + # Delay the independent speech stream until backend + # text has arrived, then cancel inside its public callback. + delayed_transcripts.append(event) + continue + if ( + boundary == "cancel_callback" + and event["type"] == "response.event" + and event["event"]["type"] == "response.output_text.done" + ): + await websocket.send(json.dumps(delayed_transcripts[0])) + sent.append(delayed_transcripts[0]) + await websocket.wait_closed() + return + await websocket.send(json.dumps(event)) + sent.append(event) + if boundary != "late_transcript" and event["type"] == "session.output_transcript.delta": + await websocket.wait_closed() + return + except ConnectionClosed: + if boundary == "late_transcript": + raise + except Exception as exc: + replay_errors.append(exc) + ready.set() + + _instrument() + async with serve(replay, "127.0.0.1", 0) as server: + port = server.sockets[0].getsockname()[1] + service = live.OpenAILiveLLMService( + api_key="unused-in-replay", + base_url=f"ws://127.0.0.1:{port}", + settings=live.OpenAILiveLLMService.Settings( + model=config["model"], system_instruction=config["instructions"] + ), + delegation=live.ResponsesDelegation( + settings=OpenAIResponsesLLMService.Settings( + model=backend["model"], system_instruction=backend["instructions"] + ) + ), + ) + context = LLMContext( + messages=[{"role": "system", "content": "Superseded context instructions."}] + if boundary == "cancel_callback" + else [], + tools=ToolsSchema( + standard_tools=[ + FunctionSchema( + name=t["name"], + description=t["description"], + properties=t["parameters"]["properties"], + required=t["parameters"]["required"], + ) + for t in backend["tools"] + ] + ), + ) + + # Pipecat executes its real missing-tool handler. No replacement tool, + # provider client, frame processor, or observer stands in for the SDK. + @service.event_handler("on_after_push_frame") + async def on_frame(_service, frame): + if isinstance(frame, _frames().LLMTextFrame) and boundary != "late_transcript": + ready.set() + if boundary == "cancel_callback": + try: + await asyncio.Event().wait() + finally: + callback_cancelled.set() + + worker = _make_worker(Pipeline([service])) + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + running = asyncio.create_task(runner.run()) + try: + await worker.queue_frame(_frames().LLMContextFrame(context)) + await asyncio.wait_for(ready.wait(), 10) + if replay_errors: + raise replay_errors[0] + if boundary == "late_transcript": + await worker.queue_frame(_frames().EndFrame()) + else: + await worker.cancel() + await asyncio.wait_for(running, 10) + finally: + if not running.done(): + await worker.cancel() + await asyncio.wait_for(running, 10) + assert not replay_errors + if boundary == "cancel_callback": + assert callback_cancelled.is_set() + + logs = memory_logger.pop() + session = _single_span(logs, "session") + model = _single_span(logs, "pipecat.live") + assert model["span_attributes"]["type"] == "llm" + assert model["metadata"]["audio.scope"] == "session" + assert model["span_parents"] == [session["span_id"]] + assert model["metadata"]["contrib.pipecat.live.finalization_confirmed"] is (boundary == "late_transcript") + assert all(row["metrics"].get("end") is not None for row in logs) + turns = _spans_named(logs, "user_turn") + _spans_named(logs, "agent_turn") + by_id = {row["span_id"]: row for row in turns} + received = _spans_named(logs, "transcript_received") + assert turns and len(turns) == len(received) + for role in ("user", "assistant"): + kind = "session.input_transcript.delta" if role == "user" else "session.output_transcript.delta" + observed = "".join(e["delta"] for e in sent if e["type"] == kind) + assert "".join(r["output"][0]["content"] for r in received if r["output"][0]["role"] == role) == observed + for row in received: + turn = by_id[row["metadata"]["turn.id"]] + assert row["span_parents"] == [turn["span_id"]] + assert turn["metadata"]["turn.id"] == turn["span_id"] + assert turn["span_attributes"]["name"] == ("user_turn" if row["output"][0]["role"] == "user" else "agent_turn") + assert turn["metrics"]["start"] <= row["metrics"]["start"] <= row["metrics"]["end"] <= turn["metrics"]["end"] + assert "audio.selections" not in row["metadata"] + assert "audio.selections" not in turn["metadata"] + assert "turn.reply_to" not in turn["metadata"] + # Cancellation, or a group opened by late text after native stop, must not + # be presented as a normally completed framework turn. + assert any(row["metadata"].get("turn.incomplete") for row in turns) + assert all("turn.id" not in row.get("metadata", {}) for row in logs if row not in turns + received) + responses = _spans_named(logs, "llm_response") + assert all(row["span_attributes"]["type"] == "task" and "tokens" not in row["metrics"] for row in responses) + assert all(not _link_targets(row, "consumes_tool_result") for row in responses) + llms = _spans_named(logs, "pipecat.live.backend_response") + if boundary == "late_transcript": + assert len(responses) == 2 + assert ( + responses[1]["metadata"]["contrib.pipecat.live.previous_response_id"] + == responses[0]["metadata"]["contrib.pipecat.live.response_id"] + ) + delegation = next(e for e in server_events if e["type"] == "session.delegation.created") + assert all( + row["metadata"]["contrib.pipecat.live.delegation_offset_ms"] == delegation["offset_ms"] + for row in responses + ) + tool = _single_span(logs, "lookup_order") + assert tool["span_parents"] == [responses[0]["span_id"]] + assert _link_targets(tool, "requested_by") + assert responses[0]["metrics"]["end"] >= tool["metrics"]["end"] + assert any(e["type"] == "response.item.create" for e in outgoing) + completions = [ + e["event"]["response"] + for e in server_events + if e["type"] == "response.event" and e["event"]["type"] == "response.completed" + ] + assert [r["metrics"]["tokens"] for r in llms] == [r["usage"]["total_tokens"] for r in completions] + completed_items = [ + e["event"]["item"] + for e in sent + if e["type"] == "response.event" and e["event"]["type"] == "response.output_item.done" + ] + assert [r["output"] for r in llms] == [[item] for item in completed_items] + closed = next(e for e in server_events if e["type"] == "session.closed") + assert model["metadata"]["contrib.pipecat.live.usage_seconds"] == closed["usage"]["seconds"] + else: + assert responses[-1]["metadata"].get("contrib.pipecat.incomplete") + assert llms[-1]["metadata"].get("contrib.pipecat.incomplete") + events = [e["event"] for e in sent if e["type"] == "response.event"] + item = [e["item"] for e in events if e["type"] == "response.output_item.added"][-1] + assert not any(e["type"] == "response.output_item.done" and e["item"]["id"] == item["id"] for e in events) + if boundary == "cancel": + arguments = "".join(e["delta"] for e in events if e["type"] == "response.function_call_arguments.delta") + assert arguments + expected = {**item, "arguments": arguments} + else: + part = next(e["part"] for e in events if e["type"] == "response.content_part.added") + text = "".join(e["delta"] for e in events if e["type"] == "response.output_text.delta") + assert text + expected = {**item, "content": [{**part, "text": text}]} + assert llms[-1]["output"] == responses[-1]["output"] == [expected] + started = next(e for e in server_events if e["type"] == "session.started") + assert [message for message in model["input"] if message.get("role") == "system"] == [ + {"role": "system", "content": started["session"]["instructions"]} + ] + by_id = {row["span_id"]: row for row in logs} + for row in logs: + for parent_id in row.get("span_parents") or []: + parent = by_id[parent_id] + parent_start = parent["metrics"]["start"] + if parent_id == session["span_id"]: + # The session uses time.time(); recorder-derived child times + # can round the same Windows clock tick one float apart. + parent_start = math.nextafter(parent_start, -math.inf) + assert parent_start <= row["metrics"]["start"], row["span_attributes"]["name"] + assert row["metrics"]["end"] <= parent["metrics"]["end"], row["span_attributes"]["name"] # Keep provider tests last: enabling OpenAI patches the client for the process. diff --git a/py/src/braintrust/integrations/pipecat/tracing/_agent.py b/py/src/braintrust/integrations/pipecat/tracing/_agent.py index ca432979..f7a97c5d 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_agent.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_agent.py @@ -107,6 +107,7 @@ class _Playback: # span and set its start only after the transport confirms an audio frame. span: Span | None = None start_ms: float = 0.0 + audio_end_ms: float = 0.0 text: SpeechText = field(default_factory=SpeechText) generators: dict[str, _Synthesis] = field(default_factory=dict) @@ -133,6 +134,7 @@ class AgentTracker: def __init__(self, spans: SpanFactory, clock: SessionClock, recorder: AudioSessionRecorder) -> None: self._spans, self._clock, self._recorder = spans, clock, recorder + self.report_uninterrupted_playback = True self._turn: _AgentTurn | None = None self._response: _ModelResponse | None = None self._last_response: Span | None = None @@ -250,6 +252,12 @@ def record_metrics(self, frame: "MetricsFrame", processor: "FrameProcessor | Non # Tools + def link_tool_request(self, call_id: str, requester: Span) -> None: + """Associate a native Live backend request with its later tool frame.""" + # Pipecat owns the tool's execution and result submission as part of + # handling this response, even when execution uses another task. + self._requesters[call_id] = requester + def start_tool(self, frame: "FunctionCallInProgressFrame | FunctionCallResultFrame") -> None: key = frame.tool_call_id if key in self._tools or key in self._finished_tools: @@ -405,6 +413,9 @@ def played_audio(self, frame: "AudioRawFrame") -> None: playback = self._playback if playback is None: return + playback.audio_end_ms = max(playback.audio_end_ms, self._clock.now_ms()) + ( + len(frame.audio) * 1000 / (2 * frame.num_channels * frame.sample_rate) + ) span = playback.span if span is None: playback.start_ms = self._clock.now_ms() @@ -470,7 +481,7 @@ def interrupt(self, *, interrupted: bool = True, end_ms: float | None = None) -> if self._playback is not None: if self._playback.span is not None: self._playback.span.log(metadata={status: True}) - self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms) + self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms, shutdown=not interrupted) if self._response is not None: self._response.span.log(metadata={status: True}) self._end_response(self._response, end_ms=end_ms) @@ -529,17 +540,22 @@ def _end_synthesis(self, synthesis: _Synthesis, *, end_ms: float | None = None) synthesis.ended = True synthesis.text.parts.clear() - def _end_playback(self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None) -> None: + def _end_playback( + self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None, shutdown: bool = False + ) -> None: if playback.span is None: self._playback = None return end_ms = self._clock.now_ms() if end_ms is None else end_ms - playback.span.log( - metadata={ - SELECTIONS: {"selection-0": selection(playback.start_ms, end_ms)}, - "contrib.pipecat.interrupted": interrupted, - } - ) + # Cleanup may wait for remote closure after the transport stops sending. + # Keep the observed span lifetime, but don't select teardown as speech. + selection_end = min(end_ms, playback.audio_end_ms) if shutdown else end_ms + metadata: dict[str, Any] = {SELECTIONS: {"selection-0": selection(playback.start_ms, selection_end)}} + # Live handles automatic interruption remotely without notifying + # Pipecat. An explicit local interruption is still observable. + if interrupted or self.report_uninterrupted_playback: + metadata["contrib.pipecat.interrupted"] = interrupted + playback.span.log(metadata=metadata) if playback.text.parts: playback.span.log(output=[{"role": "assistant", "content": str(playback.text)}]) playback.span.end(end_time=self._clock.wall(end_ms)) diff --git a/py/src/braintrust/integrations/pipecat/tracing/_live.py b/py/src/braintrust/integrations/pipecat/tracing/_live.py new file mode 100644 index 00000000..cf9315af --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/tracing/_live.py @@ -0,0 +1,444 @@ +"""Prototype GPT-Live tracing: continuous conversation and hosted Responses. + +The voice service's LLM/TTS frames are transcript grouping notifications, not +provider calls. Public frames own transcript groups; native events own hosted +Responses. The ordinary observer still owns transport audio and speech. This +intentionally supports one Live service per pipeline and hosted Responses +delegation first. +""" + +import math +from copy import deepcopy +from dataclasses import dataclass, field +from typing import Any + +from braintrust._audio.clock import SessionClock +from braintrust._audio.schema import SOURCE_ID +from braintrust.logger import Span, SpanTypeAttribute + +from ._common import SpanFactory, metadata_from_processor, tools_schema_to_openai_tools + + +# Prototype pricing bridge until the backend prices duration-based models. +# Public USD/minute rate verified 2026-11-16. Do not price unknown models. +_LIVE_USD_PER_MINUTE = {"gpt-live-1": 0.05} +_LIVE_PRICING_SOURCE = "https://developers.openai.com/api/docs/models/gpt-live-1" + + +def is_live_service(processor: Any) -> bool: + return any(base.__name__ == "OpenAILiveLLMService" for base in type(processor).__mro__) + + +@dataclass +class _Transcript: + span: Span + text: list[str] = field(default_factory=list) + start_ms: int | None = None + end_ms: int | None = None + + +@dataclass +class _Response: + wrapper: Span + llm: Span + start_ms: float + output: dict[str, Any] = field(default_factory=dict) + first_content: bool = False + llm_finished: bool = False + processing_finished: bool = False + pending_calls: set[str] = field(default_factory=set) + pending_tool_spans: set[str] = field(default_factory=set) + + def record_output(self, event: dict[str, Any]) -> None: + kind = event.get("type") + if kind in ("response.output_item.added", "response.output_item.done"): + item = event.get("item") or {} + key = item.get("id") or str(event.get("output_index", len(self.output))) + # Deltas update our snapshot, never the event Pipecat is handling. + self.output[key] = deepcopy(item) + return + item = self.output.get(event.get("item_id") or str(event.get("output_index"))) + if item is None: + return + if kind == "response.function_call_arguments.delta": + item["arguments"] = item.get("arguments", "") + event["delta"] + elif kind == "response.function_call_arguments.done": + item["arguments"] = event["arguments"] + elif kind in ("response.content_part.added", "response.content_part.done"): + content = item.setdefault("content", []) + index = event["content_index"] + if index == len(content): + content.append(deepcopy(event["part"])) + elif 0 <= index < len(content): + content[index] = deepcopy(event["part"]) + elif kind in ("response.output_text.delta", "response.output_text.done"): + content = item.get("content", []) + index = event["content_index"] + if 0 <= index < len(content): + part = content[index] + part["text"] = part.get("text", "") + event["delta"] if kind.endswith(".delta") else event["text"] + + +class LiveTracker: + def __init__(self, session: Any, service: Any) -> None: + self.session, self.service = session, service + session.agent.report_uninterrupted_playback = False + # Native callbacks and frame callbacks run in different tasks. Do not + # share the frame observer's temporarily pinned event clock. + self.clock = SessionClock(session.recorder) + self.spans = SpanFactory(session.span, self.clock) + self.frame_spans = SpanFactory(session.span, session.clock) + self.origin_ms = self.clock.now_ms() + self.turns: dict[str, Span] = {} + self.transcripts: dict[str, _Transcript] = {} + self.responses: dict[str, _Response] = {} + self.delegation_responses: dict[str, str] = {} + self.delegation_offsets: dict[str, int] = {} + self.tool_responses: dict[str, str] = {} + self.closed = False + self.model_closed = False + self.inputs: list[Any] = [] + self.outputs: list[dict[str, str]] = [] + self.first_content = False + self.model_name: str | None = None + self.model = self.spans.start( + "pipecat.live", + type=SpanTypeAttribute.LLM, + metadata={**metadata_from_processor(service), "audio.scope": "session"}, + ) + session.span.log(metadata={"contrib.pipecat.mode": "live"}) + + def event(self, event: Any) -> None: + if self.closed: + return + kind = event.type + if kind == "session.started": + self.origin_ms = self.clock.now_ms() + resource = event.session + self.model_name = resource.model + metadata = { + "provider": "openai", + "model": resource.model, + "contrib.pipecat.live.session_id": resource.id, + } + self.model.log( + metadata={ + **metadata, + "contrib.pipecat.live.input_partial": True, + "contrib.pipecat.live.transcript_grouping": "pipecat_transcript_gap", + } + ) + self.session.span.log(metadata=metadata) + context = getattr(self.service, "_context", None) + self.inputs = list(context.get_messages()) if context is not None else [] + if resource.instructions: + # Pipecat consumes a leading system message as instructions; + # the server reports the effective value after settings override it. + if self.inputs and isinstance(self.inputs[0], dict) and self.inputs[0].get("role") == "system": + self.inputs.pop(0) + self.inputs.insert(0, {"role": "system", "content": resource.instructions}) + self.model.log(input=self.inputs) + elif kind == "session.output_audio.delta": + if event.delta: + self._first_content() + elif kind in ("session.input_transcript.delta", "session.output_transcript.delta"): + if kind == "session.output_transcript.delta" and event.delta: + self._first_content() + elif kind == "session.delegation.created": + delegation = event.delegation + self.delegation_offsets[delegation.id] = event.offset_ms + if delegation.response_id: + self.delegation_responses[delegation.id] = delegation.response_id + elif kind == "response.event": + self._response(event) + elif kind in ("session.usage.updated", "session.closed"): + usage = getattr(event, "usage", None) + if usage is not None and usage.seconds is not None: + self._usage(usage.seconds) + if kind == "session.closed": + self.model.log(metadata={"contrib.pipecat.live.close_reason": event.reason}) + # Frame observations are queued separately. Closing the remote + # model must not discard transcript frames already in flight. + self._end_model(confirmed=True) + elif kind == "error": + self.model.log(error=event.error.message) + + def _first_content(self) -> None: + if not self.first_content: + self.first_content = True + self.model.log(metrics={"time_to_first_token": (self.clock.now_ms() - self.origin_ms) / 1000}) + + def _usage(self, seconds: float) -> None: + if not math.isfinite(seconds) or seconds < 0: + return + # Each provider update is cumulative: replace the estimate, never sum + # updates or substitute local wall time for the provider's billed time. + metadata: dict[str, Any] = {"contrib.pipecat.live.usage_seconds": seconds} + rate = _LIVE_USD_PER_MINUTE.get(self.model_name or "") + if rate is not None: + metadata.update( + { + "contrib.pipecat.live.pricing": { + "usd_per_minute": rate, + "source": _LIVE_PRICING_SOURCE, + "as_of": "2026-11-16", + } + } + ) + self.model.log(metrics={"estimated_cost": seconds * rate / 60}) + self.model.log(metadata=metadata) + + def frame(self, frame: Any) -> None: + if self.closed: + return + kind = type(frame).__name__ + if kind == "ProposedUserStartedSpeakingFrame": + self._open_turn("user") + elif kind == "LLMFullResponseStartFrame": + self._open_turn("assistant") + elif kind in ("InterimTranscriptionFrame", "TranscriptionFrame"): + # User frames contain the accumulated group, not a text delta. + self._transcript("user", frame.text, replace=True, event=getattr(frame, "result", None)) + elif kind == "LLMTextFrame": + self._transcript("assistant", frame.text) + elif kind == "ProposedUserStoppedSpeakingFrame": + self._end_turn("user") + elif kind == "LLMFullResponseEndFrame": + self._end_turn("assistant") + + def _open_turn(self, role: str) -> None: + if role in self.turns: + return + turn = self.turns[role] = self.frame_spans.turn("user_turn" if role == "user" else "agent_turn") + turn.log(metadata={"contrib.pipecat.live.grouping": "pipecat_transcript_gap"}) + + def _transcript(self, role: str, text: str, *, replace: bool = False, event: Any = None) -> None: + if not text or role not in self.turns: + return + transcript = self.transcripts.get(role) + if transcript is None: + transcript = self.transcripts[role] = _Transcript( + self.frame_spans.start( + "transcript_received", + turn=self.turns[role], + metadata={ + SOURCE_ID: "session-input" if role == "user" else "agent-output", + "contrib.pipecat.live.grouping": "pipecat_transcript_gap", + }, + ) + ) + if replace: + transcript.text = [text] + else: + transcript.text.append(text) + # User interim frames carry the original event; assistant text frames + # do not. Retain provider timestamps only when attached to this frame. + start_ms, end_ms = getattr(event, "start_ms", None), getattr(event, "end_ms", None) + if start_ms is not None: + transcript.start_ms = start_ms if transcript.start_ms is None else min(transcript.start_ms, start_ms) + if end_ms is not None: + transcript.end_ms = end_ms if transcript.end_ms is None else max(transcript.end_ms, end_ms) + transcript.span.log(output=[{"role": role, "content": "".join(transcript.text)}]) + if transcript.start_ms is not None and transcript.end_ms is not None: + transcript.span.log( + metadata={ + "contrib.pipecat.live.start_ms": transcript.start_ms, + "contrib.pipecat.live.end_ms": transcript.end_ms, + } + ) + + def _end_turn(self, role: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + end_time = self.session.clock.wall(end_ms) + transcript = self.transcripts.pop(role, None) + if transcript is not None: + message = {"role": role, "content": "".join(transcript.text)} + if role == "assistant": + self.outputs.append(message) + self.model.log(output=self.outputs) + else: + self.inputs.append(message) + self.model.log(input=self.inputs) + transcript.span.end(end_time=end_time) + turn = self.turns.pop(role, None) + if turn is not None: + if incomplete: + turn.log(metadata={"turn.incomplete": True}) + turn.end(end_time=end_time) + + def client_event(self, event: Any) -> None: + if self.closed or event.type != "response.item.create": + return + item = event.item + kind = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + if kind == "function_call_output": + call_id = item.get("call_id") if isinstance(item, dict) else getattr(item, "call_id", None) + # Observed submission, not server acknowledgment: Live doesn't + # acknowledge these items. Never infer which continuation uses it. + self.tool_result_submitted(call_id) + + def _response(self, envelope: Any) -> None: + event = envelope.event + kind = event.get("type", "") + snapshot = event.get("response") or {} + delegation = envelope.delegation_id or "uncorrelated" + response_id = snapshot.get("id") or event.get("response_id") or self.delegation_responses.get(delegation) + if not response_id: + return + self.delegation_responses[delegation] = response_id + response = self.responses.get(response_id) + if response is None: + # Missing an observed start is preferable to inventing one. + if kind != "response.created": + return + settings = getattr(getattr(self.service, "_delegation", None), "settings", None) + metadata = { + "provider": "openai", + "model": snapshot.get("model") or getattr(settings, "model", None), + "contrib.pipecat.live.response_id": response_id, + "contrib.pipecat.live.delegation_id": envelope.delegation_id, + "contrib.pipecat.live.input_partial": True, + } + if snapshot.get("previous_response_id"): + metadata["contrib.pipecat.live.previous_response_id"] = snapshot["previous_response_id"] + if delegation in self.delegation_offsets: + # This locates delegation creation on the provider's session + # timeline, including for subsequent responses in that delegation. + metadata["contrib.pipecat.live.delegation_offset_ms"] = self.delegation_offsets[delegation] + context = getattr(self.service, "_context", None) + tools = tools_schema_to_openai_tools(getattr(context, "tools", None)) + if tools: + metadata["tools"] = tools + instruction = getattr(settings, "system_instruction", None) + inputs = [{"role": "system", "content": instruction}] if isinstance(instruction, str) else None + wrapper = self.spans.start("llm_response", input=inputs, metadata=metadata) + llm = self.spans.start( + "pipecat.live.backend_response", + parent=wrapper, + type=SpanTypeAttribute.LLM, + input=inputs, + metadata=metadata, + ) + response = self.responses[response_id] = _Response(wrapper, llm, self.clock.now_ms()) + response.record_output(event) + if not response.first_content and ( + (kind.endswith(".delta") and event.get("delta")) or kind == "response.output_item.done" + ): + response.first_content = True + response.llm.log(metrics={"time_to_first_token": (self.clock.now_ms() - response.start_ms) / 1000}) + if kind == "response.output_item.done": + item = event.get("item") or {} + if item.get("type") == "function_call" and item.get("status") == "completed" and item.get("call_id"): + call_id = item["call_id"] + response.pending_calls.add(call_id) + response.pending_tool_spans.add(call_id) + self.tool_responses[call_id] = response_id + self.session.agent.link_tool_request(call_id, response.wrapper) + elif kind in ("response.completed", "response.failed", "response.incomplete"): + if snapshot.get("model"): + response.llm.log(metadata={"model": snapshot["model"]}) + response.llm.log(metrics=_usage_metrics(snapshot.get("usage") or {})) + if kind != "response.completed": + error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} + response.llm.log(error=error) + response.wrapper.log(error=error) + # Provider inference ends here. Pipecat may still be executing + # tools and submitting their results as part of this response. + response.llm.log(output=list(response.output.values())) + response.llm.end(end_time=self.clock.wall()) + response.llm_finished = True + + def event_processed(self, envelope: Any) -> None: + if self.closed or envelope.type != "response.event": + return + event = envelope.event + if event.get("type") not in ("response.completed", "response.failed", "response.incomplete"): + return + response_id = (event.get("response") or {}).get("id") or self.delegation_responses.get(envelope.delegation_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.processing_finished = True + self._finish_if_ready(response_id, response) + + def tool_result_submitted(self, call_id: str) -> None: + response_id = self.tool_responses.get(call_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.pending_calls.discard(call_id) + if call_id not in response.pending_tool_spans: + self.tool_responses.pop(call_id, None) + self._finish_if_ready(response_id, response) + + def tool_finished(self, call_id: str) -> None: + response_id = self.tool_responses.get(call_id) + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None: + response.pending_tool_spans.discard(call_id) + if call_id not in response.pending_calls: + self.tool_responses.pop(call_id, None) + self._finish_if_ready(response_id, response) + + def _finish_if_ready(self, response_id: str, response: _Response) -> None: + if ( + response.llm_finished + and response.processing_finished + and not response.pending_calls + and not response.pending_tool_spans + ): + self._end_response(response_id) + + def _end_response(self, response_id: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + response = self.responses.pop(response_id) + output = list(response.output.values()) + spans = (response.wrapper,) if response.llm_finished else (response.llm, response.wrapper) + for span in spans: + span.log(output=output) + if incomplete: + span.log(metadata={"contrib.pipecat.incomplete": True}) + span.end(end_time=self.clock.wall(end_ms)) + for call_id in response.pending_calls | response.pending_tool_spans: + self.tool_responses.pop(call_id, None) + + def _end_model(self, *, confirmed: bool = False, end_ms: float | None = None) -> None: + if self.model_closed: + return + self.model_closed = True + self.model.log(metadata={"contrib.pipecat.live.finalization_confirmed": confirmed}) + self.model.end(end_time=self.clock.wall(end_ms)) + + def close(self, end_ms: float) -> None: + if self.closed: + return + self.closed = True + for role in list(self.turns): + self._end_turn(role, incomplete=True, end_ms=end_ms) + for response_id in list(self.responses): + self._end_response(response_id, incomplete=True, end_ms=end_ms) + self._end_model(end_ms=end_ms) + + +def _usage_metrics(usage: dict[str, Any]) -> dict[str, int | float]: + metrics = { + target: usage[source] + for source, target in ( + ("input_tokens", "prompt_tokens"), + ("output_tokens", "completion_tokens"), + ("total_tokens", "tokens"), + ) + if isinstance(usage.get(source), (int, float)) + } + for source, key, target in ( + ("input_tokens_details", "cached_tokens", "prompt_cached_tokens"), + ("input_tokens_details", "audio_tokens", "prompt_audio_tokens"), + ("output_tokens_details", "reasoning_tokens", "completion_reasoning_tokens"), + ("output_tokens_details", "audio_tokens", "completion_audio_tokens"), + ): + value = (usage.get(source) or {}).get(key) + if isinstance(value, (int, float)): + metrics[target] = value + return metrics diff --git a/py/src/braintrust/integrations/pipecat/tracing/observer.py b/py/src/braintrust/integrations/pipecat/tracing/observer.py index fdad560d..03e134c0 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/observer.py +++ b/py/src/braintrust/integrations/pipecat/tracing/observer.py @@ -19,6 +19,7 @@ from ._agent import AgentTracker from ._common import LOG, SpanFactory, is_pipeline_sink, processor_kind, start_span from ._context import ContextLinker +from ._live import LiveTracker, is_live_service from ._user import UserTracker @@ -55,6 +56,8 @@ async def cleanup(self) -> None: _UPSTREAM_IGNORED_FRAME_TYPES = { "UserStartedSpeakingFrame", "UserStoppedSpeakingFrame", + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", "VADUserStartedSpeakingFrame", "VADUserStoppedSpeakingFrame", "BotStartedSpeakingFrame", @@ -81,6 +84,8 @@ class _Session: agent: AgentTracker # Without playback, a model response end is the agent completion boundary. has_output_transport: bool = False + live: LiveTracker | None = None + live_mode: bool = False class _FrameAdmission: @@ -204,6 +209,36 @@ async def cleanup(self) -> None: await self._finish() await super().cleanup() + def _live_event(self, service: Any, event: Any) -> None: + if self._phase is _SessionPhase.CLOSED: + return + try: + session = self._start_session() + if session.live is None: + # The model span represents an observed remote session, not + # whichever notification happens to arrive first. + if event.type != "session.started": + return + session.live_mode = True + session.live = LiveTracker(session, service) + session.live.event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live event", exc_info=True) + + def _live_event_processed(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.event_processed(event) + except Exception: + LOG.warning("Unable to finish Pipecat Live response processing", exc_info=True) + + def _live_client_event(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.client_event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live client event", exc_info=True) + async def _response_span(self, frame: "Frame", push: Callable[[], Awaitable[Any]]) -> tuple[Any, Span | None]: """Push a service's response start frame, then return its span. @@ -318,6 +353,7 @@ async def _finish(self) -> None: return end_ms = session.clock.now_ms() closers: tuple[Callable[[], None], ...] = ( + lambda: session.live.close(end_ms) if session.live is not None else None, lambda: session.user.close(end_ms), lambda: session.agent.interrupt(interrupted=False, end_ms=end_ms), lambda: session.span.end(end_time=session.clock.wall(end_ms)), @@ -340,6 +376,37 @@ async def _dispatch( self, session: _Session, frame: "Frame", processor: "FrameProcessor | None", *, played: bool ) -> None: kind = type(frame).__name__ + live_service = is_live_service(processor) + if live_service: + # Even without compatible event hooks, never reinterpret Live's + # compatibility frames as cascade inference/TTS or speech ownership. + session.live_mode = True + session.agent.report_uninterrupted_playback = False + if live_service and kind in { + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", + "InterimTranscriptionFrame", + "LLMFullResponseStartFrame", + "LLMFullResponseEndFrame", + "LLMTextFrame", + "TTSStartedFrame", + "TTSStoppedFrame", + "TTSTextFrame", + "TranscriptionFrame", + "MetricsFrame", + }: + # Live's public frames delimit transcript groups, not backend + # inference or generated TTS. Speech stays transport-owned. + if session.live is not None and self._admission.admit(frame): + session.live.frame(frame) + return + if session.live_mode and kind in { + "UserStartedSpeakingFrame", + "UserStoppedSpeakingFrame", + "LLMContextUserTurnFrame", + "LLMContextAssistantTurnFrame", + }: + return action = self._FRAME_ACTIONS.get(kind) if action is None or (kind in _TERMINAL_FRAME_TYPES and not is_pipeline_sink(processor)): return @@ -353,7 +420,7 @@ def _observe_playback(self, session: _Session, frame: "Frame") -> None: if isinstance(frame, frames.OutputAudioRawFrame): self._record_audio(session, "agent-output", frame) session.agent.played_audio(frame) - elif isinstance(frame, frames.TTSTextFrame): + elif isinstance(frame, frames.TTSTextFrame) and not session.live_mode: session.agent.played_text(frame) def _record_audio(self, session: _Session, source: _AudioSource, frame: "AudioRawFrame") -> None: @@ -470,11 +537,17 @@ def _on_tool_result( self, s: _Session, frame: "frames.FunctionCallResultFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.finish_tool(frame, s.linker) + if s.live is not None and getattr(getattr(frame, "properties", None), "is_final", True): + # Frame observation is queued separately from Live's result + # submission. Both must finish before the parent task closes. + s.live.tool_finished(frame.tool_call_id) def _on_tool_cancelled( self, s: _Session, frame: "frames.FunctionCallCancelFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.cancel_tool(frame.tool_call_id) + if s.live is not None: + s.live.tool_finished(frame.tool_call_id) def _on_tts_started( self, s: _Session, frame: "frames.TTSStartedFrame", processor: "FrameProcessor | None", _played: bool From b9e391dd0e00e21581f7e87500169b255d50c65c Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 10:46:32 -0500 Subject: [PATCH 2/4] fix(pipecat): reuse live patching and usage parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Live used a separate OpenAI usage parser that dropped reported cache-write usage: ```text cache_write_tokens -> omitted ``` Reuse the existing OpenAI parser to preserve the standard metric: ```text cache_write_tokens -> prompt_cache_creation_tokens ``` Keep Live hook compatibility checks in the patcher applicability gate and let the shared composite handle patch state and idempotency. Cover either missing hook through the existing parametrized diagnostic test. Move the caller-context assertion outside pytest.raises so it runs after recognition errors and cancellation. --- .../integrations/pipecat/patchers.py | 32 +++++++------------ .../integrations/pipecat/test_pipecat.py | 10 ++++-- .../integrations/pipecat/tracing/_live.py | 25 ++------------- 3 files changed, 20 insertions(+), 47 deletions(-) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index 87ec3ab2..f5f7587a 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -213,35 +213,24 @@ class OpenAILivePatcher(CompositeFunctionWrapperPatcher): @classmethod def applies(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: root = OpenAILiveEventPatcher.resolve_root(module, version, target=target) - return getattr(root, "OpenAILiveLLMService", None) is not None - - @classmethod - def _compatible(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + if getattr(root, "OpenAILiveLLMService", None) is None: + return False + # Both hooks are needed to track a response through tool submission. + # Let the composite own patch markers and idempotency after this gate. for patcher, argument in ((OpenAILiveEventPatcher, "evt"), (OpenAILiveClientEventPatcher, "event")): method = patcher.resolve_target(module, version, target=target) - if not inspect.iscoroutinefunction(method): - return False try: - inspect.signature(method).bind(None, **{argument: None}) + if inspect.iscoroutinefunction(method): + inspect.signature(method).bind(None, **{argument: None}) + continue except (TypeError, ValueError): - return False - return True - - @classmethod - def is_patched(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: - return cls._compatible(module, version, target=target) and super().is_patched(module, version, target=target) - - @classmethod - def patch(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: - if not cls.applies(module, version, target=target): - return False - if not cls._compatible(module, version, target=target): + pass LOG.warning( "GPT-Live model/turn tracing disabled: incompatible Pipecat Live event hooks. " "Validated with pipecat-ai 1.16.0. Audio capture and other Pipecat tracing remain available." ) return False - return super().patch(module, version, target=target) + return super().applies(module, version, target=target) def wrap_pipeline_worker( @@ -278,5 +267,6 @@ def wrap_pipeline_worker( STTServiceProcessGeneratorPatcher.wrap_target(STTService) SegmentedSTTPushSegmentFramesPatcher.wrap_target(SegmentedSTTService) # Optional on older Pipecat versions; the patchers feature-detect the module. - OpenAILivePatcher.patch(None, None) + if OpenAILivePatcher.applies(None, None): + OpenAILivePatcher.patch(None, None) return PipelineWorkerPatcher.wrap_target(PipelineWorker) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 16e6cdc7..82f40c6c 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -918,14 +918,15 @@ async def test_recognition_scope_restores_context_after_errors_and_cancellation( assert logger.current_span() is recognition await asyncio.sleep(0) raise failure("operation stopped") - assert logger.current_span() is caller + assert logger.current_span() is caller -def test_live_incompatible_hook_reports_disabled_tracing(monkeypatch, caplog, memory_logger): +@pytest.mark.parametrize("hook", ["_handle_server_event", "send_client_event"]) +def test_live_incompatible_hook_reports_disabled_tracing(monkeypatch, caplog, memory_logger, hook): live = pytest.importorskip("pipecat.services.openai.live.llm") # Simulate an upstream API removal. Provider cassettes cannot reproduce a # missing method; assertions concern the public setup diagnostic only. - monkeypatch.delattr(live.OpenAILiveLLMService, "_handle_server_event") + monkeypatch.delattr(live.OpenAILiveLLMService, hook) with caplog.at_level(logging.WARNING): _instrument() assert "GPT-Live model/turn tracing disabled" in caplog.text @@ -1133,6 +1134,9 @@ async def on_frame(_service, frame): if e["type"] == "response.event" and e["event"]["type"] == "response.completed" ] assert [r["metrics"]["tokens"] for r in llms] == [r["usage"]["total_tokens"] for r in completions] + assert [r["metrics"]["prompt_cache_creation_tokens"] for r in llms] == [ + r["usage"]["input_tokens_details"]["cache_write_tokens"] for r in completions + ] completed_items = [ e["event"]["item"] for e in sent diff --git a/py/src/braintrust/integrations/pipecat/tracing/_live.py b/py/src/braintrust/integrations/pipecat/tracing/_live.py index cf9315af..3061d9f5 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_live.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_live.py @@ -14,6 +14,7 @@ from braintrust._audio.clock import SessionClock from braintrust._audio.schema import SOURCE_ID +from braintrust.integrations.openai.tracing import _parse_metrics_from_usage from braintrust.logger import Span, SpanTypeAttribute from ._common import SpanFactory, metadata_from_processor, tools_schema_to_openai_tools @@ -336,7 +337,7 @@ def _response(self, envelope: Any) -> None: elif kind in ("response.completed", "response.failed", "response.incomplete"): if snapshot.get("model"): response.llm.log(metadata={"model": snapshot["model"]}) - response.llm.log(metrics=_usage_metrics(snapshot.get("usage") or {})) + response.llm.log(metrics=_parse_metrics_from_usage(snapshot.get("usage"))) if kind != "response.completed": error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} response.llm.log(error=error) @@ -420,25 +421,3 @@ def close(self, end_ms: float) -> None: for response_id in list(self.responses): self._end_response(response_id, incomplete=True, end_ms=end_ms) self._end_model(end_ms=end_ms) - - -def _usage_metrics(usage: dict[str, Any]) -> dict[str, int | float]: - metrics = { - target: usage[source] - for source, target in ( - ("input_tokens", "prompt_tokens"), - ("output_tokens", "completion_tokens"), - ("total_tokens", "tokens"), - ) - if isinstance(usage.get(source), (int, float)) - } - for source, key, target in ( - ("input_tokens_details", "cached_tokens", "prompt_cached_tokens"), - ("input_tokens_details", "audio_tokens", "prompt_audio_tokens"), - ("output_tokens_details", "reasoning_tokens", "completion_reasoning_tokens"), - ("output_tokens_details", "audio_tokens", "completion_audio_tokens"), - ): - value = (usage.get(source) or {}).get(key) - if isinstance(value, (int, float)): - metrics[target] = value - return metrics From 647d4aa1920572e41331296f3f4b4345eee23f8e Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 11:13:35 -0500 Subject: [PATCH 3/4] ref(pipecat): favor pipeline coverage and quiet feature detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Several tests drove private trackers and hand-built frame timelines: ```text test -> tracker methods / observer callbacks ``` Remove those tests and their shared fixture. Keep the cassette-backed real pipeline coverage and extend the voice session assertion to check playback text against the synthesized text: ```text test -> real pipeline -> recorded provider -> exported spans ``` The exact synthetic timing cases are no longer checked independently. Keep the narrow span-export failure injection in its cassette-backed pipeline test because provider recordings cannot trigger an instrumentation failure. Remove the version-specific Live compatibility warning and its diagnostic cases. Incompatible or absent hooks now skip quietly, while the existing compatibility gate still requires both hooks. --- .../integrations/pipecat/patchers.py | 17 +- .../integrations/pipecat/test_pipecat.py | 166 +----------------- 2 files changed, 5 insertions(+), 178 deletions(-) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index f5f7587a..ffc2da7f 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -8,7 +8,6 @@ from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher from .tracing import BraintrustPipecatObserver -from .tracing._common import LOG from .tracing._live import is_live_service @@ -212,24 +211,16 @@ class OpenAILivePatcher(CompositeFunctionWrapperPatcher): @classmethod def applies(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: - root = OpenAILiveEventPatcher.resolve_root(module, version, target=target) - if getattr(root, "OpenAILiveLLMService", None) is None: - return False # Both hooks are needed to track a response through tool submission. # Let the composite own patch markers and idempotency after this gate. for patcher, argument in ((OpenAILiveEventPatcher, "evt"), (OpenAILiveClientEventPatcher, "event")): method = patcher.resolve_target(module, version, target=target) + if not inspect.iscoroutinefunction(method): + return False try: - if inspect.iscoroutinefunction(method): - inspect.signature(method).bind(None, **{argument: None}) - continue + inspect.signature(method).bind(None, **{argument: None}) except (TypeError, ValueError): - pass - LOG.warning( - "GPT-Live model/turn tracing disabled: incompatible Pipecat Live event hooks. " - "Validated with pipecat-ai 1.16.0. Audio capture and other Pipecat tracing remain available." - ) - return False + return False return super().applies(module, version, target=target) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 82f40c6c..0df8e5c5 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -24,7 +24,6 @@ from urllib.parse import urlsplit, urlunsplit import pytest -import pytest_asyncio from braintrust import SpanCustomizer, logger, set_span_customizers from braintrust.integrations.pipecat import setup_pipecat, wrap_pipeline_worker from braintrust.integrations.test_utils import verify_autoinstrument_script @@ -449,6 +448,7 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play assert _link_targets(playback, "generated_by") == [synthesis["span_id"]] assert playback["metadata"]["audio.speaker_ids"] == synthesis["metadata"]["audio.speaker_ids"] assert playback["metadata"]["contrib.pipecat.interrupted"] is False + assert playback["output"] == [{"role": "assistant", "content": synthesis["input"]["text"]}] assert "audio.participant_ids" in playback["metadata"] # Both directions land in shared recordings; playback is what was sent. @@ -771,170 +771,6 @@ def failing_end(span, *args, **kwargs): assert root["metadata"]["contrib.pipecat.terminal_frame"] == "EndFrame" -@pytest_asyncio.fixture -async def tracker_observer(memory_logger): - # These regressions concern local frame bookkeeping, not provider payloads. - # Use the real trackers, clock, recorder, services and native frame classes. - _frames() - from braintrust.integrations.pipecat.tracing.observer import BraintrustPipecatObserver - - observer = BraintrustPipecatObserver(capture_audio_attachments=False) - yield observer - await observer.cleanup() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("boundary", ["turn", "response", "shutdown"]) -async def test_queued_transcript_retains_text_and_receipt_time(memory_logger, tracker_observer, boundary): - frames = _frames() - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - stt = STT(api_key="unused") - session = tracker_observer._start_session() - with session.clock.event(0): - session.user.start_speech(session.clock.now_ms()) - session.user.consume_audio(stt) - with session.clock.event(100_000_000): - session.user.stop_speech() - session.user.transcribe(frames.TranscriptionFrame("Hello", "caller", "0"), stt) - received = session.clock.wall() - with session.clock.event(200_000_000): - if boundary == "turn": - session.user.start_turn() - elif boundary == "response": - session.user.flush_transcribed() - else: - session.user.close(session.clock.now_ms()) - recognition = _single_span(memory_logger.pop(), "stt") - assert recognition["output"] == [{"role": "user", "content": "Hello"}] - assert recognition["metrics"]["end"] == received - - -@pytest.mark.asyncio -async def test_untranscribed_stt_ends_with_its_user_turn(memory_logger, tracker_observer): - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - session = tracker_observer._start_session() - with session.clock.event(0): - session.user.start_turn() - session.user.start_speech(session.clock.now_ms()) - session.user.consume_audio(STT(api_key="unused")) - with session.clock.event(100_000_000): - session.user.stop_speech() - session.user.stop_turn() - with session.clock.event(10_000_000_000): - session.user.close(session.clock.now_ms()) - logs = memory_logger.pop() - recognition = _single_span(logs, "stt") - turn = _single_span(logs, "user_turn") - assert recognition["metrics"]["end"] == turn["metrics"]["end"] - assert recognition["metadata"]["contrib.pipecat.incomplete"] is True - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - "parts,spaced,expected", - [ - (["Hello.", "How are you?"], False, "Hello. How are you?"), - (["Hello", " there", "!"], True, "Hello there!"), - ], -) -async def test_tts_and_played_text_preserve_native_spacing(memory_logger, tracker_observer, parts, spaced, expected): - frames = _frames() - session = tracker_observer._start_session() - session.agent.start_tts(frames.TTSStartedFrame(), _openai_tts()) - session.agent.start_playback() - session.agent.played_audio(frames.OutputAudioRawFrame(bytes(320), 16000, 1)) - for text in parts: - frame = frames.TTSTextFrame(text, aggregated_by="sentence") - frame.includes_inter_frame_spaces = spaced - session.agent.tts_text(frame) - # Feed the same text through the actual observer playback callback. - tracker_observer._observe_playback(session, frame) - session.agent.stop_tts(frames.TTSStoppedFrame()) - session.agent.stop_playback() - logs = memory_logger.pop() - assert _single_span(logs, "tts")["input"]["text"] == expected - assert _single_span(logs, "agent_speech")["output"] == [{"role": "assistant", "content": expected}] - - -@pytest.mark.asyncio -@pytest.mark.parametrize("boundary", ["completed", "interrupted", "shutdown"]) -async def test_playback_completion_distinguishes_shutdown_from_interruption(memory_logger, tracker_observer, boundary): - # Local observer bookkeeping: use native public events, without a provider - # or private tracker calls. Provider replay cannot control cleanup latency. - frames = _frames() - FramePushed = _import("pipecat.observers.base_observer.FramePushed") - FrameDirection = _import("pipecat.processors.frame_processor.FrameDirection") - FrameProcessor = _import("pipecat.processors.frame_processor.FrameProcessor") - Transport = _import("pipecat.transports.websocket.server.WebsocketServerTransport") - Params = _import("pipecat.transports.websocket.server.WebsocketServerParams") - transport = Transport(params=Params(audio_out_enabled=True)) - destination = FrameProcessor() - - async def push(frame, timestamp): - await tracker_observer.on_push_frame( - FramePushed(transport.output(), destination, frame, FrameDirection.DOWNSTREAM, timestamp) - ) - - await push(frames.BotStartedSpeakingFrame(), 0) - await push(frames.OutputAudioRawFrame(bytes(320), 16000, 1), 0) - if boundary == "completed": - await push(frames.BotStoppedSpeakingFrame(), 200_000_000) - elif boundary == "interrupted": - await push(frames.InterruptionFrame(), 200_000_000) - else: - # Teardown can outlast the final 10 ms of audio that was actually sent. - await asyncio.sleep(0.03) - await tracker_observer.cleanup() - - logs = memory_logger.pop() - playback = _single_span(logs, "agent_speech") - assert playback["metadata"]["contrib.pipecat.interrupted"] is (boundary == "interrupted") - if boundary == "shutdown": - assert playback["metadata"]["contrib.pipecat.incomplete"] is True - else: - assert "contrib.pipecat.incomplete" not in playback["metadata"] - window = playback["metadata"]["audio.selections"]["selection-0"] - assert window["end_offset_ms"] - window["start_offset_ms"] == (10 if boundary == "shutdown" else 200) - _assert_all_ended(logs) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError]) -async def test_recognition_scope_restores_context_after_errors_and_cancellation(tracker_observer, failure): - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - stt = STT(api_key="unused") - session = tracker_observer._start_session() - session.user.start_turn() - session.user.consume_audio(stt) - - with logger.start_span(name="caller") as caller: - with pytest.raises(failure): - with tracker_observer._current_recognition(stt): - recognition = logger.current_span() - assert recognition is not caller - # Reentrant wrapping must also leave the outer scope intact. - with tracker_observer._current_recognition(stt): - assert logger.current_span() is recognition - assert logger.current_span() is recognition - await asyncio.sleep(0) - raise failure("operation stopped") - assert logger.current_span() is caller - - -@pytest.mark.parametrize("hook", ["_handle_server_event", "send_client_event"]) -def test_live_incompatible_hook_reports_disabled_tracing(monkeypatch, caplog, memory_logger, hook): - live = pytest.importorskip("pipecat.services.openai.live.llm") - # Simulate an upstream API removal. Provider cassettes cannot reproduce a - # missing method; assertions concern the public setup diagnostic only. - monkeypatch.delattr(live.OpenAILiveLLMService, hook) - with caplog.at_level(logging.WARNING): - _instrument() - assert "GPT-Live model/turn tracing disabled" in caplog.text - caplog.clear() - wrap_pipeline_worker(_import("pipecat.pipeline.worker.PipelineWorker")) - assert "GPT-Live model/turn tracing disabled" in caplog.text - - @pytest.mark.asyncio @pytest.mark.parametrize("boundary", ["late_transcript", "cancel", "cancel_callback"]) async def test_live_observations(memory_logger, vcr_cassette_dir, boundary): From ee6e78d9c64406cbd80d1df2ae158db49f1badb3 Mon Sep 17 00:00:00 2001 From: Abhijeet Prasad Date: Wed, 18 Nov 2026 11:32:08 -0500 Subject: [PATCH 4/4] fix(pipecat): clarify live lifecycles and release retained state MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit > 🤖 **This text was AI-generated.** Live lifecycle rules were spread across independent flags and parallel maps. Delegation IDs accumulated for the session, and a retained tracker kept its conversation buffers and routing history after shutdown: ```text closed / model_closed llm_finished / processing_finished ``` Represent session and response lifecycles with explicit phases: ```text session: active -> draining -> closed response: streaming -> processing -> waiting for tools -> removed ``` Keep each turn with its transcript and each pending tool's two completion signals in one record. Combine delegation routing into one recent-history cache using the existing retention limit, keeping active routes pinned. Clear tracker-owned history and routing caches when pipeline cleanup ends. Extend the existing recorded pipeline cases to check cleanup while retaining the tracker. Full conversation transcripts and unfinished work still consume memory until completion or shutdown; captured content is not truncated. --- .../integrations/pipecat/test_pipecat.py | 5 + .../integrations/pipecat/tracing/_live.py | 281 +++++++++++------- 2 files changed, 175 insertions(+), 111 deletions(-) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index 0df8e5c5..8d9fd2e5 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -902,6 +902,8 @@ async def on_frame(_service, frame): await asyncio.wait_for(ready.wait(), 10) if replay_errors: raise replay_errors[0] + tracker = worker._braintrust_pipecat_observer._session.live + assert tracker is not None if boundary == "late_transcript": await worker.queue_frame(_frames().EndFrame()) else: @@ -912,6 +914,9 @@ async def on_frame(_service, frame): await worker.cancel() await asyncio.wait_for(running, 10) assert not replay_errors + # Cleanup releases the tracker's own payload buffers and correlation + # caches, even if a caller retains the tracker after pipeline shutdown. + assert not {name: len(value) for name, value in vars(tracker).items() if isinstance(value, (dict, list)) and value} if boundary == "cancel_callback": assert callback_cancelled.is_set() diff --git a/py/src/braintrust/integrations/pipecat/tracing/_live.py b/py/src/braintrust/integrations/pipecat/tracing/_live.py index 3061d9f5..d313ff84 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_live.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_live.py @@ -8,16 +8,18 @@ """ import math +from collections import OrderedDict from copy import deepcopy from dataclasses import dataclass, field -from typing import Any +from enum import Enum, auto +from typing import Any, Literal from braintrust._audio.clock import SessionClock from braintrust._audio.schema import SOURCE_ID from braintrust.integrations.openai.tracing import _parse_metrics_from_usage from braintrust.logger import Span, SpanTypeAttribute -from ._common import SpanFactory, metadata_from_processor, tools_schema_to_openai_tools +from ._common import MAX_PENDING, SpanFactory, metadata_from_processor, tools_schema_to_openai_tools # Prototype pricing bridge until the backend prices duration-based models. @@ -30,14 +32,45 @@ def is_live_service(processor: Any) -> bool: return any(base.__name__ == "OpenAILiveLLMService" for base in type(processor).__mro__) +class _LivePhase(Enum): + ACTIVE = auto() + DRAINING = auto() # Remote model ended; queued transcript frames may still arrive. + CLOSED = auto() # Pipeline cleanup finished. + + +class _ResponsePhase(Enum): + STREAMING = auto() + PROCESSING = auto() # Inference ended; Pipecat is handling its terminal event. + WAITING_FOR_TOOLS = auto() + + +_Role = Literal["user", "assistant"] +_ToolSignal = Literal["submission", "span"] + + @dataclass -class _Transcript: +class _Turn: span: Span + transcript: Span | None = None text: list[str] = field(default_factory=list) start_ms: int | None = None end_ms: int | None = None +@dataclass +class _Delegation: + response_id: str | None = None + offset_ms: int | None = None + + +@dataclass +class _PendingTool: + response_id: str + # Submission and observation run in different tasks and can arrive in + # either order. Neither alone proves that the response task can close. + waiting_for: set[_ToolSignal] = field(default_factory=lambda: {"submission", "span"}) + + @dataclass class _Response: wrapper: Span @@ -45,10 +78,26 @@ class _Response: start_ms: float output: dict[str, Any] = field(default_factory=dict) first_content: bool = False - llm_finished: bool = False - processing_finished: bool = False - pending_calls: set[str] = field(default_factory=set) - pending_tool_spans: set[str] = field(default_factory=set) + phase: _ResponsePhase = _ResponsePhase.STREAMING + pending_tools: set[str] = field(default_factory=set) + + @property + def ready(self) -> bool: + return self.phase is _ResponsePhase.WAITING_FOR_TOOLS and not self.pending_tools + + def finish_inference(self, kind: str, snapshot: dict[str, Any], end_time: float) -> None: + if self.phase is not _ResponsePhase.STREAMING: + return + if snapshot.get("model"): + self.llm.log(metadata={"model": snapshot["model"]}) + self.llm.log(metrics=_parse_metrics_from_usage(snapshot.get("usage"))) + if kind != "response.completed": + error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} + self.llm.log(error=error) + self.wrapper.log(error=error) + self.llm.log(output=list(self.output.values())) + self.llm.end(end_time=end_time) + self.phase = _ResponsePhase.PROCESSING def record_output(self, event: dict[str, Any]) -> None: kind = event.get("type") @@ -81,6 +130,19 @@ def record_output(self, event: dict[str, Any]) -> None: class LiveTracker: + """Observe two independent lifecycles without making either one current. + + The remote model goes ACTIVE -> DRAINING on session.closed; only pipeline + cleanup makes it CLOSED, after queued transcript frames have drained. + + Each hosted response goes STREAMING -> PROCESSING -> WAITING_FOR_TOOLS, + then is removed once both tool submissions and tool spans have finished. + Cleanup ends any remaining work as incomplete. Response/tool indexes own + active work, and at most two transcript turns are open. Delegation history + keeps recent IDs/offsets plus active responses' routing. Full session + transcripts grow until cleanup by design. + """ + def __init__(self, session: Any, service: Any) -> None: self.session, self.service = session, service session.agent.report_uninterrupted_playback = False @@ -90,14 +152,11 @@ def __init__(self, session: Any, service: Any) -> None: self.spans = SpanFactory(session.span, self.clock) self.frame_spans = SpanFactory(session.span, session.clock) self.origin_ms = self.clock.now_ms() - self.turns: dict[str, Span] = {} - self.transcripts: dict[str, _Transcript] = {} + self.turns: dict[_Role, _Turn] = {} self.responses: dict[str, _Response] = {} - self.delegation_responses: dict[str, str] = {} - self.delegation_offsets: dict[str, int] = {} - self.tool_responses: dict[str, str] = {} - self.closed = False - self.model_closed = False + self.delegations: OrderedDict[str, _Delegation] = OrderedDict() + self.tools: dict[str, _PendingTool] = {} + self.phase = _LivePhase.ACTIVE self.inputs: list[Any] = [] self.outputs: list[dict[str, str]] = [] self.first_content = False @@ -110,7 +169,7 @@ def __init__(self, session: Any, service: Any) -> None: session.span.log(metadata={"contrib.pipecat.mode": "live"}) def event(self, event: Any) -> None: - if self.closed: + if self.phase is _LivePhase.CLOSED: return kind = event.type if kind == "session.started": @@ -139,17 +198,13 @@ def event(self, event: Any) -> None: self.inputs.pop(0) self.inputs.insert(0, {"role": "system", "content": resource.instructions}) self.model.log(input=self.inputs) - elif kind == "session.output_audio.delta": + elif kind in ("session.output_audio.delta", "session.output_transcript.delta"): if event.delta: self._first_content() - elif kind in ("session.input_transcript.delta", "session.output_transcript.delta"): - if kind == "session.output_transcript.delta" and event.delta: - self._first_content() elif kind == "session.delegation.created": - delegation = event.delegation - self.delegation_offsets[delegation.id] = event.offset_ms - if delegation.response_id: - self.delegation_responses[delegation.id] = delegation.response_id + delegation = self._delegation(event.delegation.id) + delegation.offset_ms = event.offset_ms + delegation.response_id = event.delegation.response_id elif kind == "response.event": self._response(event) elif kind in ("session.usage.updated", "session.closed"): @@ -164,6 +219,25 @@ def event(self, event: Any) -> None: elif kind == "error": self.model.log(error=event.error.message) + def _delegation(self, delegation_id: str | None) -> _Delegation: + key = delegation_id or "uncorrelated" + if key not in self.delegations: + self.delegations[key] = _Delegation() + self.delegations.move_to_end(key) + self._trim_delegations(keep=key) + return self.delegations[key] + + def _trim_delegations(self, *, keep: str | None = None) -> None: + # Live has no delegation-finished notification. Bound historical IDs + # and offsets, while keeping routing needed by an active response. + if len(self.delegations) <= MAX_PENDING: + return + for key, delegation in list(self.delegations.items()): + if key != keep and delegation.response_id not in self.responses: + del self.delegations[key] + if len(self.delegations) <= MAX_PENDING: + break + def _first_content(self) -> None: if not self.first_content: self.first_content = True @@ -190,7 +264,7 @@ def _usage(self, seconds: float) -> None: self.model.log(metadata=metadata) def frame(self, frame: Any) -> None: - if self.closed: + if self.phase is _LivePhase.CLOSED: return kind = type(frame).__name__ if kind == "ProposedUserStartedSpeakingFrame": @@ -207,67 +281,65 @@ def frame(self, frame: Any) -> None: elif kind == "LLMFullResponseEndFrame": self._end_turn("assistant") - def _open_turn(self, role: str) -> None: + def _open_turn(self, role: _Role) -> None: if role in self.turns: return - turn = self.turns[role] = self.frame_spans.turn("user_turn" if role == "user" else "agent_turn") - turn.log(metadata={"contrib.pipecat.live.grouping": "pipecat_transcript_gap"}) + turn = self.turns[role] = _Turn(self.frame_spans.turn("user_turn" if role == "user" else "agent_turn")) + turn.span.log(metadata={"contrib.pipecat.live.grouping": "pipecat_transcript_gap"}) - def _transcript(self, role: str, text: str, *, replace: bool = False, event: Any = None) -> None: - if not text or role not in self.turns: + def _transcript(self, role: _Role, text: str, *, replace: bool = False, event: Any = None) -> None: + turn = self.turns.get(role) + if not text or turn is None: return - transcript = self.transcripts.get(role) - if transcript is None: - transcript = self.transcripts[role] = _Transcript( - self.frame_spans.start( - "transcript_received", - turn=self.turns[role], - metadata={ - SOURCE_ID: "session-input" if role == "user" else "agent-output", - "contrib.pipecat.live.grouping": "pipecat_transcript_gap", - }, - ) + if turn.transcript is None: + turn.transcript = self.frame_spans.start( + "transcript_received", + turn=turn.span, + metadata={ + SOURCE_ID: "session-input" if role == "user" else "agent-output", + "contrib.pipecat.live.grouping": "pipecat_transcript_gap", + }, ) if replace: - transcript.text = [text] + turn.text = [text] else: - transcript.text.append(text) + turn.text.append(text) # User interim frames carry the original event; assistant text frames # do not. Retain provider timestamps only when attached to this frame. start_ms, end_ms = getattr(event, "start_ms", None), getattr(event, "end_ms", None) if start_ms is not None: - transcript.start_ms = start_ms if transcript.start_ms is None else min(transcript.start_ms, start_ms) + turn.start_ms = start_ms if turn.start_ms is None else min(turn.start_ms, start_ms) if end_ms is not None: - transcript.end_ms = end_ms if transcript.end_ms is None else max(transcript.end_ms, end_ms) - transcript.span.log(output=[{"role": role, "content": "".join(transcript.text)}]) - if transcript.start_ms is not None and transcript.end_ms is not None: - transcript.span.log( + turn.end_ms = end_ms if turn.end_ms is None else max(turn.end_ms, end_ms) + turn.transcript.log(output=[{"role": role, "content": "".join(turn.text)}]) + if turn.start_ms is not None and turn.end_ms is not None: + turn.transcript.log( metadata={ - "contrib.pipecat.live.start_ms": transcript.start_ms, - "contrib.pipecat.live.end_ms": transcript.end_ms, + "contrib.pipecat.live.start_ms": turn.start_ms, + "contrib.pipecat.live.end_ms": turn.end_ms, } ) - def _end_turn(self, role: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + def _end_turn(self, role: _Role, *, incomplete: bool = False, end_ms: float | None = None) -> None: + turn = self.turns.pop(role, None) + if turn is None: + return end_time = self.session.clock.wall(end_ms) - transcript = self.transcripts.pop(role, None) - if transcript is not None: - message = {"role": role, "content": "".join(transcript.text)} + if turn.transcript is not None: + message = {"role": role, "content": "".join(turn.text)} if role == "assistant": self.outputs.append(message) self.model.log(output=self.outputs) else: self.inputs.append(message) self.model.log(input=self.inputs) - transcript.span.end(end_time=end_time) - turn = self.turns.pop(role, None) - if turn is not None: - if incomplete: - turn.log(metadata={"turn.incomplete": True}) - turn.end(end_time=end_time) + turn.transcript.end(end_time=end_time) + if incomplete: + turn.span.log(metadata={"turn.incomplete": True}) + turn.span.end(end_time=end_time) def client_event(self, event: Any) -> None: - if self.closed or event.type != "response.item.create": + if self.phase is _LivePhase.CLOSED or event.type != "response.item.create": return item = event.item kind = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) @@ -281,11 +353,11 @@ def _response(self, envelope: Any) -> None: event = envelope.event kind = event.get("type", "") snapshot = event.get("response") or {} - delegation = envelope.delegation_id or "uncorrelated" - response_id = snapshot.get("id") or event.get("response_id") or self.delegation_responses.get(delegation) + delegation = self._delegation(envelope.delegation_id) + response_id = snapshot.get("id") or event.get("response_id") or delegation.response_id if not response_id: return - self.delegation_responses[delegation] = response_id + delegation.response_id = response_id response = self.responses.get(response_id) if response is None: # Missing an observed start is preferable to inventing one. @@ -301,10 +373,10 @@ def _response(self, envelope: Any) -> None: } if snapshot.get("previous_response_id"): metadata["contrib.pipecat.live.previous_response_id"] = snapshot["previous_response_id"] - if delegation in self.delegation_offsets: + if delegation.offset_ms is not None: # This locates delegation creation on the provider's session # timeline, including for subsequent responses in that delegation. - metadata["contrib.pipecat.live.delegation_offset_ms"] = self.delegation_offsets[delegation] + metadata["contrib.pipecat.live.delegation_offset_ms"] = delegation.offset_ms context = getattr(self.service, "_context", None) tools = tools_schema_to_openai_tools(getattr(context, "tools", None)) if tools: @@ -330,94 +402,81 @@ def _response(self, envelope: Any) -> None: item = event.get("item") or {} if item.get("type") == "function_call" and item.get("status") == "completed" and item.get("call_id"): call_id = item["call_id"] - response.pending_calls.add(call_id) - response.pending_tool_spans.add(call_id) - self.tool_responses[call_id] = response_id + response.pending_tools.add(call_id) + self.tools[call_id] = _PendingTool(response_id) self.session.agent.link_tool_request(call_id, response.wrapper) elif kind in ("response.completed", "response.failed", "response.incomplete"): - if snapshot.get("model"): - response.llm.log(metadata={"model": snapshot["model"]}) - response.llm.log(metrics=_parse_metrics_from_usage(snapshot.get("usage"))) - if kind != "response.completed": - error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} - response.llm.log(error=error) - response.wrapper.log(error=error) # Provider inference ends here. Pipecat may still be executing # tools and submitting their results as part of this response. - response.llm.log(output=list(response.output.values())) - response.llm.end(end_time=self.clock.wall()) - response.llm_finished = True + response.finish_inference(kind, snapshot, self.clock.wall()) def event_processed(self, envelope: Any) -> None: - if self.closed or envelope.type != "response.event": + if self.phase is _LivePhase.CLOSED or envelope.type != "response.event": return event = envelope.event if event.get("type") not in ("response.completed", "response.failed", "response.incomplete"): return - response_id = (event.get("response") or {}).get("id") or self.delegation_responses.get(envelope.delegation_id) + response_id = (event.get("response") or {}).get("id") or self._delegation(envelope.delegation_id).response_id if response_id is None: return response = self.responses.get(response_id) - if response is not None: - response.processing_finished = True + if response is not None and response.phase is _ResponsePhase.PROCESSING: + response.phase = _ResponsePhase.WAITING_FOR_TOOLS self._finish_if_ready(response_id, response) def tool_result_submitted(self, call_id: str) -> None: - response_id = self.tool_responses.get(call_id) - if response_id is None: - return - response = self.responses.get(response_id) - if response is not None: - response.pending_calls.discard(call_id) - if call_id not in response.pending_tool_spans: - self.tool_responses.pop(call_id, None) - self._finish_if_ready(response_id, response) + self._tool_progress(call_id, "submission") def tool_finished(self, call_id: str) -> None: - response_id = self.tool_responses.get(call_id) - if response_id is None: + self._tool_progress(call_id, "span") + + def _tool_progress(self, call_id: str, signal: _ToolSignal) -> None: + tool = self.tools.get(call_id) + if tool is None: return - response = self.responses.get(response_id) + tool.waiting_for.discard(signal) + if tool.waiting_for: + return + del self.tools[call_id] + response = self.responses.get(tool.response_id) if response is not None: - response.pending_tool_spans.discard(call_id) - if call_id not in response.pending_calls: - self.tool_responses.pop(call_id, None) - self._finish_if_ready(response_id, response) + response.pending_tools.discard(call_id) + self._finish_if_ready(tool.response_id, response) def _finish_if_ready(self, response_id: str, response: _Response) -> None: - if ( - response.llm_finished - and response.processing_finished - and not response.pending_calls - and not response.pending_tool_spans - ): + if response.ready: self._end_response(response_id) def _end_response(self, response_id: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: response = self.responses.pop(response_id) output = list(response.output.values()) - spans = (response.wrapper,) if response.llm_finished else (response.llm, response.wrapper) + spans = (response.llm, response.wrapper) if response.phase is _ResponsePhase.STREAMING else (response.wrapper,) for span in spans: span.log(output=output) if incomplete: span.log(metadata={"contrib.pipecat.incomplete": True}) span.end(end_time=self.clock.wall(end_ms)) - for call_id in response.pending_calls | response.pending_tool_spans: - self.tool_responses.pop(call_id, None) + for call_id in response.pending_tools: + self.tools.pop(call_id, None) + self._trim_delegations() def _end_model(self, *, confirmed: bool = False, end_ms: float | None = None) -> None: - if self.model_closed: + if self.phase is not _LivePhase.ACTIVE: return - self.model_closed = True + self.phase = _LivePhase.DRAINING self.model.log(metadata={"contrib.pipecat.live.finalization_confirmed": confirmed}) self.model.end(end_time=self.clock.wall(end_ms)) def close(self, end_ms: float) -> None: - if self.closed: + if self.phase is _LivePhase.CLOSED: return - self.closed = True for role in list(self.turns): self._end_turn(role, incomplete=True, end_ms=end_ms) for response_id in list(self.responses): self._end_response(response_id, incomplete=True, end_ms=end_ms) self._end_model(end_ms=end_ms) + self.phase = _LivePhase.CLOSED + self.delegations.clear() + self.tools.clear() + self.inputs.clear() + self.outputs.clear()