diff --git a/py/src/braintrust/integrations/pipecat/__init__.py b/py/src/braintrust/integrations/pipecat/__init__.py index d11c9265..0bc0ddc0 100644 --- a/py/src/braintrust/integrations/pipecat/__init__.py +++ b/py/src/braintrust/integrations/pipecat/__init__.py @@ -24,26 +24,16 @@ def setup_pipecat( capture_agent_audio_attachments: bool | None = None, audio_recording_options: AudioRecordingOptions | None = None, ) -> bool: - """Set up Braintrust tracing for Pipecat ``PipelineWorker`` instances. + """Enable Braintrust tracing for newly created Pipecat ``PipelineWorker`` instances. - Setup instruments ``PipelineWorker`` so every newly constructed worker is - traced, the same as ``braintrust.auto_instrument()`` or - ``wrap_pipeline_worker()``. Pass ``capture_audio_attachments=True`` to attach - both user and agent audio, or configure them independently with + Set ``capture_audio_attachments=True`` to record both sides, or use ``capture_user_audio_attachments`` and ``capture_agent_audio_attachments``. - When arguments are omitted, the generic - environment variables ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` and - ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` apply. Explicit arguments - take precedence. - - Emits the audio v3 schema: a ``session`` with shared recordings, native - ``user_turn`` / ``agent_turn`` groups, speech, STT, LLM, tools and TTS. - Turn containers follow Pipecat's native boundaries. Agent recordings - contain transport-confirmed playback. - ``audio_recording_options`` configures segmentation, encoding and shutdown - using ``braintrust.audio.AudioRecordingOptions``. Recordings default - to Opus/Ogg when PyAV with libopus is available, otherwise raw PCM. - ``braintrust[audio]`` installs a pinned PyAV on Python 3.12+. + Omitted capture options use ``BRAINTRUST_CAPTURE_USER_AUDIO_ATTACHMENTS`` + and ``BRAINTRUST_CAPTURE_AGENT_AUDIO_ATTACHMENTS`` environment defaults. + Explicit arguments take precedence. + + Use ``braintrust.audio.AudioRecordingOptions`` with ``audio_recording_options`` + to configure recording format, segmentation, and shutdown. """ set_default_observer_options( capture_audio_attachments=capture_audio_attachments, diff --git a/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl new file mode 100644 index 00000000..4cc8b87b --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/cassettes/latest/test_live_observations.jsonl @@ -0,0 +1,64 @@ +{"direction":"client","event":{"event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","type":"session.start","session":{"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]},"description":"Look up a demo order."}]}}}}} +{"direction":"server","event":{"event_id":"event_ElPVUiIYrZGMNlqJLFiFG","type":"session.started","client_event_id":"fccb1e69-85c8-4a00-9a62-96f4db1e8326","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]}}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":600,"end_ms":800,"delta":" Hi","event_id":"event_ElPVWWHlfukMfxwz618Rz"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1000,"end_ms":1200,"delta":", can you","event_id":"event_ElPVWoeOkGkKIKBVgdY5i"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1400,"end_ms":1600,"delta":" check","event_id":"event_ElPVXjuV7mEbPWbmd5P9r"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":1600,"end_ms":1800,"delta":" the","event_id":"event_ElPVXFTRoQjDOBP1LlEls"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2000,"end_ms":2200,"delta":" status of","event_id":"event_ElPVXVGsDhgnQQ9OtnYEi"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2400,"end_ms":2600,"delta":" order","event_id":"event_ElPVYt0HmxVEv5CxInigK"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":2600,"end_ms":2800,"delta":" demo","event_id":"event_ElPVYbFRHP7XQMPjWJMPF"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3400,"end_ms":3600,"delta":" one-zero-four","event_id":"event_ElPVZ52wTQKYBquK1OXrG"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":3800,"end_ms":4000,"delta":"-two","event_id":"event_ElPVZ07JA6vf0lCqsIl9Z"}} +{"direction":"server","event":{"type":"session.input_transcript.delta","start_ms":4000,"end_ms":4200,"delta":" for me","event_id":"event_ElPVayzVz5Z3qTqYbCPTj"}} +{"direction":"server","event":{"type":"session.delegation.created","offset_ms":4000,"delegation":{"id":"item_ElPValIun0iiFZeK3vH3V","type":"delegation","response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","target":"responses"},"event_id":"event_ElPVaJSXQiPNp0Z0Zbv5t"}} +{"direction":"server","event":{"event_id":"event_ElPVa8w6ciMc8UdVenwku","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVajhc2ndB2mFvEoRo8","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"event_id":"event_ElPVa0BkWN8Ng98YBVgyP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"in_progress","arguments":"","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVaARgg3NqlQfyJzeKv","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"{\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"dBYZCdvQTKGArb","output_index":0,"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVaMvD2Y935CoXAYUqC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"order","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"SYKI7ZBw0MU","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVarxoCcxzRUhbjBqvC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"_id","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"XzU0Mg9SVfmc5","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVaFbiZMpGQgu3hZ1hC","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\":\"","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"Qm2eKieFwRWrX","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVad1NxeGPNTi1g2tuq","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"demo","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"3I38JeIPwUAe","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVajin91jv8dq70TkDP","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":" one","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"yAkBTo0JCQ9L","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVaWWowaT9lnEga5ejm","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-zero","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"jaS6KMVTmrF","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4400,"end_ms":4600,"delta":" Sure,","event_id":"event_ElPVaSQGfzFZCEatLEfSQ"}} +{"direction":"server","event":{"event_id":"event_ElPVaWPYYHSakH5cwvshA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-four","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"r9L1kQGfMCT","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVa5yxCYDy4X1IoZikk","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"-two","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"1sc2EYMYNuS8","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPValeQiW62DmkQY8UK3","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.delta","delta":"\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","obfuscation":"ywrVdquEX49McQ","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVaK6y8l1STM0wDSL2y","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.function_call_arguments.done","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","item_id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVacIMXw3PzZyXXrhEB","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"fc_0fc7b560e5b17e32016afd0a727fdc87d19eb64fc05cc30eeb","type":"function_call","status":"completed","arguments":"{\"order_id\":\"demo one-zero-four-two\"}","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","name":"lookup_order"},"output_index":0,"sequence_number":14}}} +{"direction":"client","event":{"event_id":"3b642bcb-fe88-4579-954b-ddb9880eaaa6","type":"response.item.create","item":{"type":"function_call_output","call_id":"call_fyEzZEW6PypCeX94RvcpGtyL","output":"\"The function `lookup_order` is not currently available.\""}}} +{"direction":"server","event":{"event_id":"event_ElPVaDCYiCRVSg52Cyz9A","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968178,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a6c3e5887d1a5707d8e3f391e9e","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":813,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":23,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":836},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":15}}} +{"direction":"client","event":{"event_id":"7ae94f2a-3c67-44c1-9ee9-634da4bb4552","type":"response.create"}} +{"direction":"server","event":{"event_id":"event_ElPVaoEvAem5AAWJcHecy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.created","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":0}}} +{"direction":"server","event":{"event_id":"event_ElPVaHJ0doxl5xMgV84ie","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.in_progress","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"in_progress","access_programs":null,"background":false,"completed_at":null,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"auto","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":null,"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":1}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":4800,"end_ms":5000,"delta":" one","event_id":"event_ElPVaISi6kc0Tl3WOIXaJ"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5000,"end_ms":5200,"delta":" moment","event_id":"event_ElPVbQgaKdets1lYHX4bV"}} +{"direction":"server","event":{"event_id":"event_ElPVbTgZ5lbR0ApghDO2N","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.added","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"in_progress","content":[],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":2}}} +{"direction":"server","event":{"event_id":"event_ElPVbeBwMyUxT9B2GNVEy","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.added","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":3}}} +{"direction":"server","event":{"event_id":"event_ElPVbVKt6gD7K22l5Hs6P","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"I","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Uin9ACIpZiskhN0","output_index":0,"sequence_number":4}}} +{"direction":"server","event":{"event_id":"event_ElPVbdO25sEOmkCPzRshx","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" can","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"nLPBMZogyfYU","output_index":0,"sequence_number":5}}} +{"direction":"server","event":{"event_id":"event_ElPVbXILZVqfGcZ7Jdfta","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":"\u2019t","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"Tj9PG9N5o066Nn","output_index":0,"sequence_number":6}}} +{"direction":"server","event":{"event_id":"event_ElPVbzHhIAI7b619Ys1Be","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" look","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"m2YnsBIuTMs","output_index":0,"sequence_number":7}}} +{"direction":"server","event":{"event_id":"event_ElPVb0SyXHpF40QYLPTi2","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" up","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"EUIFn4mTYOm6q","output_index":0,"sequence_number":8}}} +{"direction":"server","event":{"event_id":"event_ElPVbyimQMpEIoINz3oTg","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" the","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"V9ftJtijasUo","output_index":0,"sequence_number":9}}} +{"direction":"server","event":{"event_id":"event_ElPVblJhPZs8uSQ6UA2rd","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" order","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"gqHyKjmM6T","output_index":0,"sequence_number":10}}} +{"direction":"server","event":{"event_id":"event_ElPVbQfPkMUkCuJ1OeSnA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" right","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"uFjfB2AZ91","output_index":0,"sequence_number":11}}} +{"direction":"server","event":{"event_id":"event_ElPVbA0VV3QQWMGVtFYax","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":" now","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"cLca2VTTDu1u","output_index":0,"sequence_number":12}}} +{"direction":"server","event":{"event_id":"event_ElPVbnEDiWTJmTc6ERtII","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.delta","content_index":0,"delta":".","item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"obfuscation":"SAkhNXWgmld9Pfv","output_index":0,"sequence_number":13}}} +{"direction":"server","event":{"event_id":"event_ElPVb5JhvPA2NybeFOV6J","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_text.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","logprobs":[],"output_index":0,"sequence_number":14,"text":"I can\u2019t look up the order right now."}}} +{"direction":"server","event":{"event_id":"event_ElPVblv16txRZfO6LKcBA","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.content_part.done","content_index":0,"item_id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","output_index":0,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."},"sequence_number":15}}} +{"direction":"server","event":{"event_id":"event_ElPVb93iTa5CFzhNrBzDz","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.output_item.done","item":{"id":"msg_0fc7b560e5b17e32016afd0a731ed087d19ed9c2c1b0c31aa9","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"I can\u2019t look up the order right now."}],"phase":"final_answer","role":"assistant"},"output_index":0,"sequence_number":16}}} +{"direction":"server","event":{"event_id":"event_ElPVbgyrNJux5a9qjkear","type":"response.event","delegation_id":"item_ElPValIun0iiFZeK3vH3V","event":{"type":"response.completed","response":{"id":"resp_0fc7b560e5b17e32016afd0a72ca7c87d1aea8ae7f4e2318af","object":"response","created_at":1794968178,"status":"completed","access_programs":null,"background":false,"completed_at":1794968179,"error":null,"frequency_penalty":0.0,"image":{"context_compression":"disabled"},"incomplete_details":null,"input_transformations":[],"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4-mini-2026-03-17","moderation":null,"output":[],"parallel_tool_calls":true,"presence_penalty":0.0,"previous_response_id":"resp_0fc7b560e5b17e32016afd0a72038887d1aad5da79e30d6cb1","prompt_cache_key":"live:e5cd15a1dcb42156d2ef33b86f3ea050fdbbfc04234ed49fb781fc3477a","prompt_cache_retention":"in_memory","reasoning":{"context":"current_turn","effort":"none","mode":"standard","summary":null},"safety_identifier":null,"service_tier":"default","store":false,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tool_usage":{"image_gen":{"input_tokens":0,"input_tokens_details":{"image_tokens":0,"text_tokens":0},"output_tokens":0,"output_tokens_details":{"image_tokens":0,"text_tokens":0},"total_tokens":0},"web_search":{"num_requests":0}},"tools":[],"top_logprobs":0,"top_p":0.98,"truncation":"disabled","usage":{"input_tokens":858,"input_tokens_details":{"cache_write_tokens":0,"cache_write_tokens_details":{"ttl_12h_tokens":0,"ttl_30m_tokens":0},"cached_tokens":0},"output_tokens":14,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":872},"user":"user-eHOTg7f6lK20Ot29MNWkDIME","metadata":{}},"sequence_number":17}}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5400,"end_ms":5600,"delta":" while I","event_id":"event_ElPVbwgjIJZXs7w85zo58"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5600,"end_ms":5800,"delta":" check","event_id":"event_ElPVblC6bB5SM0w8rf5av"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":5800,"end_ms":6000,"delta":" that.","event_id":"event_ElPVbTj7q4QwvBIWSYP39"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":7800,"end_ms":8000,"delta":" Sorry,","event_id":"event_ElPVdeOzdi4jxkH6wKGPj"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8000,"end_ms":8200,"delta":" I","event_id":"event_ElPVe4U4kVNeVckkofX1E"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8200,"end_ms":8400,"delta":" can\u2019t","event_id":"event_ElPVely9H3CKKfrfBnYRx"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8600,"end_ms":8800,"delta":" look","event_id":"event_ElPVemy8xOZC9CX9lIiGL"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":8800,"end_ms":9000,"delta":" it up","event_id":"event_ElPVeRGzzIweT3dX1OpJ8"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9000,"end_ms":9200,"delta":" right","event_id":"event_ElPVfryUNgBPSpSqOQ3S3"}} +{"direction":"server","event":{"type":"session.output_transcript.delta","start_ms":9200,"end_ms":9400,"delta":" now.","event_id":"event_ElPVf6o5Pm0ILKBxygnAY"}} +{"direction":"client","event":{"event_id":"735fd94b-b073-4a40-8372-bd7500213598","type":"session.close"}} +{"direction":"server","event":{"event_id":"event_ElPVhGcRa2AcM4UssEkqq","type":"session.closed","reason":"close_requested","session":{"id":"live_u1_ElPVTp3oMyvwTVue5080RwW25JA82bCG","expires_at":1794975372,"model":"gpt-live-1","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","audio":{"output":{"voice":"marin"},"format":{"type":"audio/pcm","rate":24000}},"delegation":{"type":"responses","responses":{"model":"gpt-5.4-mini","instructions":"Speak English. When the caller asks about an order, use lookup_order. If it is unavailable, briefly say you cannot look it up right now.","tools":[{"type":"function","name":"lookup_order","description":"Look up a demo order.","parameters":{"type":"object","properties":{"order_id":{"type":"string"}},"required":["order_id"]}}],"tool_choice":"auto"}},"status":"active","input":[]},"usage":{"seconds":11.0},"client_event_id":"735fd94b-b073-4a40-8372-bd7500213598"}} diff --git a/py/src/braintrust/integrations/pipecat/integration.py b/py/src/braintrust/integrations/pipecat/integration.py index 7b2c1476..dacd0d75 100644 --- a/py/src/braintrust/integrations/pipecat/integration.py +++ b/py/src/braintrust/integrations/pipecat/integration.py @@ -4,6 +4,7 @@ from .patchers import ( LLMServicePushFramePatcher, + OpenAILivePatcher, PipelineWorkerPatcher, SegmentedSTTPushSegmentFramesPatcher, STTServiceProcessGeneratorPatcher, @@ -22,4 +23,5 @@ class PipecatIntegration(BaseIntegration): LLMServicePushFramePatcher, STTServiceProcessGeneratorPatcher, SegmentedSTTPushSegmentFramesPatcher, + OpenAILivePatcher, ) diff --git a/py/src/braintrust/integrations/pipecat/patchers.py b/py/src/braintrust/integrations/pipecat/patchers.py index ce9d9e75..ffc2da7f 100644 --- a/py/src/braintrust/integrations/pipecat/patchers.py +++ b/py/src/braintrust/integrations/pipecat/patchers.py @@ -1,5 +1,6 @@ """Pipecat integration patchers.""" +import inspect from contextvars import ContextVar from typing import Any @@ -7,6 +8,7 @@ from braintrust.integrations.base import CompositeFunctionWrapperPatcher, FunctionWrapperPatcher from .tracing import BraintrustPipecatObserver +from .tracing._live import is_live_service _DEFAULT_OBSERVER_OPTIONS: dict[str, Any] = { @@ -74,14 +76,18 @@ def _release_response_span(service: Any) -> None: async def traced_llm_push_frame(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: """Make an LLM service's llm_response span current while it makes the call. - Every LLM service pushes its response start frame, sends the provider - request and pushes its response end frame from the same task. Spans the + Cascade LLM services push their response start frame, send the provider + request and push their response end frame from the same task. Spans the provider integration starts in between nest under llm_response. """ observer = _ACTIVE_OBSERVER.get() frame = args[0] if args else kwargs.get("frame") kind = type(frame).__name__ - if observer is None or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame"): + if ( + observer is None + or kind not in ("LLMFullResponseStartFrame", "LLMFullResponseEndFrame") + or is_live_service(instance) + ): return await wrapped(*args, **kwargs) _release_response_span(instance) if kind == "LLMFullResponseEndFrame": @@ -166,6 +172,58 @@ class PipelineWorkerPatcher(CompositeFunctionWrapperPatcher): sub_patchers = (PipelineWorkerInitPatcher, PipelineWorkerRunPatcher) +async def traced_live_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + event = args[0] if args else kwargs.get("evt") + if observer is not None: + observer._live_event(instance, event) + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_event_processed(event) + return result + + +async def traced_live_client_event(wrapped: Any, instance: Any, args: tuple[Any, ...], kwargs: dict[str, Any]) -> Any: + observer = _ACTIVE_OBSERVER.get() + result = await wrapped(*args, **kwargs) + if observer is not None: + observer._live_client_event(args[0] if args else kwargs.get("event")) + return result + + +class OpenAILiveEventPatcher(FunctionWrapperPatcher): + name = "openai_live_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService._handle_server_event" + wrapper = staticmethod(traced_live_event) + + +class OpenAILiveClientEventPatcher(FunctionWrapperPatcher): + name = "openai_live_client_event" + target_module = "pipecat.services.openai.live.llm" + target_path = "OpenAILiveLLMService.send_client_event" + wrapper = staticmethod(traced_live_client_event) + + +class OpenAILivePatcher(CompositeFunctionWrapperPatcher): + name = "openai_live" + sub_patchers = (OpenAILiveEventPatcher, OpenAILiveClientEventPatcher) + + @classmethod + def applies(cls, module: Any | None, version: str | None, *, target: Any | None = None) -> bool: + # Both hooks are needed to track a response through tool submission. + # Let the composite own patch markers and idempotency after this gate. + for patcher, argument in ((OpenAILiveEventPatcher, "evt"), (OpenAILiveClientEventPatcher, "event")): + method = patcher.resolve_target(module, version, target=target) + if not inspect.iscoroutinefunction(method): + return False + try: + inspect.signature(method).bind(None, **{argument: None}) + except (TypeError, ValueError): + return False + return super().applies(module, version, target=target) + + def wrap_pipeline_worker( PipelineWorker: Any, *, @@ -199,4 +257,7 @@ def wrap_pipeline_worker( LLMServicePushFramePatcher.wrap_target(LLMService) STTServiceProcessGeneratorPatcher.wrap_target(STTService) SegmentedSTTPushSegmentFramesPatcher.wrap_target(SegmentedSTTService) + # Optional on older Pipecat versions; the patchers feature-detect the module. + if OpenAILivePatcher.applies(None, None): + OpenAILivePatcher.patch(None, None) return PipelineWorkerPatcher.wrap_target(PipelineWorker) diff --git a/py/src/braintrust/integrations/pipecat/test_pipecat.py b/py/src/braintrust/integrations/pipecat/test_pipecat.py index bcae36d5..8d9fd2e5 100644 --- a/py/src/braintrust/integrations/pipecat/test_pipecat.py +++ b/py/src/braintrust/integrations/pipecat/test_pipecat.py @@ -13,6 +13,7 @@ import inspect import json import logging +import math import os import socket import tempfile @@ -23,7 +24,6 @@ from urllib.parse import urlsplit, urlunsplit import pytest -import pytest_asyncio from braintrust import SpanCustomizer, logger, set_span_customizers from braintrust.integrations.pipecat import setup_pipecat, wrap_pipeline_worker from braintrust.integrations.test_utils import verify_autoinstrument_script @@ -448,6 +448,7 @@ async def test_voice_session_traces_turns_speech_recognition_generation_and_play assert _link_targets(playback, "generated_by") == [synthesis["span_id"]] assert playback["metadata"]["audio.speaker_ids"] == synthesis["metadata"]["audio.speaker_ids"] assert playback["metadata"]["contrib.pipecat.interrupted"] is False + assert playback["output"] == [{"role": "assistant", "content": synthesis["input"]["text"]}] assert "audio.participant_ids" in playback["metadata"] # Both directions land in shared recordings; playback is what was sent. @@ -770,145 +771,252 @@ def failing_end(span, *args, **kwargs): assert root["metadata"]["contrib.pipecat.terminal_frame"] == "EndFrame" -@pytest_asyncio.fixture -async def tracker_observer(memory_logger): - # These regressions concern local frame bookkeeping, not provider payloads. - # Use the real trackers, clock, recorder, services and native frame classes. - _frames() - from braintrust.integrations.pipecat.tracing.observer import BraintrustPipecatObserver - - observer = BraintrustPipecatObserver(capture_audio_attachments=False) - yield observer - await observer.cleanup() - - -@pytest.mark.asyncio -@pytest.mark.parametrize("boundary", ["turn", "response", "shutdown"]) -async def test_queued_transcript_retains_text_and_receipt_time(memory_logger, tracker_observer, boundary): - frames = _frames() - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - stt = STT(api_key="unused") - session = tracker_observer._start_session() - with session.clock.event(0): - session.user.start_speech(session.clock.now_ms()) - session.user.consume_audio(stt) - with session.clock.event(100_000_000): - session.user.stop_speech() - session.user.transcribe(frames.TranscriptionFrame("Hello", "caller", "0"), stt) - received = session.clock.wall() - with session.clock.event(200_000_000): - if boundary == "turn": - session.user.start_turn() - elif boundary == "response": - session.user.flush_transcribed() - else: - session.user.close(session.clock.now_ms()) - recognition = _single_span(memory_logger.pop(), "stt") - assert recognition["output"] == [{"role": "user", "content": "Hello"}] - assert recognition["metrics"]["end"] == received - - @pytest.mark.asyncio -async def test_untranscribed_stt_ends_with_its_user_turn(memory_logger, tracker_observer): - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - session = tracker_observer._start_session() - with session.clock.event(0): - session.user.start_turn() - session.user.start_speech(session.clock.now_ms()) - session.user.consume_audio(STT(api_key="unused")) - with session.clock.event(100_000_000): - session.user.stop_speech() - session.user.stop_turn() - with session.clock.event(10_000_000_000): - session.user.close(session.clock.now_ms()) - logs = memory_logger.pop() - recognition = _single_span(logs, "stt") - turn = _single_span(logs, "user_turn") - assert recognition["metrics"]["end"] == turn["metrics"]["end"] - assert recognition["metadata"]["contrib.pipecat.incomplete"] is True - +@pytest.mark.parametrize("boundary", ["late_transcript", "cancel", "cancel_callback"]) +async def test_live_observations(memory_logger, vcr_cassette_dir, boundary): + live = pytest.importorskip("pipecat.services.openai.live.llm") + from websockets.asyncio.server import serve + from websockets.exceptions import ConnectionClosed + + FunctionSchema = _import("pipecat.adapters.llm.schemas.function_schema.FunctionSchema") + ToolsSchema = _import("pipecat.adapters.llm.schemas.tools_schema.ToolsSchema") + Pipeline = _import("pipecat.pipeline.pipeline.Pipeline") + LLMContext = _import("pipecat.processors.aggregators.llm_context.LLMContext") + OpenAIResponsesLLMService = _import("pipecat.services.openai.responses.llm.OpenAIResponsesLLMService") + WorkerRunner = _import("pipecat.workers.runner.WorkerRunner") -@pytest.mark.asyncio -@pytest.mark.parametrize( - "parts,spaced,expected", - [ - (["Hello.", "How are you?"], False, "Hello. How are you?"), - (["Hello", " there", "!"], True, "Hello there!"), - ], -) -async def test_tts_and_played_text_preserve_native_spacing(memory_logger, tracker_observer, parts, spaced, expected): - frames = _frames() - session = tracker_observer._start_session() - session.agent.start_tts(frames.TTSStartedFrame(), _openai_tts()) - session.agent.start_playback() - session.agent.played_audio(frames.OutputAudioRawFrame(bytes(320), 16000, 1)) - for text in parts: - frame = frames.TTSTextFrame(text, aggregated_by="sentence") - frame.includes_inter_frame_spaces = spaced - session.agent.tts_text(frame) - # Feed the same text through the actual observer playback callback. - tracker_observer._observe_playback(session, frame) - session.agent.stop_tts(frames.TTSStoppedFrame()) - session.agent.stop_playback() - logs = memory_logger.pop() - assert _single_span(logs, "tts")["input"]["text"] == expected - assert _single_span(logs, "agent_speech")["output"] == [{"role": "assistant", "content": expected}] + # Native WebSocket capture, including the real built-in missing-tool result. + # Audio payloads are omitted; shared recorder coverage lives in voice tests. + recording = [ + json.loads(line) for line in (Path(vcr_cassette_dir) / "test_live_observations.jsonl").read_text().splitlines() + ] + config = recording[0]["event"]["session"] + backend = config["delegation"]["responses"] + server_events = [r["event"] for r in recording if r["direction"] == "server"] + transcripts = [ + e for e in server_events if e["type"] in ("session.input_transcript.delta", "session.output_transcript.delta") + ] + late = transcripts[-1] + sent, outgoing = [], [] + ready, callback_cancelled = asyncio.Event(), asyncio.Event() + replay_errors = [] + + async def replay(websocket): + delayed_transcripts = [] + try: + for record in recording: + event = record["event"] + if record["direction"] == "client": + actual = json.loads(await websocket.recv()) + outgoing.append(actual) + assert actual["type"] == event["type"] + if actual["type"] == "session.start": + assert actual["session"]["instructions"] == event["session"]["instructions"] + if actual["type"] == "response.item.create": + assert actual["item"] == event["item"] + if actual["type"] == "session.close": + # Deliver a captured fragment late, while the real + # service awaits the server's close acknowledgment. + await websocket.send(json.dumps(late)) + sent.append(late) + else: + if event == late: + ready.set() + continue + if boundary == "cancel_callback" and event["type"] == "session.output_transcript.delta": + # Delay the independent speech stream until backend + # text has arrived, then cancel inside its public callback. + delayed_transcripts.append(event) + continue + if ( + boundary == "cancel_callback" + and event["type"] == "response.event" + and event["event"]["type"] == "response.output_text.done" + ): + await websocket.send(json.dumps(delayed_transcripts[0])) + sent.append(delayed_transcripts[0]) + await websocket.wait_closed() + return + await websocket.send(json.dumps(event)) + sent.append(event) + if boundary != "late_transcript" and event["type"] == "session.output_transcript.delta": + await websocket.wait_closed() + return + except ConnectionClosed: + if boundary == "late_transcript": + raise + except Exception as exc: + replay_errors.append(exc) + ready.set() + _instrument() + async with serve(replay, "127.0.0.1", 0) as server: + port = server.sockets[0].getsockname()[1] + service = live.OpenAILiveLLMService( + api_key="unused-in-replay", + base_url=f"ws://127.0.0.1:{port}", + settings=live.OpenAILiveLLMService.Settings( + model=config["model"], system_instruction=config["instructions"] + ), + delegation=live.ResponsesDelegation( + settings=OpenAIResponsesLLMService.Settings( + model=backend["model"], system_instruction=backend["instructions"] + ) + ), + ) + context = LLMContext( + messages=[{"role": "system", "content": "Superseded context instructions."}] + if boundary == "cancel_callback" + else [], + tools=ToolsSchema( + standard_tools=[ + FunctionSchema( + name=t["name"], + description=t["description"], + properties=t["parameters"]["properties"], + required=t["parameters"]["required"], + ) + for t in backend["tools"] + ] + ), + ) -@pytest.mark.asyncio -@pytest.mark.parametrize("boundary", ["completed", "interrupted", "shutdown"]) -async def test_playback_completion_distinguishes_shutdown_from_interruption(memory_logger, tracker_observer, boundary): - frames = _frames() - session = tracker_observer._start_session() - with session.clock.event(0): - session.agent.start_tts(frames.TTSStartedFrame(), _openai_tts()) - session.agent.start_playback() - session.agent.played_audio(frames.OutputAudioRawFrame(bytes(320), 16000, 1)) - with session.clock.event(100_000_000): - # Synthesis can finish while its audio is still playing. - session.agent.stop_tts(frames.TTSStoppedFrame()) - with session.clock.event(200_000_000): - if boundary == "completed": - session.agent.stop_playback() - elif boundary == "interrupted": - session.agent.interrupt() - await tracker_observer._finish() + # Pipecat executes its real missing-tool handler. No replacement tool, + # provider client, frame processor, or observer stands in for the SDK. + @service.event_handler("on_after_push_frame") + async def on_frame(_service, frame): + if isinstance(frame, _frames().LLMTextFrame) and boundary != "late_transcript": + ready.set() + if boundary == "cancel_callback": + try: + await asyncio.Event().wait() + finally: + callback_cancelled.set() + + worker = _make_worker(Pipeline([service])) + runner = WorkerRunner(handle_sigint=False) + await runner.add_workers(worker) + running = asyncio.create_task(runner.run()) + try: + await worker.queue_frame(_frames().LLMContextFrame(context)) + await asyncio.wait_for(ready.wait(), 10) + if replay_errors: + raise replay_errors[0] + tracker = worker._braintrust_pipecat_observer._session.live + assert tracker is not None + if boundary == "late_transcript": + await worker.queue_frame(_frames().EndFrame()) + else: + await worker.cancel() + await asyncio.wait_for(running, 10) + finally: + if not running.done(): + await worker.cancel() + await asyncio.wait_for(running, 10) + assert not replay_errors + # Cleanup releases the tracker's own payload buffers and correlation + # caches, even if a caller retains the tracker after pipeline shutdown. + assert not {name: len(value) for name, value in vars(tracker).items() if isinstance(value, (dict, list)) and value} + if boundary == "cancel_callback": + assert callback_cancelled.is_set() logs = memory_logger.pop() - playback = _single_span(logs, "agent_speech") - assert playback["metadata"]["contrib.pipecat.interrupted"] is (boundary == "interrupted") - if boundary == "shutdown": - assert playback["metadata"]["contrib.pipecat.incomplete"] is True + session = _single_span(logs, "session") + model = _single_span(logs, "pipecat.live") + assert model["span_attributes"]["type"] == "llm" + assert model["metadata"]["audio.scope"] == "session" + assert model["span_parents"] == [session["span_id"]] + assert model["metadata"]["contrib.pipecat.live.finalization_confirmed"] is (boundary == "late_transcript") + assert all(row["metrics"].get("end") is not None for row in logs) + turns = _spans_named(logs, "user_turn") + _spans_named(logs, "agent_turn") + by_id = {row["span_id"]: row for row in turns} + received = _spans_named(logs, "transcript_received") + assert turns and len(turns) == len(received) + for role in ("user", "assistant"): + kind = "session.input_transcript.delta" if role == "user" else "session.output_transcript.delta" + observed = "".join(e["delta"] for e in sent if e["type"] == kind) + assert "".join(r["output"][0]["content"] for r in received if r["output"][0]["role"] == role) == observed + for row in received: + turn = by_id[row["metadata"]["turn.id"]] + assert row["span_parents"] == [turn["span_id"]] + assert turn["metadata"]["turn.id"] == turn["span_id"] + assert turn["span_attributes"]["name"] == ("user_turn" if row["output"][0]["role"] == "user" else "agent_turn") + assert turn["metrics"]["start"] <= row["metrics"]["start"] <= row["metrics"]["end"] <= turn["metrics"]["end"] + assert "audio.selections" not in row["metadata"] + assert "audio.selections" not in turn["metadata"] + assert "turn.reply_to" not in turn["metadata"] + # Cancellation, or a group opened by late text after native stop, must not + # be presented as a normally completed framework turn. + assert any(row["metadata"].get("turn.incomplete") for row in turns) + assert all("turn.id" not in row.get("metadata", {}) for row in logs if row not in turns + received) + responses = _spans_named(logs, "llm_response") + assert all(row["span_attributes"]["type"] == "task" and "tokens" not in row["metrics"] for row in responses) + assert all(not _link_targets(row, "consumes_tool_result") for row in responses) + llms = _spans_named(logs, "pipecat.live.backend_response") + if boundary == "late_transcript": + assert len(responses) == 2 + assert ( + responses[1]["metadata"]["contrib.pipecat.live.previous_response_id"] + == responses[0]["metadata"]["contrib.pipecat.live.response_id"] + ) + delegation = next(e for e in server_events if e["type"] == "session.delegation.created") + assert all( + row["metadata"]["contrib.pipecat.live.delegation_offset_ms"] == delegation["offset_ms"] + for row in responses + ) + tool = _single_span(logs, "lookup_order") + assert tool["span_parents"] == [responses[0]["span_id"]] + assert _link_targets(tool, "requested_by") + assert responses[0]["metrics"]["end"] >= tool["metrics"]["end"] + assert any(e["type"] == "response.item.create" for e in outgoing) + completions = [ + e["event"]["response"] + for e in server_events + if e["type"] == "response.event" and e["event"]["type"] == "response.completed" + ] + assert [r["metrics"]["tokens"] for r in llms] == [r["usage"]["total_tokens"] for r in completions] + assert [r["metrics"]["prompt_cache_creation_tokens"] for r in llms] == [ + r["usage"]["input_tokens_details"]["cache_write_tokens"] for r in completions + ] + completed_items = [ + e["event"]["item"] + for e in sent + if e["type"] == "response.event" and e["event"]["type"] == "response.output_item.done" + ] + assert [r["output"] for r in llms] == [[item] for item in completed_items] + closed = next(e for e in server_events if e["type"] == "session.closed") + assert model["metadata"]["contrib.pipecat.live.usage_seconds"] == closed["usage"]["seconds"] else: - assert "contrib.pipecat.incomplete" not in playback["metadata"] - assert playback["metadata"]["audio.selections"]["selection-0"] == { - "start_offset_ms": 0, - "end_offset_ms": 200, - } - _assert_all_ended(logs) - - -@pytest.mark.asyncio -@pytest.mark.parametrize("failure", [RuntimeError, asyncio.CancelledError]) -async def test_recognition_scope_restores_context_after_errors_and_cancellation(tracker_observer, failure): - STT = _import("pipecat.services.openai.stt.OpenAISTTService") - stt = STT(api_key="unused") - session = tracker_observer._start_session() - session.user.start_turn() - session.user.consume_audio(stt) - - with logger.start_span(name="caller") as caller: - with pytest.raises(failure): - with tracker_observer._current_recognition(stt): - recognition = logger.current_span() - assert recognition is not caller - # Reentrant wrapping must also leave the outer scope intact. - with tracker_observer._current_recognition(stt): - assert logger.current_span() is recognition - assert logger.current_span() is recognition - await asyncio.sleep(0) - raise failure("operation stopped") - assert logger.current_span() is caller + assert responses[-1]["metadata"].get("contrib.pipecat.incomplete") + assert llms[-1]["metadata"].get("contrib.pipecat.incomplete") + events = [e["event"] for e in sent if e["type"] == "response.event"] + item = [e["item"] for e in events if e["type"] == "response.output_item.added"][-1] + assert not any(e["type"] == "response.output_item.done" and e["item"]["id"] == item["id"] for e in events) + if boundary == "cancel": + arguments = "".join(e["delta"] for e in events if e["type"] == "response.function_call_arguments.delta") + assert arguments + expected = {**item, "arguments": arguments} + else: + part = next(e["part"] for e in events if e["type"] == "response.content_part.added") + text = "".join(e["delta"] for e in events if e["type"] == "response.output_text.delta") + assert text + expected = {**item, "content": [{**part, "text": text}]} + assert llms[-1]["output"] == responses[-1]["output"] == [expected] + started = next(e for e in server_events if e["type"] == "session.started") + assert [message for message in model["input"] if message.get("role") == "system"] == [ + {"role": "system", "content": started["session"]["instructions"]} + ] + by_id = {row["span_id"]: row for row in logs} + for row in logs: + for parent_id in row.get("span_parents") or []: + parent = by_id[parent_id] + parent_start = parent["metrics"]["start"] + if parent_id == session["span_id"]: + # The session uses time.time(); recorder-derived child times + # can round the same Windows clock tick one float apart. + parent_start = math.nextafter(parent_start, -math.inf) + assert parent_start <= row["metrics"]["start"], row["span_attributes"]["name"] + assert row["metrics"]["end"] <= parent["metrics"]["end"], row["span_attributes"]["name"] # Keep provider tests last: enabling OpenAI patches the client for the process. diff --git a/py/src/braintrust/integrations/pipecat/tracing/_agent.py b/py/src/braintrust/integrations/pipecat/tracing/_agent.py index ca432979..f7a97c5d 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/_agent.py +++ b/py/src/braintrust/integrations/pipecat/tracing/_agent.py @@ -107,6 +107,7 @@ class _Playback: # span and set its start only after the transport confirms an audio frame. span: Span | None = None start_ms: float = 0.0 + audio_end_ms: float = 0.0 text: SpeechText = field(default_factory=SpeechText) generators: dict[str, _Synthesis] = field(default_factory=dict) @@ -133,6 +134,7 @@ class AgentTracker: def __init__(self, spans: SpanFactory, clock: SessionClock, recorder: AudioSessionRecorder) -> None: self._spans, self._clock, self._recorder = spans, clock, recorder + self.report_uninterrupted_playback = True self._turn: _AgentTurn | None = None self._response: _ModelResponse | None = None self._last_response: Span | None = None @@ -250,6 +252,12 @@ def record_metrics(self, frame: "MetricsFrame", processor: "FrameProcessor | Non # Tools + def link_tool_request(self, call_id: str, requester: Span) -> None: + """Associate a native Live backend request with its later tool frame.""" + # Pipecat owns the tool's execution and result submission as part of + # handling this response, even when execution uses another task. + self._requesters[call_id] = requester + def start_tool(self, frame: "FunctionCallInProgressFrame | FunctionCallResultFrame") -> None: key = frame.tool_call_id if key in self._tools or key in self._finished_tools: @@ -405,6 +413,9 @@ def played_audio(self, frame: "AudioRawFrame") -> None: playback = self._playback if playback is None: return + playback.audio_end_ms = max(playback.audio_end_ms, self._clock.now_ms()) + ( + len(frame.audio) * 1000 / (2 * frame.num_channels * frame.sample_rate) + ) span = playback.span if span is None: playback.start_ms = self._clock.now_ms() @@ -470,7 +481,7 @@ def interrupt(self, *, interrupted: bool = True, end_ms: float | None = None) -> if self._playback is not None: if self._playback.span is not None: self._playback.span.log(metadata={status: True}) - self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms) + self._end_playback(self._playback, interrupted=interrupted, end_ms=end_ms, shutdown=not interrupted) if self._response is not None: self._response.span.log(metadata={status: True}) self._end_response(self._response, end_ms=end_ms) @@ -529,17 +540,22 @@ def _end_synthesis(self, synthesis: _Synthesis, *, end_ms: float | None = None) synthesis.ended = True synthesis.text.parts.clear() - def _end_playback(self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None) -> None: + def _end_playback( + self, playback: _Playback, *, interrupted: bool, end_ms: float | None = None, shutdown: bool = False + ) -> None: if playback.span is None: self._playback = None return end_ms = self._clock.now_ms() if end_ms is None else end_ms - playback.span.log( - metadata={ - SELECTIONS: {"selection-0": selection(playback.start_ms, end_ms)}, - "contrib.pipecat.interrupted": interrupted, - } - ) + # Cleanup may wait for remote closure after the transport stops sending. + # Keep the observed span lifetime, but don't select teardown as speech. + selection_end = min(end_ms, playback.audio_end_ms) if shutdown else end_ms + metadata: dict[str, Any] = {SELECTIONS: {"selection-0": selection(playback.start_ms, selection_end)}} + # Live handles automatic interruption remotely without notifying + # Pipecat. An explicit local interruption is still observable. + if interrupted or self.report_uninterrupted_playback: + metadata["contrib.pipecat.interrupted"] = interrupted + playback.span.log(metadata=metadata) if playback.text.parts: playback.span.log(output=[{"role": "assistant", "content": str(playback.text)}]) playback.span.end(end_time=self._clock.wall(end_ms)) diff --git a/py/src/braintrust/integrations/pipecat/tracing/_live.py b/py/src/braintrust/integrations/pipecat/tracing/_live.py new file mode 100644 index 00000000..d313ff84 --- /dev/null +++ b/py/src/braintrust/integrations/pipecat/tracing/_live.py @@ -0,0 +1,482 @@ +"""Prototype GPT-Live tracing: continuous conversation and hosted Responses. + +The voice service's LLM/TTS frames are transcript grouping notifications, not +provider calls. Public frames own transcript groups; native events own hosted +Responses. The ordinary observer still owns transport audio and speech. This +intentionally supports one Live service per pipeline and hosted Responses +delegation first. +""" + +import math +from collections import OrderedDict +from copy import deepcopy +from dataclasses import dataclass, field +from enum import Enum, auto +from typing import Any, Literal + +from braintrust._audio.clock import SessionClock +from braintrust._audio.schema import SOURCE_ID +from braintrust.integrations.openai.tracing import _parse_metrics_from_usage +from braintrust.logger import Span, SpanTypeAttribute + +from ._common import MAX_PENDING, SpanFactory, metadata_from_processor, tools_schema_to_openai_tools + + +# Prototype pricing bridge until the backend prices duration-based models. +# Public USD/minute rate verified 2026-11-16. Do not price unknown models. +_LIVE_USD_PER_MINUTE = {"gpt-live-1": 0.05} +_LIVE_PRICING_SOURCE = "https://developers.openai.com/api/docs/models/gpt-live-1" + + +def is_live_service(processor: Any) -> bool: + return any(base.__name__ == "OpenAILiveLLMService" for base in type(processor).__mro__) + + +class _LivePhase(Enum): + ACTIVE = auto() + DRAINING = auto() # Remote model ended; queued transcript frames may still arrive. + CLOSED = auto() # Pipeline cleanup finished. + + +class _ResponsePhase(Enum): + STREAMING = auto() + PROCESSING = auto() # Inference ended; Pipecat is handling its terminal event. + WAITING_FOR_TOOLS = auto() + + +_Role = Literal["user", "assistant"] +_ToolSignal = Literal["submission", "span"] + + +@dataclass +class _Turn: + span: Span + transcript: Span | None = None + text: list[str] = field(default_factory=list) + start_ms: int | None = None + end_ms: int | None = None + + +@dataclass +class _Delegation: + response_id: str | None = None + offset_ms: int | None = None + + +@dataclass +class _PendingTool: + response_id: str + # Submission and observation run in different tasks and can arrive in + # either order. Neither alone proves that the response task can close. + waiting_for: set[_ToolSignal] = field(default_factory=lambda: {"submission", "span"}) + + +@dataclass +class _Response: + wrapper: Span + llm: Span + start_ms: float + output: dict[str, Any] = field(default_factory=dict) + first_content: bool = False + phase: _ResponsePhase = _ResponsePhase.STREAMING + pending_tools: set[str] = field(default_factory=set) + + @property + def ready(self) -> bool: + return self.phase is _ResponsePhase.WAITING_FOR_TOOLS and not self.pending_tools + + def finish_inference(self, kind: str, snapshot: dict[str, Any], end_time: float) -> None: + if self.phase is not _ResponsePhase.STREAMING: + return + if snapshot.get("model"): + self.llm.log(metadata={"model": snapshot["model"]}) + self.llm.log(metrics=_parse_metrics_from_usage(snapshot.get("usage"))) + if kind != "response.completed": + error = snapshot.get("error") or snapshot.get("incomplete_details") or {"status": kind} + self.llm.log(error=error) + self.wrapper.log(error=error) + self.llm.log(output=list(self.output.values())) + self.llm.end(end_time=end_time) + self.phase = _ResponsePhase.PROCESSING + + def record_output(self, event: dict[str, Any]) -> None: + kind = event.get("type") + if kind in ("response.output_item.added", "response.output_item.done"): + item = event.get("item") or {} + key = item.get("id") or str(event.get("output_index", len(self.output))) + # Deltas update our snapshot, never the event Pipecat is handling. + self.output[key] = deepcopy(item) + return + item = self.output.get(event.get("item_id") or str(event.get("output_index"))) + if item is None: + return + if kind == "response.function_call_arguments.delta": + item["arguments"] = item.get("arguments", "") + event["delta"] + elif kind == "response.function_call_arguments.done": + item["arguments"] = event["arguments"] + elif kind in ("response.content_part.added", "response.content_part.done"): + content = item.setdefault("content", []) + index = event["content_index"] + if index == len(content): + content.append(deepcopy(event["part"])) + elif 0 <= index < len(content): + content[index] = deepcopy(event["part"]) + elif kind in ("response.output_text.delta", "response.output_text.done"): + content = item.get("content", []) + index = event["content_index"] + if 0 <= index < len(content): + part = content[index] + part["text"] = part.get("text", "") + event["delta"] if kind.endswith(".delta") else event["text"] + + +class LiveTracker: + """Observe two independent lifecycles without making either one current. + + The remote model goes ACTIVE -> DRAINING on session.closed; only pipeline + cleanup makes it CLOSED, after queued transcript frames have drained. + + Each hosted response goes STREAMING -> PROCESSING -> WAITING_FOR_TOOLS, + then is removed once both tool submissions and tool spans have finished. + Cleanup ends any remaining work as incomplete. Response/tool indexes own + active work, and at most two transcript turns are open. Delegation history + keeps recent IDs/offsets plus active responses' routing. Full session + transcripts grow until cleanup by design. + """ + + def __init__(self, session: Any, service: Any) -> None: + self.session, self.service = session, service + session.agent.report_uninterrupted_playback = False + # Native callbacks and frame callbacks run in different tasks. Do not + # share the frame observer's temporarily pinned event clock. + self.clock = SessionClock(session.recorder) + self.spans = SpanFactory(session.span, self.clock) + self.frame_spans = SpanFactory(session.span, session.clock) + self.origin_ms = self.clock.now_ms() + self.turns: dict[_Role, _Turn] = {} + self.responses: dict[str, _Response] = {} + self.delegations: OrderedDict[str, _Delegation] = OrderedDict() + self.tools: dict[str, _PendingTool] = {} + self.phase = _LivePhase.ACTIVE + self.inputs: list[Any] = [] + self.outputs: list[dict[str, str]] = [] + self.first_content = False + self.model_name: str | None = None + self.model = self.spans.start( + "pipecat.live", + type=SpanTypeAttribute.LLM, + metadata={**metadata_from_processor(service), "audio.scope": "session"}, + ) + session.span.log(metadata={"contrib.pipecat.mode": "live"}) + + def event(self, event: Any) -> None: + if self.phase is _LivePhase.CLOSED: + return + kind = event.type + if kind == "session.started": + self.origin_ms = self.clock.now_ms() + resource = event.session + self.model_name = resource.model + metadata = { + "provider": "openai", + "model": resource.model, + "contrib.pipecat.live.session_id": resource.id, + } + self.model.log( + metadata={ + **metadata, + "contrib.pipecat.live.input_partial": True, + "contrib.pipecat.live.transcript_grouping": "pipecat_transcript_gap", + } + ) + self.session.span.log(metadata=metadata) + context = getattr(self.service, "_context", None) + self.inputs = list(context.get_messages()) if context is not None else [] + if resource.instructions: + # Pipecat consumes a leading system message as instructions; + # the server reports the effective value after settings override it. + if self.inputs and isinstance(self.inputs[0], dict) and self.inputs[0].get("role") == "system": + self.inputs.pop(0) + self.inputs.insert(0, {"role": "system", "content": resource.instructions}) + self.model.log(input=self.inputs) + elif kind in ("session.output_audio.delta", "session.output_transcript.delta"): + if event.delta: + self._first_content() + elif kind == "session.delegation.created": + delegation = self._delegation(event.delegation.id) + delegation.offset_ms = event.offset_ms + delegation.response_id = event.delegation.response_id + elif kind == "response.event": + self._response(event) + elif kind in ("session.usage.updated", "session.closed"): + usage = getattr(event, "usage", None) + if usage is not None and usage.seconds is not None: + self._usage(usage.seconds) + if kind == "session.closed": + self.model.log(metadata={"contrib.pipecat.live.close_reason": event.reason}) + # Frame observations are queued separately. Closing the remote + # model must not discard transcript frames already in flight. + self._end_model(confirmed=True) + elif kind == "error": + self.model.log(error=event.error.message) + + def _delegation(self, delegation_id: str | None) -> _Delegation: + key = delegation_id or "uncorrelated" + if key not in self.delegations: + self.delegations[key] = _Delegation() + self.delegations.move_to_end(key) + self._trim_delegations(keep=key) + return self.delegations[key] + + def _trim_delegations(self, *, keep: str | None = None) -> None: + # Live has no delegation-finished notification. Bound historical IDs + # and offsets, while keeping routing needed by an active response. + if len(self.delegations) <= MAX_PENDING: + return + for key, delegation in list(self.delegations.items()): + if key != keep and delegation.response_id not in self.responses: + del self.delegations[key] + if len(self.delegations) <= MAX_PENDING: + break + + def _first_content(self) -> None: + if not self.first_content: + self.first_content = True + self.model.log(metrics={"time_to_first_token": (self.clock.now_ms() - self.origin_ms) / 1000}) + + def _usage(self, seconds: float) -> None: + if not math.isfinite(seconds) or seconds < 0: + return + # Each provider update is cumulative: replace the estimate, never sum + # updates or substitute local wall time for the provider's billed time. + metadata: dict[str, Any] = {"contrib.pipecat.live.usage_seconds": seconds} + rate = _LIVE_USD_PER_MINUTE.get(self.model_name or "") + if rate is not None: + metadata.update( + { + "contrib.pipecat.live.pricing": { + "usd_per_minute": rate, + "source": _LIVE_PRICING_SOURCE, + "as_of": "2026-11-16", + } + } + ) + self.model.log(metrics={"estimated_cost": seconds * rate / 60}) + self.model.log(metadata=metadata) + + def frame(self, frame: Any) -> None: + if self.phase is _LivePhase.CLOSED: + return + kind = type(frame).__name__ + if kind == "ProposedUserStartedSpeakingFrame": + self._open_turn("user") + elif kind == "LLMFullResponseStartFrame": + self._open_turn("assistant") + elif kind in ("InterimTranscriptionFrame", "TranscriptionFrame"): + # User frames contain the accumulated group, not a text delta. + self._transcript("user", frame.text, replace=True, event=getattr(frame, "result", None)) + elif kind == "LLMTextFrame": + self._transcript("assistant", frame.text) + elif kind == "ProposedUserStoppedSpeakingFrame": + self._end_turn("user") + elif kind == "LLMFullResponseEndFrame": + self._end_turn("assistant") + + def _open_turn(self, role: _Role) -> None: + if role in self.turns: + return + turn = self.turns[role] = _Turn(self.frame_spans.turn("user_turn" if role == "user" else "agent_turn")) + turn.span.log(metadata={"contrib.pipecat.live.grouping": "pipecat_transcript_gap"}) + + def _transcript(self, role: _Role, text: str, *, replace: bool = False, event: Any = None) -> None: + turn = self.turns.get(role) + if not text or turn is None: + return + if turn.transcript is None: + turn.transcript = self.frame_spans.start( + "transcript_received", + turn=turn.span, + metadata={ + SOURCE_ID: "session-input" if role == "user" else "agent-output", + "contrib.pipecat.live.grouping": "pipecat_transcript_gap", + }, + ) + if replace: + turn.text = [text] + else: + turn.text.append(text) + # User interim frames carry the original event; assistant text frames + # do not. Retain provider timestamps only when attached to this frame. + start_ms, end_ms = getattr(event, "start_ms", None), getattr(event, "end_ms", None) + if start_ms is not None: + turn.start_ms = start_ms if turn.start_ms is None else min(turn.start_ms, start_ms) + if end_ms is not None: + turn.end_ms = end_ms if turn.end_ms is None else max(turn.end_ms, end_ms) + turn.transcript.log(output=[{"role": role, "content": "".join(turn.text)}]) + if turn.start_ms is not None and turn.end_ms is not None: + turn.transcript.log( + metadata={ + "contrib.pipecat.live.start_ms": turn.start_ms, + "contrib.pipecat.live.end_ms": turn.end_ms, + } + ) + + def _end_turn(self, role: _Role, *, incomplete: bool = False, end_ms: float | None = None) -> None: + turn = self.turns.pop(role, None) + if turn is None: + return + end_time = self.session.clock.wall(end_ms) + if turn.transcript is not None: + message = {"role": role, "content": "".join(turn.text)} + if role == "assistant": + self.outputs.append(message) + self.model.log(output=self.outputs) + else: + self.inputs.append(message) + self.model.log(input=self.inputs) + turn.transcript.end(end_time=end_time) + if incomplete: + turn.span.log(metadata={"turn.incomplete": True}) + turn.span.end(end_time=end_time) + + def client_event(self, event: Any) -> None: + if self.phase is _LivePhase.CLOSED or event.type != "response.item.create": + return + item = event.item + kind = item.get("type") if isinstance(item, dict) else getattr(item, "type", None) + if kind == "function_call_output": + call_id = item.get("call_id") if isinstance(item, dict) else getattr(item, "call_id", None) + # Observed submission, not server acknowledgment: Live doesn't + # acknowledge these items. Never infer which continuation uses it. + self.tool_result_submitted(call_id) + + def _response(self, envelope: Any) -> None: + event = envelope.event + kind = event.get("type", "") + snapshot = event.get("response") or {} + delegation = self._delegation(envelope.delegation_id) + response_id = snapshot.get("id") or event.get("response_id") or delegation.response_id + if not response_id: + return + delegation.response_id = response_id + response = self.responses.get(response_id) + if response is None: + # Missing an observed start is preferable to inventing one. + if kind != "response.created": + return + settings = getattr(getattr(self.service, "_delegation", None), "settings", None) + metadata = { + "provider": "openai", + "model": snapshot.get("model") or getattr(settings, "model", None), + "contrib.pipecat.live.response_id": response_id, + "contrib.pipecat.live.delegation_id": envelope.delegation_id, + "contrib.pipecat.live.input_partial": True, + } + if snapshot.get("previous_response_id"): + metadata["contrib.pipecat.live.previous_response_id"] = snapshot["previous_response_id"] + if delegation.offset_ms is not None: + # This locates delegation creation on the provider's session + # timeline, including for subsequent responses in that delegation. + metadata["contrib.pipecat.live.delegation_offset_ms"] = delegation.offset_ms + context = getattr(self.service, "_context", None) + tools = tools_schema_to_openai_tools(getattr(context, "tools", None)) + if tools: + metadata["tools"] = tools + instruction = getattr(settings, "system_instruction", None) + inputs = [{"role": "system", "content": instruction}] if isinstance(instruction, str) else None + wrapper = self.spans.start("llm_response", input=inputs, metadata=metadata) + llm = self.spans.start( + "pipecat.live.backend_response", + parent=wrapper, + type=SpanTypeAttribute.LLM, + input=inputs, + metadata=metadata, + ) + response = self.responses[response_id] = _Response(wrapper, llm, self.clock.now_ms()) + response.record_output(event) + if not response.first_content and ( + (kind.endswith(".delta") and event.get("delta")) or kind == "response.output_item.done" + ): + response.first_content = True + response.llm.log(metrics={"time_to_first_token": (self.clock.now_ms() - response.start_ms) / 1000}) + if kind == "response.output_item.done": + item = event.get("item") or {} + if item.get("type") == "function_call" and item.get("status") == "completed" and item.get("call_id"): + call_id = item["call_id"] + response.pending_tools.add(call_id) + self.tools[call_id] = _PendingTool(response_id) + self.session.agent.link_tool_request(call_id, response.wrapper) + elif kind in ("response.completed", "response.failed", "response.incomplete"): + # Provider inference ends here. Pipecat may still be executing + # tools and submitting their results as part of this response. + response.finish_inference(kind, snapshot, self.clock.wall()) + + def event_processed(self, envelope: Any) -> None: + if self.phase is _LivePhase.CLOSED or envelope.type != "response.event": + return + event = envelope.event + if event.get("type") not in ("response.completed", "response.failed", "response.incomplete"): + return + response_id = (event.get("response") or {}).get("id") or self._delegation(envelope.delegation_id).response_id + if response_id is None: + return + response = self.responses.get(response_id) + if response is not None and response.phase is _ResponsePhase.PROCESSING: + response.phase = _ResponsePhase.WAITING_FOR_TOOLS + self._finish_if_ready(response_id, response) + + def tool_result_submitted(self, call_id: str) -> None: + self._tool_progress(call_id, "submission") + + def tool_finished(self, call_id: str) -> None: + self._tool_progress(call_id, "span") + + def _tool_progress(self, call_id: str, signal: _ToolSignal) -> None: + tool = self.tools.get(call_id) + if tool is None: + return + tool.waiting_for.discard(signal) + if tool.waiting_for: + return + del self.tools[call_id] + response = self.responses.get(tool.response_id) + if response is not None: + response.pending_tools.discard(call_id) + self._finish_if_ready(tool.response_id, response) + + def _finish_if_ready(self, response_id: str, response: _Response) -> None: + if response.ready: + self._end_response(response_id) + + def _end_response(self, response_id: str, *, incomplete: bool = False, end_ms: float | None = None) -> None: + response = self.responses.pop(response_id) + output = list(response.output.values()) + spans = (response.llm, response.wrapper) if response.phase is _ResponsePhase.STREAMING else (response.wrapper,) + for span in spans: + span.log(output=output) + if incomplete: + span.log(metadata={"contrib.pipecat.incomplete": True}) + span.end(end_time=self.clock.wall(end_ms)) + for call_id in response.pending_tools: + self.tools.pop(call_id, None) + self._trim_delegations() + + def _end_model(self, *, confirmed: bool = False, end_ms: float | None = None) -> None: + if self.phase is not _LivePhase.ACTIVE: + return + self.phase = _LivePhase.DRAINING + self.model.log(metadata={"contrib.pipecat.live.finalization_confirmed": confirmed}) + self.model.end(end_time=self.clock.wall(end_ms)) + + def close(self, end_ms: float) -> None: + if self.phase is _LivePhase.CLOSED: + return + for role in list(self.turns): + self._end_turn(role, incomplete=True, end_ms=end_ms) + for response_id in list(self.responses): + self._end_response(response_id, incomplete=True, end_ms=end_ms) + self._end_model(end_ms=end_ms) + self.phase = _LivePhase.CLOSED + self.delegations.clear() + self.tools.clear() + self.inputs.clear() + self.outputs.clear() diff --git a/py/src/braintrust/integrations/pipecat/tracing/observer.py b/py/src/braintrust/integrations/pipecat/tracing/observer.py index fdad560d..03e134c0 100644 --- a/py/src/braintrust/integrations/pipecat/tracing/observer.py +++ b/py/src/braintrust/integrations/pipecat/tracing/observer.py @@ -19,6 +19,7 @@ from ._agent import AgentTracker from ._common import LOG, SpanFactory, is_pipeline_sink, processor_kind, start_span from ._context import ContextLinker +from ._live import LiveTracker, is_live_service from ._user import UserTracker @@ -55,6 +56,8 @@ async def cleanup(self) -> None: _UPSTREAM_IGNORED_FRAME_TYPES = { "UserStartedSpeakingFrame", "UserStoppedSpeakingFrame", + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", "VADUserStartedSpeakingFrame", "VADUserStoppedSpeakingFrame", "BotStartedSpeakingFrame", @@ -81,6 +84,8 @@ class _Session: agent: AgentTracker # Without playback, a model response end is the agent completion boundary. has_output_transport: bool = False + live: LiveTracker | None = None + live_mode: bool = False class _FrameAdmission: @@ -204,6 +209,36 @@ async def cleanup(self) -> None: await self._finish() await super().cleanup() + def _live_event(self, service: Any, event: Any) -> None: + if self._phase is _SessionPhase.CLOSED: + return + try: + session = self._start_session() + if session.live is None: + # The model span represents an observed remote session, not + # whichever notification happens to arrive first. + if event.type != "session.started": + return + session.live_mode = True + session.live = LiveTracker(session, service) + session.live.event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live event", exc_info=True) + + def _live_event_processed(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.event_processed(event) + except Exception: + LOG.warning("Unable to finish Pipecat Live response processing", exc_info=True) + + def _live_client_event(self, event: Any) -> None: + try: + if self._session is not None and self._session.live is not None: + self._session.live.client_event(event) + except Exception: + LOG.warning("Unable to trace Pipecat Live client event", exc_info=True) + async def _response_span(self, frame: "Frame", push: Callable[[], Awaitable[Any]]) -> tuple[Any, Span | None]: """Push a service's response start frame, then return its span. @@ -318,6 +353,7 @@ async def _finish(self) -> None: return end_ms = session.clock.now_ms() closers: tuple[Callable[[], None], ...] = ( + lambda: session.live.close(end_ms) if session.live is not None else None, lambda: session.user.close(end_ms), lambda: session.agent.interrupt(interrupted=False, end_ms=end_ms), lambda: session.span.end(end_time=session.clock.wall(end_ms)), @@ -340,6 +376,37 @@ async def _dispatch( self, session: _Session, frame: "Frame", processor: "FrameProcessor | None", *, played: bool ) -> None: kind = type(frame).__name__ + live_service = is_live_service(processor) + if live_service: + # Even without compatible event hooks, never reinterpret Live's + # compatibility frames as cascade inference/TTS or speech ownership. + session.live_mode = True + session.agent.report_uninterrupted_playback = False + if live_service and kind in { + "ProposedUserStartedSpeakingFrame", + "ProposedUserStoppedSpeakingFrame", + "InterimTranscriptionFrame", + "LLMFullResponseStartFrame", + "LLMFullResponseEndFrame", + "LLMTextFrame", + "TTSStartedFrame", + "TTSStoppedFrame", + "TTSTextFrame", + "TranscriptionFrame", + "MetricsFrame", + }: + # Live's public frames delimit transcript groups, not backend + # inference or generated TTS. Speech stays transport-owned. + if session.live is not None and self._admission.admit(frame): + session.live.frame(frame) + return + if session.live_mode and kind in { + "UserStartedSpeakingFrame", + "UserStoppedSpeakingFrame", + "LLMContextUserTurnFrame", + "LLMContextAssistantTurnFrame", + }: + return action = self._FRAME_ACTIONS.get(kind) if action is None or (kind in _TERMINAL_FRAME_TYPES and not is_pipeline_sink(processor)): return @@ -353,7 +420,7 @@ def _observe_playback(self, session: _Session, frame: "Frame") -> None: if isinstance(frame, frames.OutputAudioRawFrame): self._record_audio(session, "agent-output", frame) session.agent.played_audio(frame) - elif isinstance(frame, frames.TTSTextFrame): + elif isinstance(frame, frames.TTSTextFrame) and not session.live_mode: session.agent.played_text(frame) def _record_audio(self, session: _Session, source: _AudioSource, frame: "AudioRawFrame") -> None: @@ -470,11 +537,17 @@ def _on_tool_result( self, s: _Session, frame: "frames.FunctionCallResultFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.finish_tool(frame, s.linker) + if s.live is not None and getattr(getattr(frame, "properties", None), "is_final", True): + # Frame observation is queued separately from Live's result + # submission. Both must finish before the parent task closes. + s.live.tool_finished(frame.tool_call_id) def _on_tool_cancelled( self, s: _Session, frame: "frames.FunctionCallCancelFrame", _p: "FrameProcessor | None", _played: bool ) -> None: s.agent.cancel_tool(frame.tool_call_id) + if s.live is not None: + s.live.tool_finished(frame.tool_call_id) def _on_tts_started( self, s: _Session, frame: "frames.TTSStartedFrame", processor: "FrameProcessor | None", _played: bool