fix: prevent voice preferences from being spoken

agent/agent.py

@@ -13,7 +13,7 @@
 from livekit.agents import llm, stt
 from livekit.plugins import openai
 
-from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, speech_prompt, spoken_text
+from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, speech_config, spoken_text
 
 log = logging.getLogger(__name__)
 server = agents.AgentServer()
@@ -42,12 +42,30 @@ def __init__(self, ctx: agents.JobContext) -> None:
         self.empty_task: asyncio.Task[None] | None = None
         self.agent_audio_source = rtc.AudioSource(24000, 1)
         self.agent_audio_track = rtc.LocalAudioTrack.create_audio_track("accessibility-agent-audio", self.agent_audio_source)
-        self.realtime_model = openai.realtime.RealtimeModel(
-            model=os.environ["OPENAI_REALTIME_MODEL"], modalities=["audio", "text"])
-        self.realtime_session = self.realtime_model.session()
+        self.realtime_models: dict[tuple[str, float], openai.realtime.RealtimeModel] = {}
+        self.realtime_sessions = {}
         self.transcriber = openai.STT(
             language="ko", model=os.environ["OPENAI_TRANSCRIBE_MODEL"], use_realtime=True)
 
+    async def speech_session(self, attributes: dict[str, str]):
+        voice, speed = speech_config(attributes)
+        key = (voice, speed)
+        session = self.realtime_sessions.get(key)
+        if session is None:
+            model = openai.realtime.RealtimeModel(
+                model=os.environ["OPENAI_REALTIME_MODEL"],
+                modalities=["audio", "text"],
+                voice=voice,
+                speed=speed,
+            )
+            session = model.session()
+            await session.update_instructions(
+                "사용자가 입력한 한국어 문장을 답하거나 요약하거나 수정하지 말고 원문만 그대로 읽으세요."
+            )
+            self.realtime_models[key] = model
+            self.realtime_sessions[key] = session
+        return session
+
     def spawn(self, coroutine: Awaitable[None]) -> None:
         task = asyncio.create_task(coroutine)
         self.tasks.add(task)
@@ -206,10 +224,11 @@ async def speak(reader: rtc.TextStreamReader, identity: str) -> None:
             started = time.monotonic()
             first_frame = True
             try:
+                session = await state.speech_session(attributes)
                 chat = llm.ChatContext()
-                chat.add_message(role="user", content=speech_prompt(text, attributes))
-                await state.realtime_session.update_chat_ctx(chat)
-                generation = await state.realtime_session.generate_reply()
+                chat.add_message(role="user", content=text)
+                await session.update_chat_ctx(chat)
+                generation = await session.generate_reply()
                 async for message in generation.message_stream:
                     async for frame in message.audio_stream:
                         await state.agent_audio_source.capture_frame(frame)
@@ -296,7 +315,6 @@ def on_track_subscription_failed(
     # Initial tracks are reconciled below because connect hydrates them without participant events.
     await ctx.connect(auto_subscribe=agents.AutoSubscribe.AUDIO_ONLY)
     await ctx.room.local_participant.publish_track(state.agent_audio_track)
-    await state.realtime_session.update_instructions("한국어 입력을 답하거나 요약하거나 수정하지 말고, 입력 원문만 그대로 읽으세요.")
 
     for participant in ctx.room.remote_participants.values():
         state.accept(participant)
@@ -314,10 +332,12 @@ async def cleanup(*_: object) -> None:
             state.empty_task.cancel()
         with contextlib.suppress(Exception):
             await state.transcriber.aclose()
-        with contextlib.suppress(Exception):
-            await state.realtime_session.aclose()
-        with contextlib.suppress(Exception):
-            await state.realtime_model.aclose()
+        for session in state.realtime_sessions.values():
+            with contextlib.suppress(Exception):
+                await session.aclose()
+        for model in state.realtime_models.values():
+            with contextlib.suppress(Exception):
+                await model.aclose()
         with contextlib.suppress(Exception):
             await state.agent_audio_source.aclose()
 

agent/logic.py

@@ -40,20 +40,20 @@ def spoken_text(text: str) -> str | None:
     return value if 0 < len(value) <= 500 else None
 
 
-def speech_prompt(text: str, attributes: dict[str, str]) -> str:
+def speech_config(attributes: dict[str, str]) -> tuple[str, float]:
     voices = {
-        "CALM": "차분하고 편안한 목소리로",
-        "BRIGHT": "밝고 친근한 목소리로",
-        "CLEAR": "또박또박 명료한 목소리로",
+        "CALM": "cedar",
+        "BRIGHT": "coral",
+        "CLEAR": "marin",
     }
     speeds = {
-        "SLOW": "평소보다 천천히",
-        "NORMAL": "자연스러운 속도로",
-        "FAST": "평소보다 빠르게",
+        "SLOW": 0.85,
+        "NORMAL": 1.0,
+        "FAST": 1.15,
     }
     voice = voices.get(attributes.get("openwindow.voice", ""), voices["CALM"])
     speed = speeds.get(attributes.get("openwindow.voice_speed", ""), speeds["NORMAL"])
-    return f"다음 문장만 그대로, {voice} {speed} 읽어 주세요: {text}"
+    return voice, speed
 
 
 def caption_attributes(segment_id: str, final: bool, track_id: str) -> dict[str, str]:

agent/tests/test_logic.py

@@ -1,7 +1,7 @@
 import asyncio
 
 from agent import RoomState
-from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, mode_from_metadata, speech_prompt, spoken_text
+from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, mode_from_metadata, speech_config, spoken_text
 
 
 def test_admission_reserves_one_slot_and_one_mode_per_identity():
@@ -31,17 +31,18 @@ def test_text_bound_and_caption_attributes():
     }
 
 
-def test_speech_prompt_uses_preferences_and_safe_defaults():
-    configured = speech_prompt(
-        "안녕하세요", {"openwindow.voice": "CLEAR", "openwindow.voice_speed": "SLOW"}
+def test_speech_config_uses_voice_ids_and_speed_values_with_safe_defaults():
+    voice, speed = speech_config(
+        {"openwindow.voice": "CLEAR", "openwindow.voice_speed": "SLOW"}
     )
-    assert "또박또박 명료한" in configured
-    assert "평소보다 천천히" in configured
-    assert configured.endswith("안녕하세요")
+    assert voice == "marin"
+    assert speed == 0.85
 
-    defaulted = speech_prompt("반갑습니다", {"openwindow.voice": "UNKNOWN"})
-    assert "차분하고 편안한" in defaulted
-    assert "자연스러운 속도" in defaulted
+    default_voice, default_speed = speech_config(
+        {"openwindow.voice": "UNKNOWN", "openwindow.voice_speed": "UNKNOWN"}
+    )
+    assert default_voice == "cedar"
+    assert default_speed == 1.0
 
 
 def test_track_cleanup_cancels_the_transcription_task():