diff --git a/agent/agent.py b/agent/agent.py index 32b5fb2..732c5a9 100644 --- a/agent/agent.py +++ b/agent/agent.py @@ -13,7 +13,7 @@ from livekit.agents import llm, stt from livekit.plugins import openai -from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, speech_prompt, spoken_text +from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, speech_config, spoken_text log = logging.getLogger(__name__) server = agents.AgentServer() @@ -42,12 +42,30 @@ def __init__(self, ctx: agents.JobContext) -> None: self.empty_task: asyncio.Task[None] | None = None self.agent_audio_source = rtc.AudioSource(24000, 1) self.agent_audio_track = rtc.LocalAudioTrack.create_audio_track("accessibility-agent-audio", self.agent_audio_source) - self.realtime_model = openai.realtime.RealtimeModel( - model=os.environ["OPENAI_REALTIME_MODEL"], modalities=["audio", "text"]) - self.realtime_session = self.realtime_model.session() + self.realtime_models: dict[tuple[str, float], openai.realtime.RealtimeModel] = {} + self.realtime_sessions = {} self.transcriber = openai.STT( language="ko", model=os.environ["OPENAI_TRANSCRIBE_MODEL"], use_realtime=True) + async def speech_session(self, attributes: dict[str, str]): + voice, speed = speech_config(attributes) + key = (voice, speed) + session = self.realtime_sessions.get(key) + if session is None: + model = openai.realtime.RealtimeModel( + model=os.environ["OPENAI_REALTIME_MODEL"], + modalities=["audio", "text"], + voice=voice, + speed=speed, + ) + session = model.session() + await session.update_instructions( + "사용자가 입력한 한국어 문장을 답하거나 요약하거나 수정하지 말고 원문만 그대로 읽으세요." + ) + self.realtime_models[key] = model + self.realtime_sessions[key] = session + return session + def spawn(self, coroutine: Awaitable[None]) -> None: task = asyncio.create_task(coroutine) self.tasks.add(task) @@ -206,10 +224,11 @@ async def speak(reader: rtc.TextStreamReader, identity: str) -> None: started = time.monotonic() first_frame = True try: + session = await state.speech_session(attributes) chat = llm.ChatContext() - chat.add_message(role="user", content=speech_prompt(text, attributes)) - await state.realtime_session.update_chat_ctx(chat) - generation = await state.realtime_session.generate_reply() + chat.add_message(role="user", content=text) + await session.update_chat_ctx(chat) + generation = await session.generate_reply() async for message in generation.message_stream: async for frame in message.audio_stream: await state.agent_audio_source.capture_frame(frame) @@ -296,7 +315,6 @@ def on_track_subscription_failed( # Initial tracks are reconciled below because connect hydrates them without participant events. await ctx.connect(auto_subscribe=agents.AutoSubscribe.AUDIO_ONLY) await ctx.room.local_participant.publish_track(state.agent_audio_track) - await state.realtime_session.update_instructions("한국어 입력을 답하거나 요약하거나 수정하지 말고, 입력 원문만 그대로 읽으세요.") for participant in ctx.room.remote_participants.values(): state.accept(participant) @@ -314,10 +332,12 @@ async def cleanup(*_: object) -> None: state.empty_task.cancel() with contextlib.suppress(Exception): await state.transcriber.aclose() - with contextlib.suppress(Exception): - await state.realtime_session.aclose() - with contextlib.suppress(Exception): - await state.realtime_model.aclose() + for session in state.realtime_sessions.values(): + with contextlib.suppress(Exception): + await session.aclose() + for model in state.realtime_models.values(): + with contextlib.suppress(Exception): + await model.aclose() with contextlib.suppress(Exception): await state.agent_audio_source.aclose() diff --git a/agent/logic.py b/agent/logic.py index ef92863..8ef3f7e 100644 --- a/agent/logic.py +++ b/agent/logic.py @@ -40,20 +40,20 @@ def spoken_text(text: str) -> str | None: return value if 0 < len(value) <= 500 else None -def speech_prompt(text: str, attributes: dict[str, str]) -> str: +def speech_config(attributes: dict[str, str]) -> tuple[str, float]: voices = { - "CALM": "차분하고 편안한 목소리로", - "BRIGHT": "밝고 친근한 목소리로", - "CLEAR": "또박또박 명료한 목소리로", + "CALM": "cedar", + "BRIGHT": "coral", + "CLEAR": "marin", } speeds = { - "SLOW": "평소보다 천천히", - "NORMAL": "자연스러운 속도로", - "FAST": "평소보다 빠르게", + "SLOW": 0.85, + "NORMAL": 1.0, + "FAST": 1.15, } voice = voices.get(attributes.get("openwindow.voice", ""), voices["CALM"]) speed = speeds.get(attributes.get("openwindow.voice_speed", ""), speeds["NORMAL"]) - return f"다음 문장만 그대로, {voice} {speed} 읽어 주세요: {text}" + return voice, speed def caption_attributes(segment_id: str, final: bool, track_id: str) -> dict[str, str]: diff --git a/agent/tests/test_logic.py b/agent/tests/test_logic.py index 785a2cb..034d6fb 100644 --- a/agent/tests/test_logic.py +++ b/agent/tests/test_logic.py @@ -1,7 +1,7 @@ import asyncio from agent import RoomState -from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, mode_from_metadata, speech_prompt, spoken_text +from logic import Mode, admitted_mode, caption_attributes, is_hearing_microphone_audio, mode_from_metadata, speech_config, spoken_text def test_admission_reserves_one_slot_and_one_mode_per_identity(): @@ -31,17 +31,18 @@ def test_text_bound_and_caption_attributes(): } -def test_speech_prompt_uses_preferences_and_safe_defaults(): - configured = speech_prompt( - "안녕하세요", {"openwindow.voice": "CLEAR", "openwindow.voice_speed": "SLOW"} +def test_speech_config_uses_voice_ids_and_speed_values_with_safe_defaults(): + voice, speed = speech_config( + {"openwindow.voice": "CLEAR", "openwindow.voice_speed": "SLOW"} ) - assert "또박또박 명료한" in configured - assert "평소보다 천천히" in configured - assert configured.endswith("안녕하세요") + assert voice == "marin" + assert speed == 0.85 - defaulted = speech_prompt("반갑습니다", {"openwindow.voice": "UNKNOWN"}) - assert "차분하고 편안한" in defaulted - assert "자연스러운 속도" in defaulted + default_voice, default_speed = speech_config( + {"openwindow.voice": "UNKNOWN", "openwindow.voice_speed": "UNKNOWN"} + ) + assert default_voice == "cedar" + assert default_speed == 1.0 def test_track_cleanup_cancels_the_transcription_task():