From 6e8ef4c8b7566507ebb7f825512aad2dcc64ec22 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Thu, 24 Sep 2026 13:53:13 -0400 Subject: [PATCH 01/34] feat(voice): configure independent speech endpoints --- src-tauri/Cargo.lock | 1 + src-tauri/Cargo.toml | 1 + src-tauri/crates/berd-call/README.md | 5 + src-tauri/crates/berd-call/src/cli_help.rs | 4 +- src-tauri/crates/berd-call/src/main.rs | 236 ++++++++++++++++-- .../berd-call/tests/session_protocol.rs | 11 +- src-tauri/src/commands/mod.rs | 1 + src-tauri/src/commands/openai_audio.rs | 58 ++--- src-tauri/src/commands/openai_realtime.rs | 28 ++- .../src/commands/openai_voice_credentials.rs | 100 +++++++- .../src/commands/openai_voice_endpoints.rs | 181 ++++++++++++++ src-tauri/src/lib.rs | 4 + .../voice-conversation/api/openAiVoice.ts | 28 +++ .../hooks/useOpenAiVoiceSetup.test.tsx | 1 + .../ui/OpenAiEndpointField.tsx | 92 +++++++ .../ui/RealtimeVoiceSettings.test.tsx | 28 ++- .../ui/RealtimeVoiceSettings.tsx | 16 +- .../ui/VoiceSettings.test.tsx | 8 + .../voice-conversation/ui/VoiceSettings.tsx | 9 + src/shared/i18n/locales/en/settings.json | 5 + src/shared/i18n/locales/es/settings.json | 5 + 21 files changed, 744 insertions(+), 78 deletions(-) create mode 100644 src-tauri/src/commands/openai_voice_endpoints.rs create mode 100644 src/features/voice-conversation/ui/OpenAiEndpointField.tsx diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index eeb77ae37..a0f8f502e 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -41,6 +41,7 @@ dependencies = [ "portable-pty", "reqwest 0.13.4", "rodio", + "security-framework 3.7.0", "semver", "serde", "serde_json", diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index 1f7ced02b..dde762384 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -97,6 +97,7 @@ windows-sys = { version = "0.59", features = [ ] } [target.'cfg(target_os = "macos")'.dependencies] +security-framework = "3.7" block2 = "0.6" coreaudio-rs = "0.14.2" keyring = { version = "3.6.3", default-features = false, features = ["apple-native"] } diff --git a/src-tauri/crates/berd-call/README.md b/src-tauri/crates/berd-call/README.md index d706a8499..b2ab8b258 100644 --- a/src-tauri/crates/berd-call/README.md +++ b/src-tauri/crates/berd-call/README.md @@ -111,8 +111,13 @@ berd-call session --tts-backend openai --rate 1.0 berd-call session --tts-backend pocket --model-dir /path/to/native-voice-v2 --voice george --rate 1.0 berd-call session --stt-backend parakeet --stt-model-dir /path/to/parakeet berd-call session --stt-backend openai +berd-call session --tts-backend openai --mode expert-spokesperson --realtime-url ws://127.0.0.1:18870/v1/realtime +berd-call session --tts-backend openai --tts-url https://proxy.example/v1/audio/speech +berd-call session --stt-backend openai --stt-url wss://proxy.example/v1/realtime?intent=transcription ``` +The three URL flags take full endpoints and override only their matching service for that call. Expert–Spokesperson uses `--realtime-url` for both input and output; `--stt-url` and `--tts-url` apply only to conventional mode. Without an override, each service uses its OpenAI endpoint (or its existing environment override). `OPENAI_API_KEY` supplies the CLI credential; the desktop app stores keys separately by endpoint URL in Keychain. + The default Siri backend still requires an exact installed voice name and language. Missing or unavailable Siri voice configuration and an unavailable current-locale macOS speech model fail startup with setup guidance. The session diff --git a/src-tauri/crates/berd-call/src/cli_help.rs b/src-tauri/crates/berd-call/src/cli_help.rs index 154ba0248..157e0a26d 100644 --- a/src-tauri/crates/berd-call/src/cli_help.rs +++ b/src-tauri/crates/berd-call/src/cli_help.rs @@ -111,10 +111,12 @@ Each form also accepts: [--rate FLOAT] [--stt-backend macos|parakeet|openai] [--stt-model-dir PATH] [--mode conventional|expert-spokesperson] + [--realtime-url WS_URL] [--stt-url WS_URL] [--tts-url HTTP_URL] The host owns microphone capture, audio playback, transcript delivery, and agent integration. See PROTOCOL.md for the framed stdin, stdout, and PCM -contracts."#; +contracts. Endpoint URL flags override only the matching service for this call +and default to OpenAI when no environment override is set."#; const SYNTHESIZE_HELP: &str = r#"Render text through a configured TTS backend into a new WAV file. diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 92974a01c..599f1e447 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -184,6 +184,14 @@ struct SessionConfig { tts: TtsBackendConfig, stt: SttBackendConfig, mode: SessionMode, + endpoints: SessionEndpointOverrides, +} + +#[derive(Clone, Debug, Default, PartialEq)] +struct SessionEndpointOverrides { + realtime: Option, + tts: Option, + stt: Option, } #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] @@ -800,7 +808,16 @@ fn parse_saved_start_args_at(args: &[String], path: PathBuf) -> Result Result<(), String abort_active(&active); return Ok(()); } - let slot = match create_tts_slot(&config.tts) { + let slot = match create_tts_slot(&config.tts, config.endpoints.tts.as_deref()) { Ok(slot) => Arc::new(slot), Err(message) => { write_protocol_fatal( @@ -2059,17 +2076,18 @@ fn run_session(config: SessionConfig, pcm_output_fd: RawFd) -> Result<(), String return Ok(()); } }; - let (runtime, mut events) = match create_input_runtime(&config.stt) { - Ok(runtime) => runtime, - Err(message) => { - write_protocol_fatal( - &mut writer, - &public_stt_startup_error(&config.stt), - &format!("STT startup failed: {message}"), - )?; - return Ok(()); - } - }; + let (runtime, mut events) = + match create_input_runtime(&config.stt, config.endpoints.stt.as_deref()) { + Ok(runtime) => runtime, + Err(message) => { + write_protocol_fatal( + &mut writer, + &public_stt_startup_error(&config.stt), + &format!("STT startup failed: {message}"), + )?; + return Ok(()); + } + }; let readiness = wait_for_input_ready(&mut events, INPUT_STARTUP_TIMEOUT); if let Err(message) = readiness { runtime.cancel(); @@ -4005,6 +4023,9 @@ fn run_expert_spokesperson_session( break; } let mut spokesperson_config = OpenAiSpokespersonConfig::from_environment()?; + if let Some(url) = &config.endpoints.realtime { + spokesperson_config.endpoint.clone_from(url); + } apply_spokesperson_startup_settings(&config, &mut spokesperson_config)?; let tts = berd_call::TtsConfigurationSnapshot { revision: 1, @@ -5229,6 +5250,7 @@ fn parse_args(args: &[String]) -> Result { let mut stt_backend = "macos"; let mut stt_model_dir = None; let mut mode = SessionMode::Conventional; + let mut endpoints = SessionEndpointOverrides::default(); let mut index = 2; while index < args.len() { let flag = args[index].as_str(); @@ -5252,6 +5274,11 @@ fn parse_args(args: &[String]) -> Result { } "--stt-backend" => stt_backend = value, "--stt-model-dir" => stt_model_dir = Some(PathBuf::from(value)), + "--realtime-url" => { + endpoints.realtime = Some(parse_endpoint_url(value, "--realtime-url", true)?) + } + "--stt-url" => endpoints.stt = Some(parse_endpoint_url(value, "--stt-url", true)?), + "--tts-url" => endpoints.tts = Some(parse_endpoint_url(value, "--tts-url", false)?), "--mode" => { mode = match value.as_str() { "conventional" => SessionMode::Conventional, @@ -5266,7 +5293,44 @@ fn parse_args(args: &[String]) -> Result { } let tts = build_tts_backend_config(backend, voice, language, model_dir, rate)?; let stt = build_stt_backend_config(stt_backend, stt_model_dir)?; - Ok(SessionConfig { tts, stt, mode }) + if endpoints.realtime.is_some() && mode != SessionMode::ExpertSpokesperson { + return Err("--realtime-url requires --mode expert-spokesperson".into()); + } + if mode == SessionMode::ExpertSpokesperson + && (endpoints.tts.is_some() || endpoints.stt.is_some()) + { + return Err("--tts-url and --stt-url apply only to conventional mode; Expert-Spokesperson uses --realtime-url".into()); + } + if endpoints.tts.is_some() && !matches!(tts, TtsBackendConfig::OpenAi { .. }) { + return Err("--tts-url requires --tts-backend openai".into()); + } + if endpoints.stt.is_some() && !matches!(stt, SttBackendConfig::OpenAi) { + return Err("--stt-url requires --stt-backend openai".into()); + } + Ok(SessionConfig { + tts, + stt, + mode, + endpoints, + }) +} + +fn parse_endpoint_url(value: &str, flag: &str, websocket: bool) -> Result { + let url = reqwest::Url::parse(value).map_err(|error| format!("{flag} is invalid: {error}"))?; + let valid_scheme = if websocket { + matches!(url.scheme(), "ws" | "wss") + } else { + matches!(url.scheme(), "http" | "https") + }; + if !valid_scheme + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err(format!("{flag} requires a full URL with the correct protocol and no embedded credentials or fragment")); + } + Ok(url.to_string()) } fn parse_pcm_output_fd(args: &[String]) -> Result { @@ -5868,7 +5932,7 @@ fn create_stt_benchmark_report( &pack, config.runs, config.mode, - || create_input_runtime(&config.stt), + || create_input_runtime(&config.stt, None), ))) } @@ -5937,12 +6001,16 @@ fn stt_benchmark_target(config: &SttBackendConfig) -> Result Result { +fn create_tts_configuration( + config: &TtsBackendConfig, + endpoint: Option<&str>, +) -> Result { match config { TtsBackendConfig::OpenAi { rate } => create_openai_tts_configuration( *rate, std::env::var("OPENAI_TTS_MODEL").unwrap_or_else(|_| "gpt-4o-mini-tts".into()), std::env::var("OPENAI_TTS_VOICE").unwrap_or_else(|_| "marin".into()), + endpoint, ), TtsBackendConfig::Siri { voice, @@ -5970,6 +6038,7 @@ fn create_openai_tts_configuration( rate: f32, model: String, voice: String, + endpoint: Option<&str>, ) -> Result { let api_key = std::env::var("OPENAI_API_KEY") .ok() @@ -5978,7 +6047,9 @@ fn create_openai_tts_configuration( let base = std::env::var("OPENAI_BASE_URL").unwrap_or_else(|_| "https://api.openai.com/v1".into()); Ok(TtsConfiguration::openai( - format!("{}/audio/speech", base.trim_end_matches('/')), + endpoint + .map(str::to_string) + .unwrap_or_else(|| format!("{}/audio/speech", base.trim_end_matches('/'))), api_key, model, voice, @@ -5986,7 +6057,10 @@ fn create_openai_tts_configuration( )) } -fn create_tts_slot(config: &TtsBackendConfig) -> Result { +fn create_tts_slot( + config: &TtsBackendConfig, + endpoint: Option<&str>, +) -> Result { #[cfg(not(target_os = "macos"))] if matches!(config, TtsBackendConfig::Siri { .. }) { return Err( @@ -5994,7 +6068,7 @@ fn create_tts_slot(config: &TtsBackendConfig) -> Result format!( @@ -6005,7 +6079,7 @@ fn create_tts_slot(config: &TtsBackendConfig) -> Result Result, String> { - let slot = create_tts_slot(config)?; + let slot = create_tts_slot(config, None)?; Ok(Arc::clone(slot.lease()?.backend())) } @@ -6016,6 +6090,7 @@ fn create_synthesis_backend(config: &SynthesisTtsConfig) -> Result Result<(), SynthesisFailure fn create_input_runtime( config: &SttBackendConfig, + endpoint_override: Option<&str>, ) -> Result< ( VoiceInputRuntime, @@ -6229,9 +6305,13 @@ fn create_input_runtime( .ok() .filter(|key| !key.trim().is_empty()) .ok_or_else(|| "OPENAI_API_KEY is required for OpenAI STT".to_string())?; - let endpoint = std::env::var("OPENAI_REALTIME_ENDPOINT") - .ok() - .filter(|value| !value.trim().is_empty()) + let endpoint = endpoint_override + .map(str::to_string) + .or_else(|| { + std::env::var("OPENAI_REALTIME_ENDPOINT") + .ok() + .filter(|value| !value.trim().is_empty()) + }) .unwrap_or_else(|| { "wss://api.openai.com/v1/realtime?intent=transcription".to_string() }); @@ -9358,6 +9438,7 @@ mod tests { }, stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); @@ -9367,6 +9448,7 @@ mod tests { tts: TtsBackendConfig::OpenAi { rate: 1.0 }, stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); } @@ -9442,6 +9524,7 @@ mod tests { }, stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); assert!(parse_args(&args(&[ @@ -9496,6 +9579,7 @@ mod tests { }, stt: SttBackendConfig::Macos, mode: SessionMode::ExpertSpokesperson, + endpoints: SessionEndpointOverrides::default(), }; let mut realtime = OpenAiSpokespersonConfig { endpoint: "ws://localhost".into(), @@ -9516,6 +9600,88 @@ mod tests { assert_eq!(realtime.voice(), "marin"); } + #[test] + fn session_endpoint_overrides_are_independent_and_require_matching_modes() { + let realtime = parse_args(&args(&[ + "berd-call", + "session", + "--tts-backend", + "openai", + "--mode", + "expert-spokesperson", + "--realtime-url", + "ws://127.0.0.1:18870/v1/realtime", + ])) + .unwrap(); + assert_eq!( + realtime.endpoints.realtime.as_deref(), + Some("ws://127.0.0.1:18870/v1/realtime") + ); + assert_eq!(realtime.endpoints.stt, None); + assert_eq!(realtime.endpoints.tts, None); + + let chained = parse_args(&args(&[ + "berd-call", + "session", + "--tts-backend", + "openai", + "--stt-backend", + "openai", + "--tts-url", + "https://proxy.example/v1/audio/speech?api-version=1", + "--stt-url", + "wss://proxy.example/v1/realtime?intent=transcription", + ])) + .unwrap(); + assert_eq!( + chained.endpoints.tts.as_deref(), + Some("https://proxy.example/v1/audio/speech?api-version=1") + ); + assert_eq!( + chained.endpoints.stt.as_deref(), + Some("wss://proxy.example/v1/realtime?intent=transcription") + ); + assert!(parse_args(&args(&[ + "berd-call", + "session", + "--tts-backend", + "openai", + "--realtime-url", + "ws://localhost/realtime" + ])) + .unwrap_err() + .contains("expert-spokesperson")); + assert!(parse_args(&args(&[ + "berd-call", + "session", + "--tts-url", + "wss://localhost/audio/speech" + ])) + .is_err()); + assert!(parse_args(&args(&[ + "berd-call", + "session", + "--tts-backend", + "openai", + "--stt-url", + "wss://localhost/realtime" + ])) + .unwrap_err() + .contains("--stt-backend openai")); + assert!(parse_args(&args(&[ + "berd-call", + "session", + "--tts-backend", + "openai", + "--mode", + "expert-spokesperson", + "--tts-url", + "https://localhost/audio/speech" + ])) + .unwrap_err() + .contains("only to conventional mode")); + } + #[test] fn cli_requires_explicit_pocket_bundle_and_voice() { assert_eq!( @@ -9538,6 +9704,7 @@ mod tests { }, stt: SttBackendConfig::Macos, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); assert!(parse_args(&args(&[ @@ -9598,6 +9765,7 @@ mod tests { model_dir: PathBuf::from("/models/parakeet") }, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); assert!(parse_args(&args(&[ @@ -10041,6 +10209,7 @@ mod tests { }, stt: SttBackendConfig::OpenAi, mode: SessionMode::Conventional, + endpoints: SessionEndpointOverrides::default(), } ); } @@ -11123,6 +11292,27 @@ mod tests { let saved = changed.saved.unwrap().1; assert_eq!(saved.tts.unwrap().rate(), 1.2); assert!(!saved.arguments.contains(&"--codex".into())); + let with_endpoint = parse_saved_start_args_at( + &args(&[ + "berd-call", + "start", + "--stt-backend", + "openai", + "--stt-url", + "wss://proxy.example/v1/realtime?intent=transcription", + ]), + directory.path().join("settings.json"), + ) + .unwrap(); + assert!(with_endpoint + .session_arguments + .contains(&"--stt-url".into())); + assert!(!with_endpoint + .saved + .unwrap() + .1 + .arguments + .contains(&"--stt-url".into())); } #[test] diff --git a/src-tauri/crates/berd-call/tests/session_protocol.rs b/src-tauri/crates/berd-call/tests/session_protocol.rs index 9960af86c..89c8339a3 100644 --- a/src-tauri/crates/berd-call/tests/session_protocol.rs +++ b/src-tauri/crates/berd-call/tests/session_protocol.rs @@ -64,9 +64,16 @@ impl ExpertSpokespersonTestSession { } let mut child = ChildGuard(Some( command - .args(["--mode", "expert-spokesperson", "--tts-backend", "openai"]) + .args([ + "--mode", + "expert-spokesperson", + "--tts-backend", + "openai", + "--realtime-url", + &endpoint, + ]) .env("OPENAI_API_KEY", "test-key") - .env("OPENAI_REALTIME_ENDPOINT", endpoint) + .env("OPENAI_REALTIME_ENDPOINT", "ws://127.0.0.1:1/unused") .env("OPENAI_REALTIME_MODEL", "test-model") .env("OPENAI_REALTIME_VOICE", "old-voice") .stdin(Stdio::piped()) diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index 0a132a071..36af3b2ca 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -40,6 +40,7 @@ pub mod notifications; pub mod openai_audio; pub mod openai_realtime; mod openai_voice_credentials; +pub(crate) mod openai_voice_endpoints; pub mod path_resolver; pub mod pocket_voice; pub mod pr_tracker; diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 3e81d4cdf..c3dabd1f5 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -28,6 +28,7 @@ use super::pocket_voice::{ use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, openai_voice_credentials::{self, OpenAiVoiceCredential}, + openai_voice_endpoints::{self, VoiceEndpointKind}, pocket_voice::VoiceInterruptionMode, voice_capture::VoiceCaptureState, }; @@ -127,6 +128,7 @@ enum OpenAiStreamCommand { pub struct OpenAiVoiceStatus { stt_configured: bool, tts_configured: bool, + realtime_configured: bool, stt_configuration_source: OpenAiVoiceConfigurationSource, tts_configuration_source: OpenAiVoiceConfigurationSource, stt_unavailable_reason: Option, @@ -205,7 +207,7 @@ fn normalize_openai_base_url(raw_url: String) -> Result { Ok(url.to_string().trim_end_matches('/').to_string()) } -fn base_url() -> Result { +pub(crate) fn base_url() -> Result { if let Some(base_url) = env_trimmed(BASE_URL_ENV) { return normalize_openai_base_url(base_url); } @@ -213,20 +215,7 @@ fn base_url() -> Result { } pub(crate) fn realtime_endpoint() -> Result { - let mut url = reqwest::Url::parse(&endpoint("realtime")?) - .map_err(|error| format!("OpenAI realtime endpoint is invalid: {error}"))?; - url.query_pairs_mut().append_pair("intent", "transcription"); - match url.scheme() { - "http" => url.set_scheme("ws").expect("compatible scheme"), - "https" => url.set_scheme("wss").expect("compatible scheme"), - "ws" | "wss" => {} - scheme => { - return Err(format!( - "OpenAI realtime endpoint has unsupported scheme: {scheme}" - )) - } - } - Ok(url.to_string()) + openai_voice_endpoints::effective_url(VoiceEndpointKind::Stt) } pub(crate) fn transcription_model() -> String { @@ -263,11 +252,7 @@ fn stt_configuration_source() -> OpenAiVoiceConfigurationSource { } } -fn endpoint(path: &str) -> Result { - endpoint_for_base_url(&base_url()?, path) -} - -fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { +pub(crate) fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { let mut url = reqwest::Url::parse(base_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; let base_path = url.path().trim_end_matches('/'); @@ -377,21 +362,29 @@ pub async fn get_openai_voice_status( let tts_available = cfg!(target_os = "macos"); let credential_revision = state.credential_revision.load(Ordering::Acquire); let credential_result = tauri::async_runtime::spawn_blocking(move || { - openai_voice_credentials::read(OpenAiVoiceCredential::SpeechToText) + ( + openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText), + openai_voice_credentials::is_present(OpenAiVoiceCredential::TextToSpeech), + openai_voice_credentials::is_present(OpenAiVoiceCredential::Realtime), + ) }) .await .map_err(|error| format!("Could not check OpenAI voice credentials: {error}"))?; - let credential_error = credential_result.as_ref().err().cloned(); - let stt_error = credential_error.clone(); - let tts_error = tts_available.then_some(credential_error).flatten(); - let stt_configured = credential_result.unwrap_or(None).is_some(); - let tts_configured = tts_available && stt_configured; + let (stt_result, tts_result, realtime_result) = credential_result; + let stt_error = stt_result.as_ref().err().cloned(); + let tts_error = tts_available + .then(|| tts_result.as_ref().err().cloned()) + .flatten(); + let stt_configured = stt_result.unwrap_or(false); + let tts_configured = tts_available && tts_result.unwrap_or(false); + let realtime_configured = realtime_result?; if state.credential_revision.load(Ordering::Acquire) == credential_revision { state.configured.store(stt_configured, Ordering::Release); } Ok(OpenAiVoiceStatus { stt_configured, tts_configured, + realtime_configured, stt_configuration_source: stt_configuration_source(), tts_configuration_source: tts_configuration_source(), stt_unavailable_reason: stt_error, @@ -472,7 +465,10 @@ pub async fn set_openai_tts_api_key( stop_openai_voice_inner(&state)?; openai_voice_credentials::store(OpenAiVoiceCredential::TextToSpeech, api_key)?; state.credential_revision.fetch_add(1, Ordering::AcqRel); - state.configured.store(true, Ordering::Release); + state.configured.store( + openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText)?, + Ordering::Release, + ); app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) }) @@ -491,7 +487,10 @@ pub async fn clear_openai_tts_api_key( stop_openai_voice_inner(&state)?; openai_voice_credentials::clear(OpenAiVoiceCredential::TextToSpeech)?; state.credential_revision.fetch_add(1, Ordering::AcqRel); - state.configured.store(false, Ordering::Release); + state.configured.store( + openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText)?, + Ordering::Release, + ); app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) }) @@ -694,6 +693,7 @@ pub fn reset_openai_voice_settings( state: State<'_, OpenAiVoiceState>, ) -> Result<(), String> { let defaults = OpenAiVoiceSettings::default(); + openai_voice_endpoints::reset()?; { let mut playback = state .playback @@ -810,7 +810,7 @@ fn run_openai_voice_stream( voice: String, ) -> Result { let tts = ConfiguredTtsSlot::new(TtsConfiguration::openai( - endpoint("audio/speech")?, + openai_voice_endpoints::effective_url(VoiceEndpointKind::Tts)?, key, speech_model(), voice, diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index f4740b500..abc2287c8 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -20,6 +20,7 @@ use std::{ use tauri::{AppHandle, Emitter, Manager, State, WebviewWindow}; use super::openai_voice_credentials::{self, OpenAiVoiceCredential}; +use super::openai_voice_endpoints::{self, VoiceEndpointKind}; use super::voice_capture::VoiceCaptureState; const OPENAI_REALTIME_CLIENT_SECRETS_URL: &str = @@ -75,17 +76,31 @@ pub struct OpenAiRealtimeSession { client_secret: String, } -fn stored_openai_api_key() -> Result, String> { - openai_voice_credentials::read(OpenAiVoiceCredential::Realtime) -} - #[tauri::command] pub async fn get_openai_realtime_status() -> Result { - let configured = stored_openai_api_key()?.is_some(); + let configured = openai_voice_credentials::is_present(OpenAiVoiceCredential::Realtime)?; Ok(OpenAiRealtimeStatus { configured }) } +#[tauri::command] +pub fn set_openai_realtime_api_key(app: AppHandle, api_key: String) -> Result<(), String> { + let api_key = api_key.trim(); + if api_key.is_empty() { + return Err("Realtime API key cannot be empty".into()); + } + openai_voice_credentials::store(OpenAiVoiceCredential::Realtime, api_key)?; + app.emit("openai-voice:settings-changed", ()) + .map_err(|error| format!("Could not refresh Realtime settings: {error}")) +} + +#[tauri::command] +pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { + openai_voice_credentials::clear(OpenAiVoiceCredential::Realtime)?; + app.emit("openai-voice:settings-changed", ()) + .map_err(|error| format!("Could not refresh Realtime settings: {error}")) +} + #[tauri::command] pub async fn create_openai_realtime_session() -> Result { let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; @@ -125,7 +140,8 @@ pub fn start_openai_realtime_spokesperson_runtime( } let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; - let config = OpenAiSpokespersonConfig::new(api_key, options, Vec::new()); + let mut config = OpenAiSpokespersonConfig::new(api_key, options, Vec::new()); + config.endpoint = openai_voice_endpoints::effective_url(VoiceEndpointKind::Realtime)?; let semantic_revision = Arc::new(AtomicU64::new(0)); let event_window = webview_window.clone(); let event_session_id = session_id.clone(); diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 725cc69e6..23dc95cb2 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -1,5 +1,9 @@ //! Berd-owned credentials for OpenAI voice services. +use sha2::{Digest, Sha256}; + +use super::openai_voice_endpoints::{self, VoiceEndpointKind}; + const KEYCHAIN_SERVICE: &str = "berd-openai-voice"; const KEYCHAIN_ACCOUNT: &str = "api-key"; @@ -11,9 +15,11 @@ pub(crate) enum OpenAiVoiceCredential { } impl OpenAiVoiceCredential { - const fn account(self) -> &'static str { + const fn kind(self) -> VoiceEndpointKind { match self { - Self::SpeechToText | Self::TextToSpeech | Self::Realtime => KEYCHAIN_ACCOUNT, + Self::SpeechToText => VoiceEndpointKind::Stt, + Self::TextToSpeech => VoiceEndpointKind::Tts, + Self::Realtime => VoiceEndpointKind::Realtime, } } @@ -32,6 +38,19 @@ impl OpenAiVoiceCredential { } } +fn account(credential: OpenAiVoiceCredential) -> Result { + let kind = credential.kind(); + let url = openai_voice_endpoints::effective_url(kind)?; + Ok(account_for_url(kind, &url)) +} + +fn account_for_url(kind: VoiceEndpointKind, url: &str) -> String { + if url == kind.default_url() { + return KEYCHAIN_ACCOUNT.to_string(); + } + format!("endpoint-{}", hex::encode(Sha256::digest(url.as_bytes()))) +} + fn entry(account: &str) -> Result { keyring::Entry::new(KEYCHAIN_SERVICE, account) .map_err(|error| format!("Could not access Berd's OpenAI voice credentials: {error}")) @@ -59,18 +78,43 @@ fn clear_account(account: &str) -> Result<(), String> { } pub(crate) fn read(credential: OpenAiVoiceCredential) -> Result, String> { - read_account(credential.account()) + read_account(&account(credential)?) +} + +/// Check only Keychain item metadata; status polling must never request secret access. +pub(crate) fn is_present(credential: OpenAiVoiceCredential) -> Result { + let account = account(credential)?; + #[cfg(target_os = "macos")] + { + use security_framework::item::{ItemClass, ItemSearchOptions}; + match ItemSearchOptions::new() + .class(ItemClass::generic_password()) + .service(KEYCHAIN_SERVICE) + .account(&account) + .load_attributes(true) + .skip_authenticated_items(true) + .search() + { + Ok(items) => Ok(!items.is_empty()), + Err(error) if error.code() == -25300 => Ok(false), + Err(error) => Err(format!("Could not check Berd's voice credential: {error}")), + } + } + #[cfg(not(target_os = "macos"))] + { + Ok(read_account(&account)?.is_some()) + } } pub(crate) fn store(credential: OpenAiVoiceCredential, api_key: &str) -> Result<(), String> { - let entry = entry(credential.account())?; + let entry = entry(&account(credential)?)?; entry .set_password(api_key) .map_err(|error| format!("Could not save Berd's OpenAI voice credential: {error}")) } pub(crate) fn clear(credential: OpenAiVoiceCredential) -> Result<(), String> { - clear_account(credential.account()) + clear_account(&account(credential)?) } pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result { @@ -81,9 +125,47 @@ pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result &'static str { + match self { + Self::Realtime => REALTIME_DEFAULT, + Self::Stt => STT_DEFAULT, + Self::Tts => TTS_DEFAULT, + } + } +} + +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct VoiceEndpointSettings { + #[serde(default)] + pub realtime: Option, + #[serde(default)] + pub stt: Option, + #[serde(default)] + pub tts: Option, +} + +impl VoiceEndpointSettings { + fn get(&self, kind: VoiceEndpointKind) -> Option<&str> { + match kind { + VoiceEndpointKind::Realtime => self.realtime.as_deref(), + VoiceEndpointKind::Stt => self.stt.as_deref(), + VoiceEndpointKind::Tts => self.tts.as_deref(), + } + } + + fn set(&mut self, kind: VoiceEndpointKind, value: Option) { + *match kind { + VoiceEndpointKind::Realtime => &mut self.realtime, + VoiceEndpointKind::Stt => &mut self.stt, + VoiceEndpointKind::Tts => &mut self.tts, + } = value; + } +} + +fn settings_path() -> Result { + Ok(crate::services::goose_config::config_path()? + .parent() + .ok_or_else(|| "Could not resolve Goose's configuration directory".to_string())? + .join("openai-voice-endpoints.json")) +} + +fn read_settings() -> Result { + let path = settings_path()?; + match std::fs::read(&path) { + Ok(bytes) => serde_json::from_slice(&bytes) + .map_err(|error| format!("Could not read OpenAI voice endpoints: {error}")), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Default::default()), + Err(error) => Err(format!("Could not read OpenAI voice endpoints: {error}")), + } +} + +fn persist(settings: &VoiceEndpointSettings) -> Result<(), String> { + let path = settings_path()?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|error| { + format!("Could not create OpenAI voice settings directory: {error}") + })?; + } + let bytes = serde_json::to_vec_pretty(settings) + .map_err(|error| format!("Could not encode OpenAI voice endpoints: {error}"))?; + write_bytes_atomically(&path, &bytes) + .map_err(|error| format!("Could not save OpenAI voice endpoints: {error}")) +} + +fn validate(kind: VoiceEndpointKind, raw: &str) -> Result, String> { + let raw = raw.trim(); + if raw.is_empty() { + return Ok(None); + } + let mut url = url::Url::parse(raw).map_err(|error| format!("Invalid endpoint URL: {error}"))?; + let allowed = match kind { + VoiceEndpointKind::Realtime | VoiceEndpointKind::Stt => ["ws", "wss"].as_slice(), + VoiceEndpointKind::Tts => ["http", "https"].as_slice(), + }; + if !allowed.contains(&url.scheme()) + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.fragment().is_some() + { + return Err("Endpoint must be an absolute URL with the correct protocol and no embedded credentials or fragment".into()); + } + url.set_fragment(None); + Ok(Some(url.to_string())) +} + +pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { + if let Some(saved) = read_settings()?.get(kind) { + return Ok(saved.to_string()); + } + if !matches!(kind, VoiceEndpointKind::Realtime) + && std::env::var_os("BERD_OPENAI_VOICE_BASE_URL").is_some() + { + let base = super::openai_audio::base_url()?; + let path = match kind { + VoiceEndpointKind::Stt => "realtime", + VoiceEndpointKind::Tts => "audio/speech", + VoiceEndpointKind::Realtime => unreachable!(), + }; + let mut url = url::Url::parse(&super::openai_audio::endpoint_for_base_url(&base, path)?) + .map_err(|error| format!("Invalid OpenAI voice endpoint: {error}"))?; + if matches!(kind, VoiceEndpointKind::Stt) { + url.set_scheme("wss").expect("https can become wss"); + url.query_pairs_mut().append_pair("intent", "transcription"); + } + return Ok(url.to_string()); + } + Ok(kind.default_url().to_string()) +} + +#[tauri::command] +pub(crate) fn get_openai_voice_endpoints() -> Result { + read_settings() +} + +#[tauri::command] +pub(crate) fn set_openai_voice_endpoint( + app: AppHandle, + kind: VoiceEndpointKind, + url: String, +) -> Result<(), String> { + let mut settings = read_settings()?; + let selected = validate(kind, &url)?; + settings.set(kind, selected.filter(|url| url != kind.default_url())); + persist(&settings)?; + app.emit(SETTINGS_CHANGED_EVENT, ()) + .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) +} + +pub(crate) fn reset() -> Result<(), String> { + persist(&VoiceEndpointSettings::default()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn endpoints_are_full_urls_with_independent_openai_defaults() { + assert_eq!(VoiceEndpointKind::Realtime.default_url(), REALTIME_DEFAULT); + assert_eq!(VoiceEndpointKind::Stt.default_url(), STT_DEFAULT); + assert_eq!(VoiceEndpointKind::Tts.default_url(), TTS_DEFAULT); + assert!(validate(VoiceEndpointKind::Stt, "wss://example.test/stt?mode=live").is_ok()); + assert!(validate( + VoiceEndpointKind::Tts, + "https://example.test/audio/speech?api-version=1" + ) + .is_ok()); + assert!(validate(VoiceEndpointKind::Tts, "wss://example.test/audio/speech").is_err()); + assert!(validate( + VoiceEndpointKind::Realtime, + "wss://key@example.test/realtime" + ) + .is_err()); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index 79cdcf904..b570db5ab 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -605,6 +605,8 @@ pub fn run() { commands::model_setup::clear_model_setup_status, commands::notifications::show_completion_notification, commands::openai_realtime::get_openai_realtime_status, + commands::openai_realtime::set_openai_realtime_api_key, + commands::openai_realtime::clear_openai_realtime_api_key, commands::openai_realtime::create_openai_realtime_session, commands::openai_realtime::start_openai_realtime_spokesperson_runtime, commands::openai_realtime::send_openai_realtime_spokesperson_runtime_event, @@ -679,6 +681,8 @@ pub fn run() { commands::pocket_voice::stop_pocket_voice, commands::pocket_voice::remove_voice_model, commands::openai_audio::get_openai_voice_status, + commands::openai_voice_endpoints::get_openai_voice_endpoints, + commands::openai_voice_endpoints::set_openai_voice_endpoint, commands::openai_audio::set_openai_stt_api_key, commands::openai_audio::clear_openai_stt_api_key, commands::openai_audio::set_openai_tts_api_key, diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index 3380b0772..e626a6ece 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -10,6 +10,7 @@ import type { export interface OpenAiVoiceStatus { sttConfigured: boolean; ttsConfigured: boolean; + realtimeConfigured: boolean; sttConfigurationSource: "default" | "environment"; ttsConfigurationSource: "default" | "environment"; sttUnavailableReason: string | null; @@ -38,6 +39,33 @@ export function setOpenAiTtsApiKey(apiKey: string): Promise { return invoke("set_openai_tts_api_key", { apiKey }); } +export type OpenAiVoiceEndpointKind = "realtime" | "stt" | "tts"; + +export interface OpenAiVoiceEndpoints { + realtime: string | null; + stt: string | null; + tts: string | null; +} + +export function getOpenAiVoiceEndpoints(): Promise { + return invoke("get_openai_voice_endpoints"); +} + +export function setOpenAiVoiceEndpoint( + kind: OpenAiVoiceEndpointKind, + url: string, +): Promise { + return invoke("set_openai_voice_endpoint", { kind, url }); +} + +export function setOpenAiRealtimeApiKey(apiKey: string): Promise { + return invoke("set_openai_realtime_api_key", { apiKey }); +} + +export function clearOpenAiRealtimeApiKey(): Promise { + return invoke("clear_openai_realtime_api_key"); +} + export function setOpenAiSttApiKey(apiKey: string): Promise { return invoke("set_openai_stt_api_key", { apiKey }); } diff --git a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx index 7cdd391a5..9ea56817b 100644 --- a/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx +++ b/src/features/voice-conversation/hooks/useOpenAiVoiceSetup.test.tsx @@ -37,6 +37,7 @@ function status(configured: boolean): OpenAiVoiceStatus { return { sttConfigured: configured, ttsConfigured: configured, + realtimeConfigured: configured, sttConfigurationSource: "default", ttsConfigurationSource: "default", sttUnavailableReason: null, diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx new file mode 100644 index 000000000..fb18c2b6a --- /dev/null +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -0,0 +1,92 @@ +import { useEffect, useId, useState } from "react"; +import { useTranslation } from "react-i18next"; +import { + getOpenAiVoiceEndpoints, + setOpenAiVoiceEndpoint, + type OpenAiVoiceEndpointKind, +} from "../api/openAiVoice"; +import { Button } from "@/shared/ui/button"; +import { Input } from "@/shared/ui/input"; + +const DEFAULT_URLS: Record = { + realtime: "wss://api.openai.com/v1/realtime", + stt: "wss://api.openai.com/v1/realtime?intent=transcription", + tts: "https://api.openai.com/v1/audio/speech", +}; + +export function OpenAiEndpointField({ + kind, + label, +}: { + kind: OpenAiVoiceEndpointKind; + label: string; +}) { + const { t } = useTranslation("settings"); + const id = useId(); + const [url, setUrl] = useState(""); + const [saving, setSaving] = useState(false); + const [error, setError] = useState(null); + + useEffect(() => { + let active = true; + void getOpenAiVoiceEndpoints().then( + (settings) => { + if (active) setUrl(settings[kind] ?? ""); + }, + (cause) => { + if (active) setError(String(cause)); + }, + ); + return () => { + active = false; + }; + }, [kind]); + + const save = async () => { + setSaving(true); + setError(null); + try { + await setOpenAiVoiceEndpoint(kind, url); + setUrl((await getOpenAiVoiceEndpoints())[kind] ?? ""); + } catch (cause) { + setError(cause instanceof Error ? cause.message : String(cause)); + } finally { + setSaving(false); + } + }; + + return ( +
+ +
+ setUrl(event.target.value)} + placeholder={DEFAULT_URLS[kind]} + autoComplete="off" + spellCheck={false} + /> + +
+

+ {t("voice.endpointDefaultHint")} +

+ {error ? ( +

+ {error} +

+ ) : null} +
+ ); +} diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx index 1d1569c0f..84a3ce074 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx @@ -7,16 +7,22 @@ import { RealtimeVoiceSettings } from "./RealtimeVoiceSettings"; const openAiVoiceMocks = vi.hoisted(() => ({ clearApiKey: vi.fn(() => Promise.resolve()), - getStatus: vi.fn(() => Promise.resolve({ sttConfigured: true })), + getStatus: vi.fn(() => Promise.resolve({ realtimeConfigured: true })), + getEndpoints: vi.fn(() => + Promise.resolve({ realtime: null, stt: null, tts: null }), + ), + setEndpoint: vi.fn(() => Promise.resolve()), listenToSettings: vi.fn(() => Promise.resolve(() => undefined)), setApiKey: vi.fn(() => Promise.resolve()), })); vi.mock("../api/openAiVoice", () => ({ - clearOpenAiSttApiKey: openAiVoiceMocks.clearApiKey, + clearOpenAiRealtimeApiKey: openAiVoiceMocks.clearApiKey, + getOpenAiVoiceEndpoints: openAiVoiceMocks.getEndpoints, + setOpenAiVoiceEndpoint: openAiVoiceMocks.setEndpoint, getOpenAiVoiceStatus: openAiVoiceMocks.getStatus, listenToOpenAiVoiceSettings: openAiVoiceMocks.listenToSettings, - setOpenAiSttApiKey: openAiVoiceMocks.setApiKey, + setOpenAiRealtimeApiKey: openAiVoiceMocks.setApiKey, })); describe("RealtimeVoiceSettings", () => { @@ -61,10 +67,24 @@ describe("RealtimeVoiceSettings", () => { ).toHaveTextContent("Debug — show agent routing"); }); - it("stores the Realtime key through the shared OpenAI voice credential path", async () => { + it("shows the default realtime URL above its URL-scoped key", async () => { const user = userEvent.setup(); renderWithProviders(); + expect(screen.getByLabelText("Realtime endpoint URL")).toHaveAttribute( + "placeholder", + "wss://api.openai.com/v1/realtime", + ); + await user.type( + screen.getByLabelText("Realtime endpoint URL"), + "ws://127.0.0.1:18870/v1/realtime", + ); + await user.click(screen.getByRole("button", { name: "Save URL" })); + expect(openAiVoiceMocks.setEndpoint).toHaveBeenCalledWith( + "realtime", + "ws://127.0.0.1:18870/v1/realtime", + ); + await user.type(screen.getByLabelText("OpenAI API key"), " sk-shared "); await user.click(screen.getByRole("button", { name: "Save key" })); diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx index b7c9a33fc..50f9d6c66 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx @@ -26,9 +26,13 @@ import { type RealtimeTurnDetection, useRealtimeVoicePreference, } from "../lib/realtimeVoicePreference"; -import { clearOpenAiSttApiKey, setOpenAiSttApiKey } from "../api/openAiVoice"; +import { + clearOpenAiRealtimeApiKey, + setOpenAiRealtimeApiKey, +} from "../api/openAiVoice"; import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; +import { OpenAiEndpointField } from "./OpenAiEndpointField"; import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; import { SimpleVoicePickerDialog } from "./SimpleVoicePickerDialog"; import { @@ -125,11 +129,15 @@ export function RealtimeVoiceSettings() { return (
+
diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index f26a185ff..af48e7c32 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -98,6 +98,10 @@ const openAiStatusState = vi.hoisted(() => ({ }, })); const openAiApiMocks = vi.hoisted(() => ({ + getEndpoints: vi.fn(() => + Promise.resolve({ realtime: null, stt: null, tts: null }), + ), + setEndpoint: vi.fn(() => Promise.resolve()), setSttApiKey: vi.fn(() => Promise.resolve()), clearSttApiKey: vi.fn(() => Promise.resolve()), setTtsApiKey: vi.fn(() => Promise.resolve()), @@ -109,6 +113,10 @@ const openAiApiMocks = vi.hoisted(() => ({ })); vi.mock("../api/openAiVoice", () => ({ + setOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), + clearOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), + getOpenAiVoiceEndpoints: openAiApiMocks.getEndpoints, + setOpenAiVoiceEndpoint: openAiApiMocks.setEndpoint, setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), setOpenAiSpeechVoice: openAiApiMocks.setSpeechVoice, setOpenAiSttApiKey: openAiApiMocks.setSttApiKey, diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index eea396b17..f5e383e64 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -63,6 +63,7 @@ import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; import { SimpleVoicePickerDialog } from "./SimpleVoicePickerDialog"; import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; +import { OpenAiEndpointField } from "./OpenAiEndpointField"; import { RealtimeVoiceSettings } from "./RealtimeVoiceSettings"; import { getDefaultRealtimeVoicePreference, @@ -392,6 +393,10 @@ export function VoiceSettings() { details={ input.backend === "openai" ? (
+ + Date: Thu, 24 Sep 2026 13:56:19 -0400 Subject: [PATCH 02/34] fix(voice): guard dictation against custom realtime keys --- src-tauri/src/commands/openai_realtime.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index abc2287c8..3de536baa 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -103,6 +103,10 @@ pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { #[tauri::command] pub async fn create_openai_realtime_session() -> Result { + let endpoint = openai_voice_endpoints::effective_url(VoiceEndpointKind::Realtime)?; + if endpoint != VoiceEndpointKind::Realtime.default_url() { + return Err("Realtime dictation requires the default OpenAI endpoint; custom Realtime URLs are supported for Expert-Spokesperson conversations".into()); + } let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; let response = realtime_transcription_client_secret_request(&reqwest::Client::new(), &api_key) .send() From 0b9c6dc21474a5a83a69c65a39b76f9746f8e837 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 11:42:50 -0400 Subject: [PATCH 03/34] test(voice): cover independent endpoint settings in browser --- playwright.config.ts | 1 + tests/e2e/fixtures/tauri-mock.ts | 14 +++++ tests/e2e/voice-endpoints.spec.ts | 98 +++++++++++++++++++++++++++++++ 3 files changed, 113 insertions(+) create mode 100644 tests/e2e/voice-endpoints.spec.ts diff --git a/playwright.config.ts b/playwright.config.ts index bd022eeb2..29eab0b9a 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -85,6 +85,7 @@ export default defineConfig({ name: "voice-conversation", testMatch: [ "**/voice-conversation.spec.ts", + "**/voice-endpoints.spec.ts", "**/voice-settings-visual.spec.ts", ], use: { diff --git a/tests/e2e/fixtures/tauri-mock.ts b/tests/e2e/fixtures/tauri-mock.ts index 997e99844..d29b2cd22 100644 --- a/tests/e2e/fixtures/tauri-mock.ts +++ b/tests/e2e/fixtures/tauri-mock.ts @@ -160,6 +160,10 @@ export function buildInitScript(options?: { const CALLBACKS = new Map(); const EVENT_LISTENERS = new Map(); const ACP_SOCKETS = new Set(); + const VOICE_ENDPOINTS = JSON.parse( + localStorage.getItem("goose:e2e:voice-endpoints") ?? + '{"realtime":null,"stt":null,"tts":null}', + ); const LAYOUT_CONSTRAINTS = { minCenter: -1000000, maxCenter: 1000000, @@ -754,6 +758,16 @@ export function buildInitScript(options?: { ttsAvailable: true, unavailableReason: null, }); + case "get_openai_voice_endpoints": + return Promise.resolve(clone(VOICE_ENDPOINTS)); + case "set_openai_voice_endpoint": + VOICE_ENDPOINTS[args.kind] = args.url.trim() || null; + localStorage.setItem( + "goose:e2e:voice-endpoints", + JSON.stringify(VOICE_ENDPOINTS), + ); + emitTauriEvent("openai-voice:settings-changed", null); + return Promise.resolve(null); case "speak_pocket_voice": POCKET_VOICE_SPOKEN_TEXTS.push(args?.text); return new Promise((resolve) => diff --git a/tests/e2e/voice-endpoints.spec.ts b/tests/e2e/voice-endpoints.spec.ts new file mode 100644 index 000000000..64f895618 --- /dev/null +++ b/tests/e2e/voice-endpoints.spec.ts @@ -0,0 +1,98 @@ +import { test as base, expect, type Page } from "@playwright/test"; + +import { buildInitScript } from "./fixtures/tauri-mock"; + +const test = base.extend<{ settings: Page }>({ + settings: async ({ page }, use) => { + await page.addInitScript({ + content: buildInitScript({ enabledExperiments: ["voice-conversation"] }), + }); + await page.addInitScript(() => { + if (localStorage.getItem("goose:voice-conversation-mode") === null) { + localStorage.setItem("goose:voice-conversation-mode", "chained"); + } + if (localStorage.getItem("goose:voice-input-backend") === null) { + localStorage.setItem("goose:voice-input-backend", "openai"); + } + if (localStorage.getItem("goose:voice-output-backend") === null) { + localStorage.setItem("goose:voice-output-backend", "openai"); + } + }); + await use(page); + }, +}); + +test.use({ screenshot: "off", trace: "off", video: "off" }); + +async function saveEndpoint(page: Page, label: string, url: string) { + const input = page.getByLabel(label); + await input.fill(url); + await input.locator("..").getByRole("button", { name: "Save URL" }).click(); + await expect(input).toHaveValue(url); +} + +test("keeps Realtime, STT, and TTS URLs independent across settings reloads", async ({ + settings: page, +}) => { + await page.goto("/"); + await page.locator("[data-sidebar-nav-id=settings]").click(); + await page.locator("[data-sidebar-nav-id=settings-voice]").click(); + + const stt = page.getByLabel("Speech-to-text endpoint URL"); + const tts = page.getByLabel("Text-to-speech endpoint URL"); + await expect(stt).toHaveAttribute( + "placeholder", + "wss://api.openai.com/v1/realtime?intent=transcription", + ); + await expect(tts).toHaveAttribute( + "placeholder", + "https://api.openai.com/v1/audio/speech", + ); + await saveEndpoint( + page, + "Speech-to-text endpoint URL", + "ws://127.0.0.1:18870/v1/realtime?intent=transcription", + ); + await saveEndpoint( + page, + "Text-to-speech endpoint URL", + "http://127.0.0.1:18870/v1/audio/speech", + ); + + await page + .getByRole("radio", { name: /Talk through a voice assistant/ }) + .click(); + const realtime = page.getByLabel("Realtime endpoint URL"); + await expect(realtime).toHaveAttribute( + "placeholder", + "wss://api.openai.com/v1/realtime", + ); + await saveEndpoint( + page, + "Realtime endpoint URL", + "ws://127.0.0.1:18870/v1/realtime", + ); + + await page.goto("/"); + await page.locator("[data-sidebar-nav-id=settings]").click(); + await page.locator("[data-sidebar-nav-id=settings-voice]").click(); + await expect(page.getByLabel("Realtime endpoint URL")).toHaveValue( + "ws://127.0.0.1:18870/v1/realtime", + ); + await page.getByRole("radio", { name: /Talk to your coding agent/ }).click(); + await expect(page.getByLabel("Speech-to-text endpoint URL")).toHaveValue( + "ws://127.0.0.1:18870/v1/realtime?intent=transcription", + ); + await expect(page.getByLabel("Text-to-speech endpoint URL")).toHaveValue( + "http://127.0.0.1:18870/v1/audio/speech", + ); + + await saveEndpoint(page, "Speech-to-text endpoint URL", ""); + await page.goto("/"); + await page.locator("[data-sidebar-nav-id=settings]").click(); + await page.locator("[data-sidebar-nav-id=settings-voice]").click(); + await expect(page.getByLabel("Speech-to-text endpoint URL")).toHaveValue(""); + await expect(page.getByLabel("Text-to-speech endpoint URL")).toHaveValue( + "http://127.0.0.1:18870/v1/audio/speech", + ); +}); From 7efdf648dbb85ba59ab75f2b8784e8f32f79d4d9 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 12:48:35 -0400 Subject: [PATCH 04/34] fix(voice): clarify per-endpoint key guidance --- .../src/commands/openai_voice_credentials.rs | 19 +++++-- .../ui/VoiceSettings.test.tsx | 50 ++++++++++++++++--- src/shared/i18n/locales/en/settings.json | 22 ++++---- 3 files changed, 69 insertions(+), 22 deletions(-) diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 23dc95cb2..1e7f1f0ae 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -26,13 +26,13 @@ impl OpenAiVoiceCredential { const fn missing_message(self) -> &'static str { match self { Self::SpeechToText => { - "OpenAI speech-to-text is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + "OpenAI speech-to-text is not configured. Add an API key for the selected transcription URL in Voice settings, then try again." } Self::TextToSpeech => { - "OpenAI text-to-speech is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + "OpenAI text-to-speech is not configured. Add an API key for the selected playback URL in Voice settings, then try again." } Self::Realtime => { - "OpenAI Realtime voice is not configured. Add the shared OpenAI voice API key in Voice settings, then try again." + "OpenAI Realtime voice is not configured. Add an API key for the selected Realtime URL in Voice settings, then try again." } } } @@ -168,4 +168,17 @@ mod tests { account_for_url(VoiceEndpointKind::Realtime, "wss://other.test/v1/realtime") ); } + + #[test] + fn missing_key_guidance_applies_to_the_selected_endpoint() { + for credential in [ + OpenAiVoiceCredential::SpeechToText, + OpenAiVoiceCredential::TextToSpeech, + OpenAiVoiceCredential::Realtime, + ] { + let message = credential.missing_message(); + assert!(message.contains("selected")); + assert!(!message.contains("shared")); + } + } } diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index af48e7c32..c0e7b5f98 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -99,7 +99,11 @@ const openAiStatusState = vi.hoisted(() => ({ })); const openAiApiMocks = vi.hoisted(() => ({ getEndpoints: vi.fn(() => - Promise.resolve({ realtime: null, stt: null, tts: null }), + Promise.resolve({ + realtime: null as string | null, + stt: null as string | null, + tts: null as string | null, + }), ), setEndpoint: vi.fn(() => Promise.resolve()), setSttApiKey: vi.fn(() => Promise.resolve()), @@ -336,6 +340,11 @@ describe("VoiceSettings", () => { openAiApiMocks.setSttApiKey.mockClear(); openAiApiMocks.clearSttApiKey.mockClear(); openAiApiMocks.setSpeechVoice.mockClear(); + openAiApiMocks.getEndpoints.mockReset().mockResolvedValue({ + realtime: null, + stt: null, + tts: null, + }); openAiApiMocks.resetAll.mockReset().mockResolvedValue(undefined); openAiApiMocks.resetPocket.mockClear(); openAiApiMocks.resetSiri.mockClear(); @@ -628,9 +637,7 @@ describe("VoiceSettings", () => { ).toBeInTheDocument(); expect(screen.getByText("Playback speed")).toBeInTheDocument(); expect( - screen.getAllByText( - "Saved securely and shared by OpenAI transcription and voice playback.", - ), + screen.getAllByText("Saved securely for this endpoint URL."), ).toHaveLength(2); }); @@ -708,7 +715,7 @@ describe("VoiceSettings", () => { expect( await screen.findByText( - "OpenAI transcription is not ready. Add the shared OpenAI voice API key below, then try again.", + "OpenAI transcription is not ready. Add an API key for the selected transcription URL below, then try again.", ), ).toBeInTheDocument(); expect( @@ -716,6 +723,33 @@ describe("VoiceSettings", () => { ).not.toBeInTheDocument(); }); + it("does not call a custom endpoint key shared", async () => { + inputState.backend = "openai"; + outputState.backend = "openai"; + openAiStatusState.current = { + ...openAiStatusState.current, + sttConfigured: false, + ttsConfigured: false, + unavailableReason: "missingApiKey", + }; + openAiApiMocks.getEndpoints.mockResolvedValue({ + realtime: null, + stt: "ws://127.0.0.1:18870/v1/realtime?intent=transcription", + tts: "http://127.0.0.1:18870/v1/audio/speech", + }); + setupState.current = setup(pocketStatus()); + renderWithProviders(); + + expect( + await screen.findByDisplayValue( + "ws://127.0.0.1:18870/v1/realtime?intent=transcription", + ), + ).toBeInTheDocument(); + expect( + screen.queryAllByText(/shared OpenAI|shared by OpenAI|their shared/i), + ).toHaveLength(0); + }); + it("reports missing OpenAI input and Pocket output together", async () => { inputState.backend = "openai"; outputState.backend = "pocket"; @@ -729,7 +763,7 @@ describe("VoiceSettings", () => { expect( await screen.findByText( - "The shared OpenAI voice API key is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", + "The API key for the selected transcription URL is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", ), ).toBeInTheDocument(); }); @@ -757,7 +791,7 @@ describe("VoiceSettings", () => { expect( await screen.findByText( - "The shared OpenAI voice API key is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + "The API key for the selected transcription URL is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", ), ).toBeInTheDocument(); }); @@ -792,7 +826,7 @@ describe("VoiceSettings", () => { expect( await screen.findByText( - "OpenAI voice playback is not ready. Add the shared OpenAI voice API key below, then try again.", + "OpenAI voice playback is not ready. Add an API key for the selected playback URL below, then try again.", ), ).toBeInTheDocument(); expect( diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index f13a9d481..3095f17de 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -1046,19 +1046,19 @@ "modelMissingSize": "Not installed · {{size}} download", "modelNotInstalled": "Not installed", "notReadyInput": "Parakeet STT is not installed. Download it below to use Voice Conversation.", - "notReadyInputAndOpenAiOutput": "Parakeet STT is not installed, and the shared OpenAI voice API key is missing. Complete both steps below to use Voice Conversation.", + "notReadyInputAndOpenAiOutput": "Parakeet STT is not installed, and the API key for the selected playback URL is missing. Complete both steps below to use Voice Conversation.", "notReadyInputAndPocketOutput": "Parakeet STT and Pocket TTS are not installed. Download both below to use Voice Conversation.", "notReadyInputAndSiriOutput": "Parakeet STT is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyMacInput": "Apple's on-device dictation model is not installed. Download it below to use Voice Conversation.", - "notReadyMacInputAndOpenAiOutput": "Apple's on-device dictation model is not installed, and the shared OpenAI voice API key is missing. Complete both steps below to use Voice Conversation.", + "notReadyMacInputAndOpenAiOutput": "Apple's on-device dictation model is not installed, and the API key for the selected playback URL is missing. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndPocketOutput": "Apple's on-device dictation model and Pocket TTS are not installed. Complete both steps below to use Voice Conversation.", "notReadyMacInputAndSiriOutput": "Apple's on-device dictation model is not installed, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", "notReadyOpenAi": "OpenAI voice is not ready. Add the required API key below, then try again.", - "notReadyOpenAiStt": "OpenAI transcription is not ready. Add the shared OpenAI voice API key below, then try again.", - "notReadyOpenAiSttAndPocketOutput": "The shared OpenAI voice API key is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", - "notReadyOpenAiSttAndSiriOutput": "The shared OpenAI voice API key is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", - "notReadyOpenAiSttAndTts": "OpenAI transcription and voice playback are not ready. Add their shared OpenAI voice API key below, then try again.", - "notReadyOpenAiTts": "OpenAI voice playback is not ready. Add the shared OpenAI voice API key below, then try again.", + "notReadyOpenAiStt": "OpenAI transcription is not ready. Add an API key for the selected transcription URL below, then try again.", + "notReadyOpenAiSttAndPocketOutput": "The API key for the selected transcription URL is missing, and Pocket TTS is not installed. Complete both steps below to use Voice Conversation.", + "notReadyOpenAiSttAndSiriOutput": "The API key for the selected transcription URL is missing, and no installed Siri voice is selected. Complete both steps below to use Voice Conversation.", + "notReadyOpenAiSttAndTts": "OpenAI transcription and voice playback are not ready. Add the required API key below, then try again.", + "notReadyOpenAiTts": "OpenAI voice playback is not ready. Add an API key for the selected playback URL below, then try again.", "notReadyPocketOutput": "Pocket TTS is not installed. Download it below to use Voice Conversation.", "notReadySiriOutput": "No installed Siri voice is selected. Download or select one below to use Voice Conversation.", "notReadyTitle": "Voice Conversation isn't ready", @@ -1067,17 +1067,17 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", - "openAiApiKeyConfigured": "Saved securely and shared by OpenAI transcription and voice playback.", - "openAiApiKeyNotConfigured": "This key is shared by OpenAI transcription and voice playback. It is separate from provider credentials used by Goose.", + "openAiApiKeyConfigured": "Saved securely for this endpoint URL.", + "openAiApiKeyNotConfigured": "Keys are stored per endpoint URL; default OpenAI endpoints share a key. They are separate from Goose provider credentials.", "openAiApiKeySaved": "API key saved", "openAiChecking": "Checking OpenAI voice settings…", "openAiEnvironmentOverride": "Development configuration is overridden by the Berd process environment.", "openAiSttApiKey": "OpenAI speech-to-text API key", "openAiSttConfigured": "Uses {{model}}.", - "openAiSttNotConfigured": "Add the shared OpenAI voice API key to use OpenAI transcription.", + "openAiSttNotConfigured": "Add an API key for the selected transcription URL to use it.", "openAiTtsApiKey": "OpenAI text-to-speech API key", "openAiTtsConfigured": "Uses {{model}} and the {{voice}} voice. OpenAI voices are AI-generated.", - "openAiTtsNeedsKey": "Add the shared OpenAI voice API key to use this voice.", + "openAiTtsNeedsKey": "Add an API key for the selected playback URL to use this voice.", "openAiTtsUnsupportedPlatform": "OpenAI voice playback is currently supported on macOS only.", "outputBackendDescription": "Choose a backend.", "playbackSpeed": "Playback speed", From 3c98c06c2a760324a581e46879d7fa71c3d27607 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 13:25:53 -0400 Subject: [PATCH 05/34] Save each voice endpoint URL and key together --- .../ui/OpenAiApiKeyField.tsx | 101 ------------- .../ui/OpenAiEndpointField.tsx | 139 ++++++++++++++++-- .../ui/RealtimeVoiceSettings.test.tsx | 57 ++++++- .../ui/RealtimeVoiceSettings.tsx | 10 +- .../ui/VoiceSettings.test.tsx | 38 ++++- .../voice-conversation/ui/VoiceSettings.tsx | 17 +-- src/shared/i18n/locales/en/settings.json | 14 +- src/shared/i18n/locales/es/settings.json | 14 +- tests/e2e/voice-endpoints.spec.ts | 2 +- 9 files changed, 234 insertions(+), 158 deletions(-) delete mode 100644 src/features/voice-conversation/ui/OpenAiApiKeyField.tsx diff --git a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx b/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx deleted file mode 100644 index d856892d3..000000000 --- a/src/features/voice-conversation/ui/OpenAiApiKeyField.tsx +++ /dev/null @@ -1,101 +0,0 @@ -import { useId, useState } from "react"; -import { useTranslation } from "react-i18next"; -import { Button } from "@/shared/ui/button"; -import { Input } from "@/shared/ui/input"; - -interface OpenAiApiKeyFieldProps { - label: string; - configured: boolean; - onSave: (apiKey: string) => Promise; - onClear: () => Promise; - description?: string; -} - -export function OpenAiApiKeyField({ - label, - configured, - onSave, - onClear, - description, -}: OpenAiApiKeyFieldProps) { - const { t } = useTranslation("settings"); - const inputId = useId(); - const [apiKey, setApiKey] = useState(""); - const [saving, setSaving] = useState(false); - const [error, setError] = useState(null); - - const save = async () => { - setSaving(true); - setError(null); - try { - await onSave(apiKey); - setApiKey(""); - } catch (cause) { - setError(cause instanceof Error ? cause.message : String(cause)); - } finally { - setSaving(false); - } - }; - - const clear = async () => { - setSaving(true); - setError(null); - try { - await onClear(); - setApiKey(""); - } catch (cause) { - setError(cause instanceof Error ? cause.message : String(cause)); - } finally { - setSaving(false); - } - }; - - return ( -
- -
- setApiKey(event.target.value)} - placeholder={configured ? t("voice.openAiApiKeySaved") : "sk-…"} - autoComplete="off" - spellCheck={false} - /> - - {configured ? ( - - ) : null} -
-

- {description ?? - (configured - ? t("voice.openAiApiKeyConfigured") - : t("voice.openAiApiKeyNotConfigured"))} -

- {error ? ( -

- {error} -

- ) : null} -
- ); -} diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index fb18c2b6a..8fe3a8c9a 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -2,6 +2,7 @@ import { useEffect, useId, useState } from "react"; import { useTranslation } from "react-i18next"; import { getOpenAiVoiceEndpoints, + getOpenAiVoiceStatus, setOpenAiVoiceEndpoint, type OpenAiVoiceEndpointKind, } from "../api/openAiVoice"; @@ -17,13 +18,30 @@ const DEFAULT_URLS: Record = { export function OpenAiEndpointField({ kind, label, + keyLabel, + configured, + onSaveKey, + onClearKey, }: { kind: OpenAiVoiceEndpointKind; label: string; + keyLabel: string; + configured: boolean; + onSaveKey: (apiKey: string) => Promise; + onClearKey: () => Promise; }) { const { t } = useTranslation("settings"); const id = useId(); + const keyId = useId(); const [url, setUrl] = useState(""); + const [savedUrl, setSavedUrl] = useState(""); + const [statusUrl, setStatusUrl] = useState(""); + const [localKeyStatus, setLocalKeyStatus] = useState<{ + url: string; + configured: boolean; + } | null>(null); + const [apiKey, setApiKey] = useState(""); + const [loaded, setLoaded] = useState(false); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); @@ -31,7 +49,13 @@ export function OpenAiEndpointField({ let active = true; void getOpenAiVoiceEndpoints().then( (settings) => { - if (active) setUrl(settings[kind] ?? ""); + if (active) { + const value = settings[kind] ?? ""; + setUrl(value); + setSavedUrl(value); + setStatusUrl(value); + setLoaded(true); + } }, (cause) => { if (active) setError(String(cause)); @@ -42,12 +66,65 @@ export function OpenAiEndpointField({ }; }, [kind]); + const changed = url.trim() !== savedUrl; + const keyConfigured = + !changed && + (localKeyStatus?.url === savedUrl + ? localKeyStatus.configured + : statusUrl === savedUrl && configured); + const save = async () => { setSaving(true); setError(null); + let targetUrl = savedUrl; + let urlSaved = false; + const savingKey = Boolean(apiKey.trim()); try { - await setOpenAiVoiceEndpoint(kind, url); - setUrl((await getOpenAiVoiceEndpoints())[kind] ?? ""); + // The key command uses the persisted endpoint; commit the displayed URL first. + if (changed) { + await setOpenAiVoiceEndpoint(kind, url); + urlSaved = true; + targetUrl = (await getOpenAiVoiceEndpoints())[kind] ?? ""; + setUrl(targetUrl); + setSavedUrl(targetUrl); + setLocalKeyStatus({ url: targetUrl, configured: false }); + } + if (savingKey) { + await onSaveKey(apiKey); + setApiKey(""); + setLocalKeyStatus({ url: targetUrl, configured: true }); + } else if (changed) { + // Metadata-only lookup; never request the Keychain secret to render settings. + const status = await getOpenAiVoiceStatus(); + setLocalKeyStatus({ + url: targetUrl, + configured: status[`${kind}Configured`], + }); + } + } catch (cause) { + const message = cause instanceof Error ? cause.message : String(cause); + setError( + urlSaved + ? t( + savingKey + ? "voice.endpointSavedKeyError" + : "voice.endpointSavedStatusError", + { error: message }, + ) + : message, + ); + } finally { + setSaving(false); + } + }; + + const clear = async () => { + setSaving(true); + setError(null); + try { + await onClearKey(); + setApiKey(""); + setLocalKeyStatus({ url: savedUrl, configured: false }); } catch (cause) { setError(cause instanceof Error ? cause.message : String(cause)); } finally { @@ -60,27 +137,57 @@ export function OpenAiEndpointField({ -
- setUrl(event.target.value)} - placeholder={DEFAULT_URLS[kind]} - autoComplete="off" - spellCheck={false} - /> + setUrl(event.target.value)} + placeholder={DEFAULT_URLS[kind]} + autoComplete="off" + spellCheck={false} + /> +

+ {t("voice.endpointDefaultHint")} +

+ + setApiKey(event.target.value)} + placeholder={keyConfigured ? "••••••••" : "sk-…"} + autoComplete="off" + spellCheck={false} + /> +
+ {keyConfigured ? ( + + ) : null}

- {t("voice.endpointDefaultHint")} + {changed + ? t("voice.endpointUnsavedHint") + : keyConfigured + ? t("voice.openAiApiKeyConfigured") + : t("voice.openAiApiKeyNotConfigured")}

{error ? (

diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx index 84a3ce074..d65312e0c 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx @@ -29,6 +29,7 @@ describe("RealtimeVoiceSettings", () => { beforeEach(async () => { window.localStorage.clear(); vi.clearAllMocks(); + openAiVoiceMocks.getStatus.mockResolvedValue({ realtimeConfigured: true }); await i18n.changeLanguage("en"); }); @@ -67,7 +68,7 @@ describe("RealtimeVoiceSettings", () => { ).toHaveTextContent("Debug — show agent routing"); }); - it("shows the default realtime URL above its URL-scoped key", async () => { + it("saves the displayed realtime URL before its URL-scoped key with one action", async () => { const user = userEvent.setup(); renderWithProviders(); @@ -79,16 +80,62 @@ describe("RealtimeVoiceSettings", () => { screen.getByLabelText("Realtime endpoint URL"), "ws://127.0.0.1:18870/v1/realtime", ); - await user.click(screen.getByRole("button", { name: "Save URL" })); + await user.type(screen.getByLabelText("OpenAI API key"), " sk-shared "); + expect(screen.getAllByRole("button", { name: "Save" })).toHaveLength(1); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(openAiVoiceMocks.setEndpoint).toHaveBeenCalledWith( "realtime", "ws://127.0.0.1:18870/v1/realtime", ); + expect(openAiVoiceMocks.setApiKey).toHaveBeenCalledWith(" sk-shared "); + expect( + openAiVoiceMocks.setEndpoint.mock.invocationCallOrder[0], + ).toBeLessThan(openAiVoiceMocks.setApiKey.mock.invocationCallOrder[0]); + }); - await user.type(screen.getByLabelText("OpenAI API key"), " sk-shared "); - await user.click(screen.getByRole("button", { name: "Save key" })); + it("does not save a key if the endpoint URL is rejected", async () => { + openAiVoiceMocks.setEndpoint.mockRejectedValueOnce( + new Error("Invalid URL"), + ); + const user = userEvent.setup(); + renderWithProviders(); + await user.type(screen.getByLabelText("Realtime endpoint URL"), "invalid"); + await user.type(screen.getByLabelText("OpenAI API key"), "new-key"); + await user.click(screen.getByRole("button", { name: "Save" })); - expect(openAiVoiceMocks.setApiKey).toHaveBeenCalledWith(" sk-shared "); + expect(await screen.findByRole("alert")).toHaveTextContent("Invalid URL"); + expect(openAiVoiceMocks.setApiKey).not.toHaveBeenCalled(); + }); + + it("shows masked placeholder text for a saved key without filling the input", async () => { + renderWithProviders(); + expect( + await screen.findByText("Key saved for this URL in macOS Keychain."), + ).toBeInTheDocument(); + const input = screen.getByLabelText("OpenAI API key"); + expect(input).toHaveValue(""); + expect(input).toHaveAttribute("placeholder", "••••••••"); + }); + + it("saves a URL without a key but shows that it cannot be used yet", async () => { + openAiVoiceMocks.getStatus.mockResolvedValue({ + realtimeConfigured: false, + }); + const user = userEvent.setup(); + renderWithProviders(); + await user.type( + screen.getByLabelText("Realtime endpoint URL"), + "ws://127.0.0.1:18870/v1/realtime", + ); + await user.click(screen.getByRole("button", { name: "Save" })); + + expect(openAiVoiceMocks.setApiKey).not.toHaveBeenCalled(); + expect( + await screen.findByText( + "No key set. Enter a key for this URL before using it. Default OpenAI endpoints share one key.", + ), + ).toBeInTheDocument(); }); it("reveals the supported advanced session controls", async () => { diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx index 50f9d6c66..33e7d317b 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.tsx @@ -31,7 +31,6 @@ import { setOpenAiRealtimeApiKey, } from "../api/openAiVoice"; import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; -import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; import { OpenAiEndpointField } from "./OpenAiEndpointField"; import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; import { SimpleVoicePickerDialog } from "./SimpleVoicePickerDialog"; @@ -132,13 +131,10 @@ export function RealtimeVoiceSettings() { -

diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index c0e7b5f98..149e119e2 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -98,6 +98,9 @@ const openAiStatusState = vi.hoisted(() => ({ }, })); const openAiApiMocks = vi.hoisted(() => ({ + getStatus: vi.fn(() => + Promise.resolve({ sttKeySaved: false, ttsKeySaved: false }), + ), getEndpoints: vi.fn(() => Promise.resolve({ realtime: null as string | null, @@ -120,6 +123,7 @@ vi.mock("../api/openAiVoice", () => ({ setOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), clearOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), getOpenAiVoiceEndpoints: openAiApiMocks.getEndpoints, + getOpenAiVoiceStatus: openAiApiMocks.getStatus, setOpenAiVoiceEndpoint: openAiApiMocks.setEndpoint, setOpenAiPlaybackSpeed: vi.fn(() => Promise.resolve()), setOpenAiSpeechVoice: openAiApiMocks.setSpeechVoice, @@ -637,7 +641,7 @@ describe("VoiceSettings", () => { ).toBeInTheDocument(); expect(screen.getByText("Playback speed")).toBeInTheDocument(); expect( - screen.getAllByText("Saved securely for this endpoint URL."), + screen.getAllByText("Key saved for this URL in macOS Keychain."), ).toHaveLength(2); }); @@ -651,11 +655,39 @@ describe("VoiceSettings", () => { screen.getByLabelText("OpenAI speech-to-text API key"), "stt-secret", ); - await user.click(screen.getAllByRole("button", { name: "Save key" })[0]); + await user.click(screen.getAllByRole("button", { name: "Save" })[0]); expect(openAiApiMocks.setSttApiKey).toHaveBeenCalledWith("stt-secret"); }); + it.each([ + ["stt", "Speech-to-text endpoint URL", "OpenAI speech-to-text API key"], + ["tts", "Text-to-speech endpoint URL", "OpenAI text-to-speech API key"], + ] as const)("saves a custom %s URL before its key with one click", async (kind, urlLabel, keyLabel) => { + inputState.backend = kind === "stt" ? "openai" : "parakeet"; + outputState.backend = kind === "tts" ? "openai" : "pocket"; + setupState.current = setup(pocketStatus()); + renderWithProviders(); + const user = userEvent.setup(); + const url = + kind === "stt" + ? "ws://127.0.0.1:18870/v1/realtime?intent=transcription" + : "http://127.0.0.1:18870/v1/audio/speech"; + await user.type(screen.getByLabelText(urlLabel), url); + await user.type(screen.getByLabelText(keyLabel), "local-test"); + expect(screen.getAllByRole("button", { name: "Save" })).toHaveLength(1); + await user.click(screen.getByRole("button", { name: "Save" })); + const saveKey = + kind === "stt" + ? openAiApiMocks.setSttApiKey + : openAiApiMocks.setTtsApiKey; + expect(openAiApiMocks.setEndpoint).toHaveBeenCalledWith(kind, url); + expect(saveKey).toHaveBeenCalledWith("local-test"); + expect(openAiApiMocks.setEndpoint.mock.invocationCallOrder[0]).toBeLessThan( + saveKey.mock.invocationCallOrder[0], + ); + }); + it("labels purpose-specific environment overrides", async () => { outputState.backend = "openai"; openAiStatusState.current = { @@ -698,7 +730,7 @@ describe("VoiceSettings", () => { screen.getByLabelText("OpenAI text-to-speech API key"), "tts-secret", ); - await user.click(screen.getByRole("button", { name: "Save key" })); + await user.click(screen.getByRole("button", { name: "Save" })); expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); }); diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index f5e383e64..380984816 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -62,7 +62,6 @@ import { SiriVoiceSettings } from "./SiriVoiceSettings"; import { PlaybackSpeedRow } from "./PlaybackSpeedRow"; import { SimpleVoicePickerDialog } from "./SimpleVoicePickerDialog"; import { useOpenAiVoiceSetup } from "../hooks/useOpenAiVoiceSetup"; -import { OpenAiApiKeyField } from "./OpenAiApiKeyField"; import { OpenAiEndpointField } from "./OpenAiEndpointField"; import { RealtimeVoiceSettings } from "./RealtimeVoiceSettings"; import { @@ -396,12 +395,10 @@ export function VoiceSettings() { -

{openAiError ?? @@ -491,12 +488,10 @@ export function VoiceSettings() { -

{openAiError ?? diff --git a/src/shared/i18n/locales/en/settings.json b/src/shared/i18n/locales/en/settings.json index 3095f17de..e67455d51 100644 --- a/src/shared/i18n/locales/en/settings.json +++ b/src/shared/i18n/locales/en/settings.json @@ -952,9 +952,11 @@ "realtimeEndpoint": "Realtime endpoint URL", "openAiSttEndpoint": "Speech-to-text endpoint URL", "openAiTtsEndpoint": "Text-to-speech endpoint URL", - "saveEndpoint": "Save URL", - "endpointDefaultHint": "Leave blank to use the OpenAI URL shown above. API keys are saved separately for each URL.", - "realtimeApiKeyDescription": "Saved securely in macOS Keychain and shared by Realtime listening and speech.", + "saveEndpointSettings": "Save", + "endpointDefaultHint": "Leave blank to use the OpenAI URL shown above. Custom URLs have separate keys; default OpenAI URLs share one.", + "endpointUnsavedHint": "Save this URL and key together. If you leave the key blank, an existing key for this URL is kept; a URL without a key cannot be used.", + "endpointSavedKeyError": "URL saved, but the key was not saved: {{error}}", + "endpointSavedStatusError": "URL saved, but its key status could not be checked: {{error}}", "realtimeCreateResponse": "Respond automatically", "realtimeCreateResponseDescription": "Generate a voice assistant response when a detected user turn ends.", "realtimeEagerness": "Turn-taking eagerness", @@ -1067,9 +1069,8 @@ "noVoiceSelected": "No voice selected", "openMicrophoneSettings": "Open Microphone Settings", "openMicrophoneSettingsError": "Couldn't open Microphone Settings. Open System Settings and select Privacy & Security > Microphone.", - "openAiApiKeyConfigured": "Saved securely for this endpoint URL.", - "openAiApiKeyNotConfigured": "Keys are stored per endpoint URL; default OpenAI endpoints share a key. They are separate from Goose provider credentials.", - "openAiApiKeySaved": "API key saved", + "openAiApiKeyConfigured": "Key saved for this URL in macOS Keychain.", + "openAiApiKeyNotConfigured": "No key set. Enter a key for this URL before using it. Default OpenAI endpoints share one key.", "openAiChecking": "Checking OpenAI voice settings…", "openAiEnvironmentOverride": "Development configuration is overridden by the Berd process environment.", "openAiSttApiKey": "OpenAI speech-to-text API key", @@ -1092,7 +1093,6 @@ "removeModelTitle": "Remove {{model}}?", "removingModel": "Removing model…", "retryDownload": "Retry model download", - "saveApiKey": "Save key", "settingsDescription": "Choose how speech and your coding agent work together.", "siriLanguage": "Language", "siriLoading": "Loading Siri voices…", diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 0447b0dbe..743b4928f 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -951,9 +951,11 @@ "realtimeEndpoint": "URL del servicio en tiempo real", "openAiSttEndpoint": "URL del servicio de voz a texto", "openAiTtsEndpoint": "URL del servicio de texto a voz", - "saveEndpoint": "Guardar URL", - "endpointDefaultHint": "Deja este campo vacío para usar la URL de OpenAI indicada arriba. Las claves API se guardan por separado para cada URL.", - "realtimeApiKeyDescription": "Se guarda de forma segura en el llavero de macOS y se comparte entre la escucha y la voz de Realtime.", + "saveEndpointSettings": "Guardar", + "endpointDefaultHint": "Deja este campo vacío para usar la URL de OpenAI indicada arriba. Las URL personalizadas tienen claves separadas; las URL predeterminadas de OpenAI comparten una.", + "endpointUnsavedHint": "Guarda esta URL y la clave juntas. Si dejas la clave vacía, se conserva la clave existente para esta URL; una URL sin clave no se puede usar.", + "endpointSavedKeyError": "Se guardó la URL, pero no la clave: {{error}}", + "endpointSavedStatusError": "Se guardó la URL, pero no se pudo comprobar el estado de la clave: {{error}}", "realtimeCreateResponse": "Responder automáticamente", "realtimeCreateResponseDescription": "Genera una respuesta del asistente de voz cuando termina un turno detectado del usuario.", "realtimeEagerness": "Rapidez para tomar el turno", @@ -1066,9 +1068,8 @@ "noVoiceSelected": "No hay ninguna voz seleccionada", "openMicrophoneSettings": "Abrir ajustes del micrófono", "openMicrophoneSettingsError": "No se pudieron abrir los ajustes del micrófono. Abre Ajustes del Sistema y selecciona Privacidad y seguridad > Micrófono.", - "openAiApiKeyConfigured": "Guardada de forma segura y compartida por la transcripción y la reproducción de voz de OpenAI.", - "openAiApiKeyNotConfigured": "Esta clave se comparte entre la transcripción y la reproducción de voz de OpenAI. Es independiente de las credenciales de proveedor que usa Goose.", - "openAiApiKeySaved": "Clave API guardada", + "openAiApiKeyConfigured": "Clave guardada para esta URL en el llavero de macOS.", + "openAiApiKeyNotConfigured": "No hay ninguna clave configurada. Introduce una para esta URL antes de usarla. Los servicios predeterminados de OpenAI comparten una clave.", "openAiChecking": "Comprobando los ajustes de voz de OpenAI…", "openAiEnvironmentOverride": "La configuración de desarrollo está reemplazada por el entorno del proceso de Berd.", "openAiSttApiKey": "Clave API de voz a texto de OpenAI", @@ -1091,7 +1092,6 @@ "removeModelTitle": "¿Eliminar {{model}}?", "removingModel": "Eliminando modelo…", "retryDownload": "Reintentar descarga del modelo", - "saveApiKey": "Guardar clave", "settingsDescription": "Elige cómo funcionan juntos la voz y tu agente de programación.", "siriLanguage": "Idioma", "siriLoading": "Cargando voces de Siri…", diff --git a/tests/e2e/voice-endpoints.spec.ts b/tests/e2e/voice-endpoints.spec.ts index 64f895618..101c3c101 100644 --- a/tests/e2e/voice-endpoints.spec.ts +++ b/tests/e2e/voice-endpoints.spec.ts @@ -27,7 +27,7 @@ test.use({ screenshot: "off", trace: "off", video: "off" }); async function saveEndpoint(page: Page, label: string, url: string) { const input = page.getByLabel(label); await input.fill(url); - await input.locator("..").getByRole("button", { name: "Save URL" }).click(); + await input.locator("..").getByRole("button", { name: "Save" }).click(); await expect(input).toHaveValue(url); } From 40b8b8cf0b7a8512547d68ebc58da759884c0e9c Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 15:42:02 -0400 Subject: [PATCH 06/34] Clarify saved voice endpoint key masking --- src/features/voice-conversation/ui/OpenAiEndpointField.tsx | 2 +- .../voice-conversation/ui/RealtimeVoiceSettings.test.tsx | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index 8fe3a8c9a..39e5691b2 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -157,7 +157,7 @@ export function OpenAiEndpointField({ type="password" value={apiKey} onChange={(event) => setApiKey(event.target.value)} - placeholder={keyConfigured ? "••••••••" : "sk-…"} + placeholder={keyConfigured ? "••••••••••••••••••••" : "sk-…"} autoComplete="off" spellCheck={false} /> diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx index d65312e0c..bf27e8273 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx @@ -115,7 +115,7 @@ describe("RealtimeVoiceSettings", () => { ).toBeInTheDocument(); const input = screen.getByLabelText("OpenAI API key"); expect(input).toHaveValue(""); - expect(input).toHaveAttribute("placeholder", "••••••••"); + expect(input).toHaveAttribute("placeholder", "••••••••••••••••••••"); }); it("saves a URL without a key but shows that it cannot be used yet", async () => { @@ -131,6 +131,10 @@ describe("RealtimeVoiceSettings", () => { await user.click(screen.getByRole("button", { name: "Save" })); expect(openAiVoiceMocks.setApiKey).not.toHaveBeenCalled(); + expect(screen.getByLabelText("OpenAI API key")).toHaveAttribute( + "placeholder", + "sk-…", + ); expect( await screen.findByText( "No key set. Enter a key for this URL before using it. Default OpenAI endpoints share one key.", From b5d06ea35cf24218b94401a74a6dc457ffaa7be0 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 15:49:03 -0400 Subject: [PATCH 07/34] Keep spoken assistant replies in one transcript card --- .../projection/buildTranscriptItems.ts | 5 ++++ .../transcriptProjectionCache.test.ts | 27 +++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.ts b/src/features/chat/transcript/projection/buildTranscriptItems.ts index 0b24f6686..2c0d0a686 100644 --- a/src/features/chat/transcript/projection/buildTranscriptItems.ts +++ b/src/features/chat/transcript/projection/buildTranscriptItems.ts @@ -1297,6 +1297,11 @@ function canProjectAssistantTextFragments( if (visibleContent.length !== 1 || visibleContent[0]?.type !== "text") { return false; } + // Speech belongs to the whole assistant reply. Fragmenting its text would + // duplicate the playback status and draw each paragraph as a separate card. + if (visibleContent[0].speech) { + return false; + } if ( message.metadata?.attachments?.length || message.metadata?.chips?.length diff --git a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts index 7a5db3617..629415887 100644 --- a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts +++ b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts @@ -123,6 +123,33 @@ describe("transcript projection cache", () => { ); }); + it("keeps a spoken multi-paragraph reply in one assistant card", () => { + const assistant = messageWithContent( + "assistant-voice-story", + "assistant", + [ + { + type: "text", + text: multiParagraphText("story paragraph", 3, 20), + speech: { status: "speaking" }, + }, + ], + utc(2026, 6, 4, 10), + { completionStatus: "completed" }, + ); + + const snapshot = update(createTranscriptProjectionCache(), [assistant]); + + expect( + snapshot.rows.filter((row) => row.messageId === assistant.id), + ).toHaveLength(1); + expect(messageRow(snapshot, assistant.id)).toMatchObject({ + kind: "message", + rowId: "message:assistant-voice-story", + }); + expect(snapshot.fragmentRowCount).toBe(0); + }); + it("keeps long markdown tables on whole-message rows", () => { const cache = createTranscriptProjectionCache(); const assistant = message( From df35a6db2cf8ce446d7fe66763306c68450ee418 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 17:01:36 -0400 Subject: [PATCH 08/34] fix(voice): secure endpoint URLs and reset saved overrides --- src-tauri/crates/berd-call/src/main.rs | 12 ++- src-tauri/src/commands/openai_audio.rs | 74 ---------------- .../src/commands/openai_voice_endpoints.rs | 86 ++++++++++++++++++- src-tauri/src/commands/voice_settings.rs | 51 ++++++++++- 4 files changed, 142 insertions(+), 81 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 599f1e447..c9229e4d5 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -5324,11 +5324,18 @@ fn parse_endpoint_url(value: &str, flag: &str, websocket: bool) -> Result() + .is_ok_and(|address| address.is_loopback()) + })) || !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() { - return Err(format!("{flag} requires a full URL with the correct protocol and no embedded credentials or fragment")); + return Err(format!("{flag} requires a full URL with the correct protocol, HTTPS/WSS outside loopback, and no embedded credentials or fragment")); } Ok(url.to_string()) } @@ -9641,6 +9648,9 @@ mod tests { chained.endpoints.stt.as_deref(), Some("wss://proxy.example/v1/realtime?intent=transcription") ); + assert!(parse_endpoint_url("http://example.test/speech", "--tts-url", false).is_err()); + assert!(parse_endpoint_url("ws://example.test/realtime", "--stt-url", true).is_err()); + assert!(parse_endpoint_url("http://127.0.0.1:18870/speech", "--tts-url", false).is_ok()); assert!(parse_args(&args(&[ "berd-call", "session", diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index c3dabd1f5..8bf9a7038 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -38,7 +38,6 @@ use berd_call::input::InputDuringTtsPolicy; #[cfg(any(test, target_os = "macos"))] use std::time::Instant; -const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const DEFAULT_TRANSCRIPTION_MODEL: &str = "gpt-live-transcribe"; const DEFAULT_TTS_MODEL: &str = "gpt-4o-mini-tts"; const DEFAULT_TTS_VOICE: &str = "marin"; @@ -186,34 +185,6 @@ pub(crate) fn stt_api_key() -> Result { openai_voice_credentials::require(OpenAiVoiceCredential::SpeechToText) } -fn normalize_openai_base_url(raw_url: String) -> Result { - let mut url = reqwest::Url::parse(&raw_url) - .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; - if url.scheme() != "https" { - return Err("OpenAI voice endpoint must use HTTPS".to_string()); - } - let path = url.path().trim_end_matches('/').to_string(); - if path.is_empty() { - let path = if path.ends_with("/v1") { - path - } else { - format!("{path}/v1") - }; - url.set_path(&path); - } else { - url.set_path(&path); - } - url.set_fragment(None); - Ok(url.to_string().trim_end_matches('/').to_string()) -} - -pub(crate) fn base_url() -> Result { - if let Some(base_url) = env_trimmed(BASE_URL_ENV) { - return normalize_openai_base_url(base_url); - } - Ok(DEFAULT_BASE_URL.to_string()) -} - pub(crate) fn realtime_endpoint() -> Result { openai_voice_endpoints::effective_url(VoiceEndpointKind::Stt) } @@ -252,14 +223,6 @@ fn stt_configuration_source() -> OpenAiVoiceConfigurationSource { } } -pub(crate) fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { - let mut url = reqwest::Url::parse(base_url) - .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; - let base_path = url.path().trim_end_matches('/'); - url.set_path(&format!("{base_path}/{}", path.trim_start_matches('/'))); - Ok(url.to_string()) -} - fn voice_settings_path() -> Result { Ok(crate::services::goose_config::config_path()? .parent() @@ -1163,43 +1126,6 @@ mod tests { assert!(!active.load(Ordering::SeqCst)); } - #[test] - fn voice_base_url_configuration_resolves_to_the_v1_api_root() { - assert_eq!( - normalize_openai_base_url("https://proxy.example".to_string()).unwrap(), - "https://proxy.example/v1" - ); - assert_eq!( - normalize_openai_base_url("https://proxy.example/v1/".to_string()).unwrap(), - "https://proxy.example/v1" - ); - } - - #[test] - fn openai_voice_endpoints_require_https() { - assert_eq!( - normalize_openai_base_url("http://proxy.example".to_string()) - .expect_err("plaintext endpoint must be rejected"), - "OpenAI voice endpoint must use HTTPS" - ); - } - - #[test] - fn openai_base_url_preserves_custom_paths_and_query_parameters() { - assert_eq!( - normalize_openai_base_url("https://proxy.example".to_string()).unwrap(), - "https://proxy.example/v1" - ); - let base = normalize_openai_base_url( - "https://proxy.example/openai?api-version=2026-01-01".to_string(), - ) - .unwrap(); - assert_eq!( - endpoint_for_base_url(&base, "audio/speech").unwrap(), - "https://proxy.example/openai/audio/speech?api-version=2026-01-01" - ); - } - #[test] fn voice_configuration_uses_berd_scoped_environment_names() { assert_eq!(BASE_URL_ENV, "BERD_OPENAI_VOICE_BASE_URL"); diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index 637a9c166..ac7540f7f 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -9,6 +9,8 @@ pub(crate) const REALTIME_DEFAULT: &str = "wss://api.openai.com/v1/realtime"; pub(crate) const STT_DEFAULT: &str = "wss://api.openai.com/v1/realtime?intent=transcription"; pub(crate) const TTS_DEFAULT: &str = "https://api.openai.com/v1/audio/speech"; const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; +const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; +const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -99,16 +101,60 @@ fn validate(kind: VoiceEndpointKind, raw: &str) -> Result, String }; if !allowed.contains(&url.scheme()) || url.host_str().is_none() + || (matches!(url.scheme(), "http" | "ws") && !is_loopback(&url)) || !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() { - return Err("Endpoint must be an absolute URL with the correct protocol and no embedded credentials or fragment".into()); + return Err("Endpoint must be a full URL with the correct protocol, HTTPS/WSS outside loopback, and no embedded credentials or fragment".into()); } url.set_fragment(None); Ok(Some(url.to_string())) } +fn is_loopback(url: &url::Url) -> bool { + match url.host() { + Some(url::Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), + Some(url::Host::Ipv4(address)) => address.is_loopback(), + Some(url::Host::Ipv6(address)) => address.is_loopback(), + None => false, + } +} + +fn normalize_base_url(raw_url: String) -> Result { + let mut url = url::Url::parse(&raw_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + if url.scheme() != "https" { + return Err("OpenAI voice endpoint must use HTTPS".to_string()); + } + let path = url.path().trim_end_matches('/').to_string(); + if path.is_empty() { + url.set_path("/v1"); + } else { + url.set_path(&path); + } + url.set_fragment(None); + Ok(url.to_string().trim_end_matches('/').to_string()) +} + +fn base_url() -> Result { + let base = std::env::var(BASE_URL_ENV) + .ok() + .map(|value| value.trim().to_string()); + match base.filter(|value| !value.is_empty()) { + Some(base) => normalize_base_url(base), + None => Ok(DEFAULT_BASE_URL.to_string()), + } +} + +fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { + let mut url = url::Url::parse(base_url) + .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; + let base_path = url.path().trim_end_matches('/'); + url.set_path(&format!("{base_path}/{}", path.trim_start_matches('/'))); + Ok(url.to_string()) +} + pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { if let Some(saved) = read_settings()?.get(kind) { return Ok(saved.to_string()); @@ -116,13 +162,13 @@ pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { if !matches!(kind, VoiceEndpointKind::Realtime) && std::env::var_os("BERD_OPENAI_VOICE_BASE_URL").is_some() { - let base = super::openai_audio::base_url()?; + let base = base_url()?; let path = match kind { VoiceEndpointKind::Stt => "realtime", VoiceEndpointKind::Tts => "audio/speech", VoiceEndpointKind::Realtime => unreachable!(), }; - let mut url = url::Url::parse(&super::openai_audio::endpoint_for_base_url(&base, path)?) + let mut url = url::Url::parse(&endpoint_for_base_url(&base, path)?) .map_err(|error| format!("Invalid OpenAI voice endpoint: {error}"))?; if matches!(kind, VoiceEndpointKind::Stt) { url.set_scheme("wss").expect("https can become wss"); @@ -156,10 +202,36 @@ pub(crate) fn reset() -> Result<(), String> { persist(&VoiceEndpointSettings::default()) } +pub(crate) fn restore(settings: &VoiceEndpointSettings) -> Result<(), String> { + persist(settings) +} + #[cfg(test)] mod tests { use super::*; + #[test] + fn legacy_base_url_keeps_the_v1_root_and_custom_path() { + assert_eq!( + normalize_base_url("https://proxy.example".into()).unwrap(), + "https://proxy.example/v1" + ); + assert_eq!( + normalize_base_url("https://proxy.example/v1/".into()).unwrap(), + "https://proxy.example/v1" + ); + assert_eq!( + normalize_base_url("http://proxy.example".into()).unwrap_err(), + "OpenAI voice endpoint must use HTTPS" + ); + let base = normalize_base_url("https://proxy.example/openai?api-version=2026-01-01".into()) + .unwrap(); + assert_eq!( + endpoint_for_base_url(&base, "audio/speech").unwrap(), + "https://proxy.example/openai/audio/speech?api-version=2026-01-01" + ); + } + #[test] fn endpoints_are_full_urls_with_independent_openai_defaults() { assert_eq!(VoiceEndpointKind::Realtime.default_url(), REALTIME_DEFAULT); @@ -172,6 +244,14 @@ mod tests { ) .is_ok()); assert!(validate(VoiceEndpointKind::Tts, "wss://example.test/audio/speech").is_err()); + assert!(validate(VoiceEndpointKind::Tts, "http://example.test/audio/speech").is_err()); + assert!(validate(VoiceEndpointKind::Stt, "ws://example.test/realtime").is_err()); + assert!(validate( + VoiceEndpointKind::Tts, + "http://localhost:18870/v1/audio/speech" + ) + .is_ok()); + assert!(validate(VoiceEndpointKind::Stt, "ws://[::1]:18870/v1/realtime").is_ok()); assert!(validate( VoiceEndpointKind::Realtime, "wss://key@example.test/realtime" diff --git a/src-tauri/src/commands/voice_settings.rs b/src-tauri/src/commands/voice_settings.rs index b58dba7ea..7ae40b3c1 100644 --- a/src-tauri/src/commands/voice_settings.rs +++ b/src-tauri/src/commands/voice_settings.rs @@ -1,6 +1,6 @@ use tauri::{AppHandle, Emitter, State}; -use super::{openai_audio, pocket_voice, siri_voice}; +use super::{openai_audio, openai_voice_endpoints, pocket_voice, siri_voice}; const OPENAI_SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; @@ -58,6 +58,24 @@ fn reset_transaction( Ok(()) } +fn reset_openai_with_endpoints( + previous_endpoints: &E, + mut reset_endpoints: impl FnMut() -> Result<(), String>, + mut reset_playback: impl FnMut() -> Result<(), String>, + mut restore_endpoints: impl FnMut(&E) -> Result<(), String>, +) -> Result<(), String> { + reset_endpoints()?; + if let Err(error) = reset_playback() { + let rollback = restore_endpoints(previous_endpoints) + .err() + .map(|cause| format!("endpoints: {cause}")) + .into_iter() + .collect(); + return Err(transaction_error(error, rollback)); + } + Ok(()) +} + #[tauri::command] pub fn reset_all_voice_backend_settings( app: AppHandle, @@ -74,12 +92,20 @@ pub fn reset_all_voice_backend_settings( let previous_pocket = pocket_voice::settings(&pocket_base); let previous_siri = siri_voice::read_settings(&siri_path); + let previous_endpoints = openai_voice_endpoints::get_openai_voice_endpoints()?; reset_transaction( &previous_pocket, &previous_siri, || pocket_voice::write_settings(&pocket_base, &Default::default()), || siri_voice::write_settings(&siri_path, &Default::default()), - || openai_audio::replace_voice_settings(&openai_state, &Default::default()), + || { + reset_openai_with_endpoints( + &previous_endpoints, + openai_voice_endpoints::reset, + || openai_audio::replace_voice_settings(&openai_state, &Default::default()), + openai_voice_endpoints::restore, + ) + }, |settings| pocket_voice::write_settings(&pocket_base, settings), |settings| siri_voice::write_settings(&siri_path, settings), )?; @@ -94,7 +120,7 @@ pub fn reset_all_voice_backend_settings( mod tests { use std::cell::Cell; - use super::reset_transaction; + use super::{reset_openai_with_endpoints, reset_transaction}; #[test] fn restores_completed_resets_when_a_later_backend_fails() { @@ -127,4 +153,23 @@ mod tests { assert_eq!(pocket.get(), 7); assert_eq!(siri.get(), 8); } + + #[test] + fn restores_endpoint_urls_when_playback_reset_fails() { + let endpoints = Cell::new(7); + let result = reset_openai_with_endpoints( + &7, + || { + endpoints.set(0); + Ok(()) + }, + || Err("playback unavailable".to_string()), + |previous| { + endpoints.set(*previous); + Ok(()) + }, + ); + assert!(result.is_err()); + assert_eq!(endpoints.get(), 7); + } } From 9572cf72da3ab8df48d602054dd2a04286cc70cb Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 17:02:09 -0400 Subject: [PATCH 09/34] fix(voice): use the default OpenAI key for dictation --- src-tauri/src/commands/openai_realtime.rs | 8 ++------ src-tauri/src/commands/openai_voice_credentials.rs | 14 ++++++++++++++ 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index 3de536baa..271da0840 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -78,7 +78,7 @@ pub struct OpenAiRealtimeSession { #[tauri::command] pub async fn get_openai_realtime_status() -> Result { - let configured = openai_voice_credentials::is_present(OpenAiVoiceCredential::Realtime)?; + let configured = openai_voice_credentials::is_default_present()?; Ok(OpenAiRealtimeStatus { configured }) } @@ -103,11 +103,7 @@ pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { #[tauri::command] pub async fn create_openai_realtime_session() -> Result { - let endpoint = openai_voice_endpoints::effective_url(VoiceEndpointKind::Realtime)?; - if endpoint != VoiceEndpointKind::Realtime.default_url() { - return Err("Realtime dictation requires the default OpenAI endpoint; custom Realtime URLs are supported for Expert-Spokesperson conversations".into()); - } - let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; + let api_key = openai_voice_credentials::require_default_realtime()?; let response = realtime_transcription_client_secret_request(&reqwest::Client::new(), &api_key) .send() .await diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 1e7f1f0ae..356f58d32 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -84,6 +84,14 @@ pub(crate) fn read(credential: OpenAiVoiceCredential) -> Result, /// Check only Keychain item metadata; status polling must never request secret access. pub(crate) fn is_present(credential: OpenAiVoiceCredential) -> Result { let account = account(credential)?; + is_present_account(&account) +} + +pub(crate) fn is_default_present() -> Result { + is_present_account(KEYCHAIN_ACCOUNT) +} + +fn is_present_account(account: &str) -> Result { #[cfg(target_os = "macos")] { use security_framework::item::{ItemClass, ItemSearchOptions}; @@ -121,6 +129,12 @@ pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result Result { + read_account(KEYCHAIN_ACCOUNT)?.ok_or_else(|| { + "OpenAI Realtime dictation needs an API key for the default OpenAI endpoint".to_string() + }) +} + #[cfg(test)] mod tests { use super::*; From 6d47cb83403c98f2586b51fc4aee8166e52793a3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 28 Sep 2026 17:02:49 -0400 Subject: [PATCH 10/34] fix(voice): refresh endpoint key state across settings changes --- .../ui/OpenAiEndpointField.test.tsx | 40 +++++++++++++++++++ .../ui/OpenAiEndpointField.tsx | 24 +++++++++-- .../voice-conversation/ui/VoiceSettings.tsx | 4 ++ 3 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx new file mode 100644 index 000000000..10094178c --- /dev/null +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx @@ -0,0 +1,40 @@ +import { screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, expect, it, vi } from "vitest"; +import { renderWithProviders } from "@/test/render"; +import { OpenAiEndpointField } from "./OpenAiEndpointField"; + +const voiceApi = vi.hoisted(() => ({ + getEndpoints: vi.fn(async () => ({ realtime: null, stt: null, tts: null })), + getStatus: vi.fn(async () => ({ sttConfigured: true })), +})); + +vi.mock("../api/openAiVoice", () => ({ + getOpenAiVoiceEndpoints: voiceApi.getEndpoints, + getOpenAiVoiceStatus: voiceApi.getStatus, + setOpenAiVoiceEndpoint: vi.fn(), +})); + +beforeEach(() => vi.clearAllMocks()); + +it("updates a locally saved key indicator when another default service clears the shared key", async () => { + const user = userEvent.setup(); + const onSaveKey = vi.fn(async () => {}); + const props = { + kind: "stt" as const, + label: "STT URL", + keyLabel: "STT key", + onSaveKey, + onClearKey: vi.fn(async () => {}), + }; + const view = renderWithProviders( + , + ); + await screen.findByText("Key saved for this URL in macOS Keychain."); + await user.type(screen.getByLabelText("STT key"), "new-key"); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(onSaveKey).toHaveBeenCalledWith("new-key"); + + view.rerender(); + expect(await screen.findByText(/No key set/)).toBeInTheDocument(); +}); diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index 39e5691b2..4984e32b4 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -39,6 +39,7 @@ export function OpenAiEndpointField({ const [localKeyStatus, setLocalKeyStatus] = useState<{ url: string; configured: boolean; + observedConfigured: boolean; } | null>(null); const [apiKey, setApiKey] = useState(""); const [loaded, setLoaded] = useState(false); @@ -69,7 +70,8 @@ export function OpenAiEndpointField({ const changed = url.trim() !== savedUrl; const keyConfigured = !changed && - (localKeyStatus?.url === savedUrl + (localKeyStatus?.url === savedUrl && + localKeyStatus.observedConfigured === configured ? localKeyStatus.configured : statusUrl === savedUrl && configured); @@ -87,18 +89,28 @@ export function OpenAiEndpointField({ targetUrl = (await getOpenAiVoiceEndpoints())[kind] ?? ""; setUrl(targetUrl); setSavedUrl(targetUrl); - setLocalKeyStatus({ url: targetUrl, configured: false }); + setStatusUrl(targetUrl); + setLocalKeyStatus({ + url: targetUrl, + configured: false, + observedConfigured: configured, + }); } if (savingKey) { await onSaveKey(apiKey); setApiKey(""); - setLocalKeyStatus({ url: targetUrl, configured: true }); + setLocalKeyStatus({ + url: targetUrl, + configured: true, + observedConfigured: configured, + }); } else if (changed) { // Metadata-only lookup; never request the Keychain secret to render settings. const status = await getOpenAiVoiceStatus(); setLocalKeyStatus({ url: targetUrl, configured: status[`${kind}Configured`], + observedConfigured: configured, }); } } catch (cause) { @@ -124,7 +136,11 @@ export function OpenAiEndpointField({ try { await onClearKey(); setApiKey(""); - setLocalKeyStatus({ url: savedUrl, configured: false }); + setLocalKeyStatus({ + url: savedUrl, + configured: false, + observedConfigured: configured, + }); } catch (cause) { setError(cause instanceof Error ? cause.message : String(cause)); } finally { diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index 380984816..f83c11a64 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -158,6 +158,7 @@ export function VoiceSettings() { const [resetDialogOpen, setResetDialogOpen] = useState(false); const [resetting, setResetting] = useState(false); const [resetError, setResetError] = useState(null); + const [endpointResetRevision, setEndpointResetRevision] = useState(0); const input = useVoiceInputPreference( isMacSpeechAvailable(macSpeechSetup.status, macSpeechSetup.loading), ); @@ -238,6 +239,7 @@ export function VoiceSettings() { setResetError(null); try { await resetAllVoiceBackendSettings(); + setEndpointResetRevision((revision) => revision + 1); await setup.refreshSettings(); await siriSetup.refreshSettings(); setRealtimeVoicePreference(getDefaultRealtimeVoicePreference()); @@ -393,6 +395,7 @@ export function VoiceSettings() { input.backend === "openai" ? (

Date: Mon, 28 Sep 2026 17:06:26 -0400 Subject: [PATCH 11/34] fix(voice): pass keychain account without needless borrow --- src-tauri/src/commands/openai_voice_credentials.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 356f58d32..d879195e2 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -98,7 +98,7 @@ fn is_present_account(account: &str) -> Result { match ItemSearchOptions::new() .class(ItemClass::generic_password()) .service(KEYCHAIN_SERVICE) - .account(&account) + .account(account) .load_attributes(true) .skip_authenticated_items(true) .search() @@ -110,7 +110,7 @@ fn is_present_account(account: &str) -> Result { } #[cfg(not(target_os = "macos"))] { - Ok(read_account(&account)?.is_some()) + Ok(read_account(account)?.is_some()) } } From 9632d109848f394036da28a8a4d9960602c75098 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 08:53:16 -0400 Subject: [PATCH 12/34] Set Siri synthesis rate explicitly at 1x --- .../crates/berd-call/native/siri_tts_bridge.m | 9 ++++--- .../native/tests/siri_tts_stream_regression.m | 26 +++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/src-tauri/crates/berd-call/native/siri_tts_bridge.m b/src-tauri/crates/berd-call/native/siri_tts_bridge.m index e8ac60a1a..ce06b7a13 100644 --- a/src-tauri/crates/berd-call/native/siri_tts_bridge.m +++ b/src-tauri/crates/berd-call/native/siri_tts_bridge.m @@ -201,6 +201,11 @@ - (void)finish:(NSError *)error { } if (completion) completion(error); } +static void BerdSetSiriRequestRate(id request, float rate) { + if ([request respondsToSelector:@selector(setRate:)]) { + ((void (*)(id, SEL, float))objc_msgSend)(request, @selector(setRate:), rate); + } +} - (void)synthesizeText:(NSString *)text language:(NSString *)language voiceName:(NSString *)voiceName rate:(float)rate completion:(void (^)(NSError *))completion { @@ -218,9 +223,7 @@ - (void)synthesizeText:(NSString *)text language:(NSString *)language } typedef id (*InitializeRequest)(id, SEL, id, id); id request = ((InitializeRequest)objc_msgSend)([requestClass alloc], selector, text, voice); - if (rate != 1.0f && [request respondsToSelector:@selector(setRate:)]) { - ((void (*)(id, SEL, float))objc_msgSend)(request, @selector(setRate:), rate); - } + BerdSetSiriRequestRate(request, rate); NSXPCConnection *connection = [[NSXPCConnection alloc] initWithMachServiceName:@"com.apple.sirittsd" options:0]; diff --git a/src-tauri/crates/berd-call/native/tests/siri_tts_stream_regression.m b/src-tauri/crates/berd-call/native/tests/siri_tts_stream_regression.m index b86534017..413c27d7d 100644 --- a/src-tauri/crates/berd-call/native/tests/siri_tts_stream_regression.m +++ b/src-tauri/crates/berd-call/native/tests/siri_tts_stream_regression.m @@ -3,6 +3,28 @@ #import "../siri_tts_bridge.m" +@interface BerdRateRecordingRequest : NSObject +@property(nonatomic, assign) float rate; +@property(nonatomic, assign) NSUInteger rateAssignments; +@end + +@implementation BerdRateRecordingRequest +- (void)setRate:(float)rate { + _rate = rate; + _rateAssignments += 1; +} +@end + +static BOOL BerdTestExplicitUnitRate(NSError **error) { + BerdRateRecordingRequest *request = [BerdRateRecordingRequest new]; + BerdSetSiriRequestRate(request, 1.0f); + if (request.rateAssignments != 1 || request.rate != 1.0f) { + if (error) *error = BerdError(106, @"Siri rate 1.0 was not set explicitly."); + return NO; + } + return YES; +} + @interface BerdCapturedSiriPacket : NSObject @property(nonatomic, strong) NSData *data; @property(nonatomic, assign) AudioStreamBasicDescription format; @@ -211,6 +233,10 @@ static BerdAudioComparison BerdCompareAudio(NSData *actualData, NSData *expected int main(void) { @autoreleasepool { NSError *error = nil; + if (!BerdTestExplicitUnitRate(&error)) { + fprintf(stderr, "set Siri rate: %s\n", error.localizedDescription.UTF8String); + return 1; + } if (!BerdTestPCMNormalization(&error)) { fprintf(stderr, "normalize PCM: %s\n", error.localizedDescription.UTF8String); return 1; From b59365e5e114127cf960cd8aa0cf589a29ffb34b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 11:34:55 -0400 Subject: [PATCH 13/34] Return accepted handoff tool output from Berd Call CLI --- src-tauri/crates/berd-call/src/main.rs | 52 +++++++++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index c9229e4d5..6f945b567 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -22,7 +22,7 @@ use berd_call::input::{ VoiceInputRuntime, INPUT_FRAME_SAMPLES, }; use berd_call::openai_realtime_protocol::{ - expert_handoff_message, expert_transcript_message, RealtimeExpertMessage, + accepted_handoff_tool_output, expert_handoff_message, expert_transcript_message, RealtimeExpertMessage, RealtimeExpertMessageMode, RealtimeExpertSpokespersonSession, RealtimeHandoffReminder, RealtimeTranscriptSpeaker, }; @@ -3615,11 +3615,21 @@ fn run_expert_spokesperson_session( call_id, message, } => { - record_and_emit_live_event( + let handoff_id = record_and_emit_live_event( &mut core, &mut emitted_live_token, - LiveSideEvent::Handoff { call_id, message }, + LiveSideEvent::Handoff { + call_id: call_id.clone(), + message, + }, &mut writer, + )? + .ok_or("live handoff did not produce a handoff ID")?; + runtime.as_ref().expect("initialized runtime").send( + SpokespersonCommand::Provider(accepted_handoff_tool_output( + &call_id, + &handoff_id, + )?), )?; } event @ (SpokespersonEvent::Expired(_) | SpokespersonEvent::SessionLost(_)) => { @@ -5006,8 +5016,12 @@ fn record_and_emit_live_event( emitted_live_token: &mut u64, event: LiveSideEvent, writer: &mut impl Write, -) -> Result<(), String> { - let (_, expert_delivery) = core.record_live_event_with_delivery(event)?; +) -> Result, String> { + let (recorded, expert_delivery) = core.record_live_event_with_delivery(event)?; + let handoff_id = match recorded.payload { + LiveSideEvent::Handoff { call_id, .. } => Some(call_id), + _ => None, + }; emit_live_events(core, emitted_live_token, writer)?; if let Some(delivery) = expert_delivery { write_message( @@ -5020,7 +5034,7 @@ fn record_and_emit_live_event( }, )?; } - Ok(()) + Ok(handoff_id) } fn publish_live_response_if_complete( @@ -7574,6 +7588,32 @@ mod tests { assert_eq!(messages[1]["confirmed_token"], 1); } + #[test] + fn live_handoff_exposes_its_id_for_provider_tool_output() { + let mut core = RealtimeExpertSpokespersonSession::new(0, "external-test"); + let mut emitted_token = 0; + let mut output = Vec::new(); + let handoff_id = record_and_emit_live_event( + &mut core, + &mut emitted_token, + LiveSideEvent::Handoff { + call_id: "provider-call-1".into(), + message: "Look something up".into(), + }, + &mut output, + ) + .unwrap() + .expect("a live handoff must expose its generated ID"); + assert_eq!(handoff_id, "handoff-external-test-1"); + let tool_output = berd_call::openai_realtime_protocol::accepted_handoff_tool_output( + "provider-call-1", + &handoff_id, + ) + .unwrap(); + assert_eq!(tool_output["type"], "conversation.item.create"); + assert_eq!(tool_output["item"]["call_id"], "provider-call-1"); + } + #[test] fn interrupted_spokesperson_history_keeps_only_estimated_delivered_prefix() { let mut response = LiveResponse::new(None, Some(3)); From 9d586cb7143e19aae8e228ea6d7120e1bfaffaad Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:07:24 -0400 Subject: [PATCH 14/34] fix(voice): serialize endpoint setting updates --- .../src/commands/openai_voice_endpoints.rs | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index ac7540f7f..be4305343 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -11,6 +11,7 @@ pub(crate) const TTS_DEFAULT: &str = "https://api.openai.com/v1/audio/speech"; const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; +static SETTINGS_UPDATE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[derive(Clone, Copy, Debug, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] @@ -68,7 +69,11 @@ fn settings_path() -> Result { fn read_settings() -> Result { let path = settings_path()?; - match std::fs::read(&path) { + read_settings_from(&path) +} + +fn read_settings_from(path: &std::path::Path) -> Result { + match std::fs::read(path) { Ok(bytes) => serde_json::from_slice(&bytes) .map_err(|error| format!("Could not read OpenAI voice endpoints: {error}")), Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(Default::default()), @@ -77,7 +82,14 @@ fn read_settings() -> Result { } fn persist(settings: &VoiceEndpointSettings) -> Result<(), String> { + let _guard = SETTINGS_UPDATE_LOCK + .lock() + .map_err(|_| "OpenAI voice settings lock is poisoned".to_string())?; let path = settings_path()?; + persist_to(&path, settings) +} + +fn persist_to(path: &std::path::Path, settings: &VoiceEndpointSettings) -> Result<(), String> { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).map_err(|error| { format!("Could not create OpenAI voice settings directory: {error}") @@ -85,10 +97,22 @@ fn persist(settings: &VoiceEndpointSettings) -> Result<(), String> { } let bytes = serde_json::to_vec_pretty(settings) .map_err(|error| format!("Could not encode OpenAI voice endpoints: {error}"))?; - write_bytes_atomically(&path, &bytes) + write_bytes_atomically(path, &bytes) .map_err(|error| format!("Could not save OpenAI voice endpoints: {error}")) } +fn update_settings_file( + path: &std::path::Path, + update: impl FnOnce(&mut VoiceEndpointSettings), +) -> Result<(), String> { + let _guard = SETTINGS_UPDATE_LOCK + .lock() + .map_err(|_| "OpenAI voice settings lock is poisoned".to_string())?; + let mut settings = read_settings_from(path)?; + update(&mut settings); + persist_to(path, &settings) +} + fn validate(kind: VoiceEndpointKind, raw: &str) -> Result, String> { let raw = raw.trim(); if raw.is_empty() { @@ -190,10 +214,10 @@ pub(crate) fn set_openai_voice_endpoint( kind: VoiceEndpointKind, url: String, ) -> Result<(), String> { - let mut settings = read_settings()?; let selected = validate(kind, &url)?; - settings.set(kind, selected.filter(|url| url != kind.default_url())); - persist(&settings)?; + update_settings_file(&settings_path()?, |settings| { + settings.set(kind, selected.filter(|url| url != kind.default_url())); + })?; app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) } @@ -209,6 +233,42 @@ pub(crate) fn restore(settings: &VoiceEndpointSettings) -> Result<(), String> { #[cfg(test)] mod tests { use super::*; + use std::sync::mpsc; + use std::time::Duration; + + #[test] + fn concurrent_endpoint_updates_preserve_both_services() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("endpoints.json"); + let first_path = path.clone(); + let second_path = path.clone(); + let (first_started_tx, first_started_rx) = mpsc::channel(); + let (second_finished_tx, second_finished_rx) = mpsc::channel(); + + let first = std::thread::spawn(move || { + update_settings_file(&first_path, |settings| { + settings.set(VoiceEndpointKind::Stt, Some("wss://stt.example".into())); + first_started_tx.send(()).unwrap(); + // An unlocked update lets the second save complete against stale settings. + let _ = second_finished_rx.recv_timeout(Duration::from_millis(300)); + }) + .unwrap(); + }); + first_started_rx.recv().unwrap(); + let second = std::thread::spawn(move || { + update_settings_file(&second_path, |settings| { + settings.set(VoiceEndpointKind::Tts, Some("https://tts.example".into())); + }) + .unwrap(); + let _ = second_finished_tx.send(()); + }); + first.join().unwrap(); + second.join().unwrap(); + + let saved = read_settings_from(&path).unwrap(); + assert_eq!(saved.stt.as_deref(), Some("wss://stt.example")); + assert_eq!(saved.tts.as_deref(), Some("https://tts.example")); + } #[test] fn legacy_base_url_keeps_the_v1_root_and_custom_path() { From b145509481a6945917d84f9b8283b6dc15dedd02 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:08:12 -0400 Subject: [PATCH 15/34] fix(voice): clarify Spanish endpoint key guidance --- src/shared/i18n/locales/es/settings.json | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/shared/i18n/locales/es/settings.json b/src/shared/i18n/locales/es/settings.json index 743b4928f..96221ddbb 100644 --- a/src/shared/i18n/locales/es/settings.json +++ b/src/shared/i18n/locales/es/settings.json @@ -1047,19 +1047,19 @@ "modelMissingSize": "No instalado · descarga de {{size}}", "modelNotInstalled": "No instalado", "notReadyInput": "Parakeet STT no está instalado. Descárgalo abajo para usar la conversación por voz.", - "notReadyInputAndOpenAiOutput": "Parakeet STT no está instalado y falta la clave API compartida de voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyInputAndOpenAiOutput": "Parakeet STT no está instalado y falta la clave API para la URL de reproducción seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyInputAndPocketOutput": "Parakeet STT y Pocket TTS no están instalados. Descarga ambos abajo para usar la conversación por voz.", "notReadyInputAndSiriOutput": "Parakeet STT no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInput": "El modelo de dictado de Apple no está instalado. Descárgalo abajo para usar la conversación por voz.", - "notReadyMacInputAndOpenAiOutput": "El modelo de dictado de Apple no está instalado y falta la clave API compartida de voz de OpenAI. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyMacInputAndOpenAiOutput": "El modelo de dictado de Apple no está instalado y falta la clave API para la URL de reproducción seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndPocketOutput": "El modelo de dictado de Apple y Pocket TTS no están instalados. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyMacInputAndSiriOutput": "El modelo de dictado de Apple no está instalado y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", "notReadyOpenAi": "La voz de OpenAI no está lista. Añade abajo la clave API necesaria e inténtalo de nuevo.", - "notReadyOpenAiStt": "La transcripción de OpenAI no está lista. Añade abajo la clave API compartida de voz de OpenAI e inténtalo de nuevo.", - "notReadyOpenAiSttAndPocketOutput": "Falta la clave API compartida de voz de OpenAI y Pocket TTS no está instalado. Completa ambos pasos abajo para usar la conversación por voz.", - "notReadyOpenAiSttAndSiriOutput": "Falta la clave API compartida de voz de OpenAI y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", - "notReadyOpenAiSttAndTts": "La transcripción y la reproducción de voz de OpenAI no están listas. Añade abajo su clave API compartida de voz de OpenAI e inténtalo de nuevo.", - "notReadyOpenAiTts": "La reproducción de voz de OpenAI no está lista. Añade abajo la clave API compartida de voz de OpenAI e inténtalo de nuevo.", + "notReadyOpenAiStt": "La transcripción de OpenAI no está lista. Añade abajo una clave API para la URL de transcripción seleccionada e inténtalo de nuevo.", + "notReadyOpenAiSttAndPocketOutput": "Falta la clave API para la URL de transcripción seleccionada y Pocket TTS no está instalado. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAiSttAndSiriOutput": "Falta la clave API para la URL de transcripción seleccionada y no hay ninguna voz de Siri instalada seleccionada. Completa ambos pasos abajo para usar la conversación por voz.", + "notReadyOpenAiSttAndTts": "La transcripción y la reproducción de voz de OpenAI no están listas. Añade abajo las claves API necesarias e inténtalo de nuevo.", + "notReadyOpenAiTts": "La reproducción de voz de OpenAI no está lista. Añade abajo una clave API para la URL de reproducción seleccionada e inténtalo de nuevo.", "notReadyPocketOutput": "Pocket TTS no está instalado. Descárgalo abajo para usar la conversación por voz.", "notReadySiriOutput": "No hay ninguna voz de Siri instalada seleccionada. Descarga o selecciona una abajo para usar la conversación por voz.", "notReadyTitle": "La conversación por voz no está lista", @@ -1074,10 +1074,10 @@ "openAiEnvironmentOverride": "La configuración de desarrollo está reemplazada por el entorno del proceso de Berd.", "openAiSttApiKey": "Clave API de voz a texto de OpenAI", "openAiSttConfigured": "Usa {{model}}.", - "openAiSttNotConfigured": "Añade la clave API compartida de voz de OpenAI para usar la transcripción de OpenAI.", + "openAiSttNotConfigured": "Añade una clave API para la URL de transcripción seleccionada para usarla.", "openAiTtsApiKey": "Clave API de texto a voz de OpenAI", "openAiTtsConfigured": "Usa {{model}} y la voz {{voice}}. Las voces de OpenAI son generadas por IA.", - "openAiTtsNeedsKey": "Añade la clave API compartida de voz de OpenAI para usar esta voz.", + "openAiTtsNeedsKey": "Añade una clave API para la URL de reproducción seleccionada para usar esta voz.", "openAiTtsUnsupportedPlatform": "La reproducción de voz de OpenAI solo es compatible actualmente con macOS.", "outputBackendDescription": "Elige un backend.", "playbackSpeed": "Velocidad de reproducción", From 6b9a0364c8ca4085382a68c31f488bdecddbe912 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:08:52 -0400 Subject: [PATCH 16/34] refactor(voice): name fixed Realtime dictation credential --- src-tauri/src/commands/openai_realtime.rs | 5 ++- .../src/commands/openai_voice_credentials.rs | 43 ++++++++++--------- 2 files changed, 25 insertions(+), 23 deletions(-) diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index 271da0840..9b612059f 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -78,7 +78,8 @@ pub struct OpenAiRealtimeSession { #[tauri::command] pub async fn get_openai_realtime_status() -> Result { - let configured = openai_voice_credentials::is_default_present()?; + let configured = + openai_voice_credentials::is_present(OpenAiVoiceCredential::RealtimeDictation)?; Ok(OpenAiRealtimeStatus { configured }) } @@ -103,7 +104,7 @@ pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { #[tauri::command] pub async fn create_openai_realtime_session() -> Result { - let api_key = openai_voice_credentials::require_default_realtime()?; + let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::RealtimeDictation)?; let response = realtime_transcription_client_secret_request(&reqwest::Client::new(), &api_key) .send() .await diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index d879195e2..6a759f1e6 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -12,14 +12,16 @@ pub(crate) enum OpenAiVoiceCredential { SpeechToText, TextToSpeech, Realtime, + RealtimeDictation, } impl OpenAiVoiceCredential { - const fn kind(self) -> VoiceEndpointKind { + const fn selected_kind(self) -> Option { match self { - Self::SpeechToText => VoiceEndpointKind::Stt, - Self::TextToSpeech => VoiceEndpointKind::Tts, - Self::Realtime => VoiceEndpointKind::Realtime, + Self::SpeechToText => Some(VoiceEndpointKind::Stt), + Self::TextToSpeech => Some(VoiceEndpointKind::Tts), + Self::Realtime => Some(VoiceEndpointKind::Realtime), + Self::RealtimeDictation => None, } } @@ -34,12 +36,17 @@ impl OpenAiVoiceCredential { Self::Realtime => { "OpenAI Realtime voice is not configured. Add an API key for the selected Realtime URL in Voice settings, then try again." } + Self::RealtimeDictation => { + "OpenAI Realtime dictation needs an API key for the default OpenAI endpoint" + } } } } fn account(credential: OpenAiVoiceCredential) -> Result { - let kind = credential.kind(); + let Some(kind) = credential.selected_kind() else { + return Ok(KEYCHAIN_ACCOUNT.to_string()); + }; let url = openai_voice_endpoints::effective_url(kind)?; Ok(account_for_url(kind, &url)) } @@ -87,10 +94,6 @@ pub(crate) fn is_present(credential: OpenAiVoiceCredential) -> Result Result { - is_present_account(KEYCHAIN_ACCOUNT) -} - fn is_present_account(account: &str) -> Result { #[cfg(target_os = "macos")] { @@ -129,29 +132,27 @@ pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result Result { - read_account(KEYCHAIN_ACCOUNT)?.ok_or_else(|| { - "OpenAI Realtime dictation needs an API key for the default OpenAI endpoint".to_string() - }) -} - #[cfg(test)] mod tests { use super::*; #[test] fn speech_services_use_independent_endpoints() { assert!(matches!( - OpenAiVoiceCredential::SpeechToText.kind(), - VoiceEndpointKind::Stt + OpenAiVoiceCredential::SpeechToText.selected_kind(), + Some(VoiceEndpointKind::Stt) )); assert!(matches!( - OpenAiVoiceCredential::TextToSpeech.kind(), - VoiceEndpointKind::Tts + OpenAiVoiceCredential::TextToSpeech.selected_kind(), + Some(VoiceEndpointKind::Tts) )); assert!(matches!( - OpenAiVoiceCredential::Realtime.kind(), - VoiceEndpointKind::Realtime + OpenAiVoiceCredential::Realtime.selected_kind(), + Some(VoiceEndpointKind::Realtime) )); + assert_eq!( + account(OpenAiVoiceCredential::RealtimeDictation).unwrap(), + KEYCHAIN_ACCOUNT + ); } #[test] From a15d57501ed73f092931d91f020bc852ddddb3be Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:09:31 -0400 Subject: [PATCH 17/34] refactor(call): return recorded live event to handoff caller --- src-tauri/crates/berd-call/src/main.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 6f945b567..dd4b1ebba 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -3615,7 +3615,7 @@ fn run_expert_spokesperson_session( call_id, message, } => { - let handoff_id = record_and_emit_live_event( + let handoff_id = match record_and_emit_live_event( &mut core, &mut emitted_live_token, LiveSideEvent::Handoff { @@ -3623,8 +3623,10 @@ fn run_expert_spokesperson_session( message, }, &mut writer, - )? - .ok_or("live handoff did not produce a handoff ID")?; + )? { + LiveSideEvent::Handoff { call_id, .. } => call_id, + _ => return Err("live handoff did not produce a handoff ID".into()), + }; runtime.as_ref().expect("initialized runtime").send( SpokespersonCommand::Provider(accepted_handoff_tool_output( &call_id, @@ -5016,12 +5018,8 @@ fn record_and_emit_live_event( emitted_live_token: &mut u64, event: LiveSideEvent, writer: &mut impl Write, -) -> Result, String> { +) -> Result { let (recorded, expert_delivery) = core.record_live_event_with_delivery(event)?; - let handoff_id = match recorded.payload { - LiveSideEvent::Handoff { call_id, .. } => Some(call_id), - _ => None, - }; emit_live_events(core, emitted_live_token, writer)?; if let Some(delivery) = expert_delivery { write_message( @@ -5034,7 +5032,7 @@ fn record_and_emit_live_event( }, )?; } - Ok(handoff_id) + Ok(recorded.payload) } fn publish_live_response_if_complete( @@ -7593,7 +7591,7 @@ mod tests { let mut core = RealtimeExpertSpokespersonSession::new(0, "external-test"); let mut emitted_token = 0; let mut output = Vec::new(); - let handoff_id = record_and_emit_live_event( + let handoff_id = match record_and_emit_live_event( &mut core, &mut emitted_token, LiveSideEvent::Handoff { @@ -7603,7 +7601,10 @@ mod tests { &mut output, ) .unwrap() - .expect("a live handoff must expose its generated ID"); + { + LiveSideEvent::Handoff { call_id, .. } => call_id, + _ => panic!("a live handoff must expose its generated ID"), + }; assert_eq!(handoff_id, "handoff-external-test-1"); let tool_output = berd_call::openai_realtime_protocol::accepted_handoff_tool_output( "provider-call-1", From cf2f8d40fe480690c8b0e874e5ea14fa47da89d3 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:26:47 -0400 Subject: [PATCH 18/34] fix(call): keep endpoint parser formatting stable across platforms --- src-tauri/crates/berd-call/src/main.rs | 28 +++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index dd4b1ebba..85c536385 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -21,11 +21,14 @@ use berd_call::input::{ VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, VoiceInputRuntime, INPUT_FRAME_SAMPLES, }; -use berd_call::openai_realtime_protocol::{ - accepted_handoff_tool_output, expert_handoff_message, expert_transcript_message, RealtimeExpertMessage, - RealtimeExpertMessageMode, RealtimeExpertSpokespersonSession, RealtimeHandoffReminder, - RealtimeTranscriptSpeaker, -}; +use berd_call::openai_realtime_protocol::accepted_handoff_tool_output; +use berd_call::openai_realtime_protocol::expert_handoff_message; +use berd_call::openai_realtime_protocol::expert_transcript_message; +use berd_call::openai_realtime_protocol::RealtimeExpertMessage; +use berd_call::openai_realtime_protocol::RealtimeExpertMessageMode; +use berd_call::openai_realtime_protocol::RealtimeExpertSpokespersonSession; +use berd_call::openai_realtime_protocol::RealtimeHandoffReminder; +use berd_call::openai_realtime_protocol::RealtimeTranscriptSpeaker; use berd_call::openai_spokesperson::{ OpenAiSpokespersonConfig, OpenAiSpokespersonRuntime, SpokespersonCommand, SpokespersonEvent, SpokespersonResponseStatus, @@ -5334,15 +5337,16 @@ fn parse_endpoint_url(value: &str, flag: &str, websocket: bool) -> Result() + .is_ok_and(|address| address.is_loopback()) + }); if !valid_scheme || url.host_str().is_none() - || (matches!(url.scheme(), "http" | "ws") && !url.host_str().is_some_and(|host| { - host.eq_ignore_ascii_case("localhost") - || host - .trim_matches(['[', ']']) - .parse::() - .is_ok_and(|address| address.is_loopback()) - })) + || (matches!(url.scheme(), "http" | "ws") && !loopback_host) || !url.username().is_empty() || url.password().is_some() || url.fragment().is_some() From 0d6003b8db157caf5636ca4a4efd621b449abb0b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:41:00 -0400 Subject: [PATCH 19/34] refactor(voice): share endpoint URL security policy --- .../crates/berd-call/src/endpoint_url.rs | 60 +++++++++++++++++++ src-tauri/crates/berd-call/src/lib.rs | 1 + src-tauri/crates/berd-call/src/main.rs | 22 ++----- .../src/commands/openai_voice_endpoints.rs | 26 ++------ 4 files changed, 72 insertions(+), 37 deletions(-) create mode 100644 src-tauri/crates/berd-call/src/endpoint_url.rs diff --git a/src-tauri/crates/berd-call/src/endpoint_url.rs b/src-tauri/crates/berd-call/src/endpoint_url.rs new file mode 100644 index 000000000..dcc3c6acd --- /dev/null +++ b/src-tauri/crates/berd-call/src/endpoint_url.rs @@ -0,0 +1,60 @@ +//! URL policy shared by saved voice endpoints and `berd-call` overrides. + +use reqwest::Url; + +#[derive(Clone, Copy)] +pub enum EndpointProtocol { + WebSocket, + Http, +} + +pub fn is_allowed_endpoint_url(url: &Url, protocol: EndpointProtocol) -> bool { + let scheme_allowed = match protocol { + EndpointProtocol::WebSocket => matches!(url.scheme(), "ws" | "wss"), + EndpointProtocol::Http => matches!(url.scheme(), "http" | "https"), + }; + let loopback = url.host_str().is_some_and(|host| { + host.eq_ignore_ascii_case("localhost") + || host + .trim_matches(['[', ']']) + .parse::() + .is_ok_and(|address| address.is_loopback()) + }); + scheme_allowed + && url.host_str().is_some() + && (!matches!(url.scheme(), "http" | "ws") || loopback) + && url.username().is_empty() + && url.password().is_none() + && url.fragment().is_none() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn enforces_protocol_and_loopback_policy() { + assert!(is_allowed_endpoint_url( + &Url::parse("ws://[::1]:18870/realtime").unwrap(), + EndpointProtocol::WebSocket, + )); + assert!(is_allowed_endpoint_url( + &Url::parse("http://localhost:18870/speech").unwrap(), + EndpointProtocol::Http, + )); + for raw in [ + "http://example.test/speech", + "http://user@example.test/speech", + "https://example.test/speech#fragment", + ] { + assert!(!is_allowed_endpoint_url( + &Url::parse(raw).unwrap(), + EndpointProtocol::Http, + )); + } + assert!(!is_allowed_endpoint_url( + &Url::parse("https://example.test/speech").unwrap(), + EndpointProtocol::WebSocket, + )); + } +} diff --git a/src-tauri/crates/berd-call/src/lib.rs b/src-tauri/crates/berd-call/src/lib.rs index 831260bec..76574f097 100644 --- a/src-tauri/crates/berd-call/src/lib.rs +++ b/src-tauri/crates/berd-call/src/lib.rs @@ -7,6 +7,7 @@ mod asset_verification; mod audio_output; pub mod benchmark; pub mod causal_inbox; +pub mod endpoint_url; mod configured_tts; pub mod expert_spokesperson; pub mod input; diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 85c536385..3e2c1dfec 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -16,6 +16,7 @@ use berd_call::benchmark::{ SttBenchmarkTarget, TtsBenchmarkMode, TtsBenchmarkPromptManifest, TtsBenchmarkTarget, }; use berd_call::expert_spokesperson::{ExpertDirectiveOutcome, LiveSideEvent}; +use berd_call::endpoint_url::{is_allowed_endpoint_url, EndpointProtocol}; use berd_call::input::{ AssistantActivityGuard, InputDuringTtsSlot, InputDuringTtsSnapshot, VoiceInputConfig, VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, @@ -5332,25 +5333,12 @@ fn parse_args(args: &[String]) -> Result { fn parse_endpoint_url(value: &str, flag: &str, websocket: bool) -> Result { let url = reqwest::Url::parse(value).map_err(|error| format!("{flag} is invalid: {error}"))?; - let valid_scheme = if websocket { - matches!(url.scheme(), "ws" | "wss") + let protocol = if websocket { + EndpointProtocol::WebSocket } else { - matches!(url.scheme(), "http" | "https") + EndpointProtocol::Http }; - let loopback_host = url.host_str().is_some_and(|host| { - host.eq_ignore_ascii_case("localhost") - || host - .trim_matches(['[', ']']) - .parse::() - .is_ok_and(|address| address.is_loopback()) - }); - if !valid_scheme - || url.host_str().is_none() - || (matches!(url.scheme(), "http" | "ws") && !loopback_host) - || !url.username().is_empty() - || url.password().is_some() - || url.fragment().is_some() - { + if !is_allowed_endpoint_url(&url, protocol) { return Err(format!("{flag} requires a full URL with the correct protocol, HTTPS/WSS outside loopback, and no embedded credentials or fragment")); } Ok(url.to_string()) diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index be4305343..5c2602335 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -1,5 +1,6 @@ //! Independent, user-selected OpenAI-compatible voice endpoints. +use berd_call::endpoint_url::{is_allowed_endpoint_url, EndpointProtocol}; use serde::{Deserialize, Serialize}; use tauri::{AppHandle, Emitter}; @@ -8,7 +9,7 @@ use crate::services::atomic_file::write_bytes_atomically; pub(crate) const REALTIME_DEFAULT: &str = "wss://api.openai.com/v1/realtime"; pub(crate) const STT_DEFAULT: &str = "wss://api.openai.com/v1/realtime?intent=transcription"; pub(crate) const TTS_DEFAULT: &str = "https://api.openai.com/v1/audio/speech"; -const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; +pub(crate) const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; static SETTINGS_UPDATE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); @@ -119,32 +120,17 @@ fn validate(kind: VoiceEndpointKind, raw: &str) -> Result, String return Ok(None); } let mut url = url::Url::parse(raw).map_err(|error| format!("Invalid endpoint URL: {error}"))?; - let allowed = match kind { - VoiceEndpointKind::Realtime | VoiceEndpointKind::Stt => ["ws", "wss"].as_slice(), - VoiceEndpointKind::Tts => ["http", "https"].as_slice(), + let protocol = match kind { + VoiceEndpointKind::Realtime | VoiceEndpointKind::Stt => EndpointProtocol::WebSocket, + VoiceEndpointKind::Tts => EndpointProtocol::Http, }; - if !allowed.contains(&url.scheme()) - || url.host_str().is_none() - || (matches!(url.scheme(), "http" | "ws") && !is_loopback(&url)) - || !url.username().is_empty() - || url.password().is_some() - || url.fragment().is_some() - { + if !is_allowed_endpoint_url(&url, protocol) { return Err("Endpoint must be a full URL with the correct protocol, HTTPS/WSS outside loopback, and no embedded credentials or fragment".into()); } url.set_fragment(None); Ok(Some(url.to_string())) } -fn is_loopback(url: &url::Url) -> bool { - match url.host() { - Some(url::Host::Domain(host)) => host.eq_ignore_ascii_case("localhost"), - Some(url::Host::Ipv4(address)) => address.is_loopback(), - Some(url::Host::Ipv6(address)) => address.is_loopback(), - None => false, - } -} - fn normalize_base_url(raw_url: String) -> Result { let mut url = url::Url::parse(&raw_url) .map_err(|error| format!("OpenAI voice endpoint is invalid: {error}"))?; From 8092e6e87ed340e1ae73f36c822c1d885c836870 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:41:33 -0400 Subject: [PATCH 20/34] refactor(voice): reuse settings-changed event name --- src-tauri/src/commands/openai_realtime.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index 9b612059f..ca4aa541f 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -91,14 +91,14 @@ pub fn set_openai_realtime_api_key(app: AppHandle, api_key: String) -> Result<() return Err("Realtime API key cannot be empty".into()); } openai_voice_credentials::store(OpenAiVoiceCredential::Realtime, api_key)?; - app.emit("openai-voice:settings-changed", ()) + app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } #[tauri::command] pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { openai_voice_credentials::clear(OpenAiVoiceCredential::Realtime)?; - app.emit("openai-voice:settings-changed", ()) + app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } From fbc98bda03fc950c74eaaed3ee38fd767f03ec30 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 13:42:06 -0400 Subject: [PATCH 21/34] refactor(voice): simplify key status and stabilize endpoint test --- .../ui/OpenAiEndpointField.tsx | 7 ++----- tests/e2e/voice-endpoints.spec.ts | 17 ++++++++++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index 4984e32b4..e961a41a3 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -35,7 +35,6 @@ export function OpenAiEndpointField({ const keyId = useId(); const [url, setUrl] = useState(""); const [savedUrl, setSavedUrl] = useState(""); - const [statusUrl, setStatusUrl] = useState(""); const [localKeyStatus, setLocalKeyStatus] = useState<{ url: string; configured: boolean; @@ -54,7 +53,6 @@ export function OpenAiEndpointField({ const value = settings[kind] ?? ""; setUrl(value); setSavedUrl(value); - setStatusUrl(value); setLoaded(true); } }, @@ -73,7 +71,7 @@ export function OpenAiEndpointField({ (localKeyStatus?.url === savedUrl && localKeyStatus.observedConfigured === configured ? localKeyStatus.configured - : statusUrl === savedUrl && configured); + : configured); const save = async () => { setSaving(true); @@ -89,7 +87,6 @@ export function OpenAiEndpointField({ targetUrl = (await getOpenAiVoiceEndpoints())[kind] ?? ""; setUrl(targetUrl); setSavedUrl(targetUrl); - setStatusUrl(targetUrl); setLocalKeyStatus({ url: targetUrl, configured: false, @@ -149,7 +146,7 @@ export function OpenAiEndpointField({ }; return ( -
+
diff --git a/tests/e2e/voice-endpoints.spec.ts b/tests/e2e/voice-endpoints.spec.ts index 101c3c101..d17d5c881 100644 --- a/tests/e2e/voice-endpoints.spec.ts +++ b/tests/e2e/voice-endpoints.spec.ts @@ -24,10 +24,18 @@ const test = base.extend<{ settings: Page }>({ test.use({ screenshot: "off", trace: "off", video: "off" }); -async function saveEndpoint(page: Page, label: string, url: string) { +async function saveEndpoint( + page: Page, + kind: "realtime" | "stt" | "tts", + label: string, + url: string, +) { const input = page.getByLabel(label); await input.fill(url); - await input.locator("..").getByRole("button", { name: "Save" }).click(); + await page + .getByTestId(`openai-${kind}-endpoint-settings`) + .getByRole("button", { name: "Save" }) + .click(); await expect(input).toHaveValue(url); } @@ -50,11 +58,13 @@ test("keeps Realtime, STT, and TTS URLs independent across settings reloads", as ); await saveEndpoint( page, + "stt", "Speech-to-text endpoint URL", "ws://127.0.0.1:18870/v1/realtime?intent=transcription", ); await saveEndpoint( page, + "tts", "Text-to-speech endpoint URL", "http://127.0.0.1:18870/v1/audio/speech", ); @@ -69,6 +79,7 @@ test("keeps Realtime, STT, and TTS URLs independent across settings reloads", as ); await saveEndpoint( page, + "realtime", "Realtime endpoint URL", "ws://127.0.0.1:18870/v1/realtime", ); @@ -87,7 +98,7 @@ test("keeps Realtime, STT, and TTS URLs independent across settings reloads", as "http://127.0.0.1:18870/v1/audio/speech", ); - await saveEndpoint(page, "Speech-to-text endpoint URL", ""); + await saveEndpoint(page, "stt", "Speech-to-text endpoint URL", ""); await page.goto("/"); await page.locator("[data-sidebar-nav-id=settings]").click(); await page.locator("[data-sidebar-nav-id=settings-voice]").click(); From 10f931b3cbd1b5d5ddcb0aeb1d2de90d1b57dcae Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 14:07:28 -0400 Subject: [PATCH 22/34] fix(call): order shared endpoint imports for Windows fmt --- src-tauri/crates/berd-call/src/lib.rs | 2 +- src-tauri/crates/berd-call/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src-tauri/crates/berd-call/src/lib.rs b/src-tauri/crates/berd-call/src/lib.rs index 76574f097..d8c593610 100644 --- a/src-tauri/crates/berd-call/src/lib.rs +++ b/src-tauri/crates/berd-call/src/lib.rs @@ -7,8 +7,8 @@ mod asset_verification; mod audio_output; pub mod benchmark; pub mod causal_inbox; -pub mod endpoint_url; mod configured_tts; +pub mod endpoint_url; pub mod expert_spokesperson; pub mod input; pub mod local_assets; diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 3e2c1dfec..8d3078a78 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -15,8 +15,8 @@ use berd_call::benchmark::{ load_bundled_tts_prompt_manifest, SttBenchmarkEnvironment, SttBenchmarkMode, SttBenchmarkTarget, TtsBenchmarkMode, TtsBenchmarkPromptManifest, TtsBenchmarkTarget, }; -use berd_call::expert_spokesperson::{ExpertDirectiveOutcome, LiveSideEvent}; use berd_call::endpoint_url::{is_allowed_endpoint_url, EndpointProtocol}; +use berd_call::expert_spokesperson::{ExpertDirectiveOutcome, LiveSideEvent}; use berd_call::input::{ AssistantActivityGuard, InputDuringTtsSlot, InputDuringTtsSnapshot, VoiceInputConfig, VoiceInputControls, VoiceInputEngineConfig, VoiceInputEvent, VoiceInputFrame, From 064727913f4543419b5ac432e5beeda664362477 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 14:08:02 -0400 Subject: [PATCH 23/34] refactor(voice): distinguish selected and default Realtime keys --- src-tauri/src/commands/openai_audio.rs | 6 ++---- src-tauri/src/commands/openai_realtime.rs | 12 +++++++----- .../src/commands/openai_voice_credentials.rs | 18 +++++++++--------- .../src/commands/openai_voice_endpoints.rs | 4 ++-- 4 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index 8bf9a7038..d7a928b7b 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -28,7 +28,7 @@ use super::pocket_voice::{ use super::{ native_voice::{InterruptionSensitivity, NativeVoiceState}, openai_voice_credentials::{self, OpenAiVoiceCredential}, - openai_voice_endpoints::{self, VoiceEndpointKind}, + openai_voice_endpoints::{self, VoiceEndpointKind, BASE_URL_ENV, SETTINGS_CHANGED_EVENT}, pocket_voice::VoiceInterruptionMode, voice_capture::VoiceCaptureState, }; @@ -45,11 +45,9 @@ const TTS_VOICES: &[&str] = &[ "alloy", "ash", "ballad", "cedar", "coral", "echo", "fable", "marin", "nova", "onyx", "sage", "shimmer", "verse", ]; -const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; const STT_MODEL_ENV: &str = "BERD_OPENAI_STT_MODEL"; const TTS_MODEL_ENV: &str = "BERD_OPENAI_TTS_MODEL"; const TTS_VOICE_ENV: &str = "BERD_OPENAI_TTS_VOICE"; -const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; #[cfg(target_os = "macos")] const TTS_SAMPLE_RATE: u32 = 24_000; // Avoid starting the audio device from a tiny first network chunk that can drain @@ -328,7 +326,7 @@ pub async fn get_openai_voice_status( ( openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText), openai_voice_credentials::is_present(OpenAiVoiceCredential::TextToSpeech), - openai_voice_credentials::is_present(OpenAiVoiceCredential::Realtime), + openai_voice_credentials::is_present(OpenAiVoiceCredential::SelectedRealtimeAssistant), ) }) .await diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index ca4aa541f..cef522d4e 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -79,7 +79,7 @@ pub struct OpenAiRealtimeSession { #[tauri::command] pub async fn get_openai_realtime_status() -> Result { let configured = - openai_voice_credentials::is_present(OpenAiVoiceCredential::RealtimeDictation)?; + openai_voice_credentials::is_present(OpenAiVoiceCredential::DefaultRealtimeDictation)?; Ok(OpenAiRealtimeStatus { configured }) } @@ -90,21 +90,22 @@ pub fn set_openai_realtime_api_key(app: AppHandle, api_key: String) -> Result<() if api_key.is_empty() { return Err("Realtime API key cannot be empty".into()); } - openai_voice_credentials::store(OpenAiVoiceCredential::Realtime, api_key)?; + openai_voice_credentials::store(OpenAiVoiceCredential::SelectedRealtimeAssistant, api_key)?; app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } #[tauri::command] pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { - openai_voice_credentials::clear(OpenAiVoiceCredential::Realtime)?; + openai_voice_credentials::clear(OpenAiVoiceCredential::SelectedRealtimeAssistant)?; app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } #[tauri::command] pub async fn create_openai_realtime_session() -> Result { - let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::RealtimeDictation)?; + let api_key = + openai_voice_credentials::require(OpenAiVoiceCredential::DefaultRealtimeDictation)?; let response = realtime_transcription_client_secret_request(&reqwest::Client::new(), &api_key) .send() .await @@ -140,7 +141,8 @@ pub fn start_openai_realtime_spokesperson_runtime( return Err("This window already owns an OpenAI Realtime runtime session".into()); } - let api_key = openai_voice_credentials::require(OpenAiVoiceCredential::Realtime)?; + let api_key = + openai_voice_credentials::require(OpenAiVoiceCredential::SelectedRealtimeAssistant)?; let mut config = OpenAiSpokespersonConfig::new(api_key, options, Vec::new()); config.endpoint = openai_voice_endpoints::effective_url(VoiceEndpointKind::Realtime)?; let semantic_revision = Arc::new(AtomicU64::new(0)); diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 6a759f1e6..a7ea8a1cb 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -11,8 +11,8 @@ const KEYCHAIN_ACCOUNT: &str = "api-key"; pub(crate) enum OpenAiVoiceCredential { SpeechToText, TextToSpeech, - Realtime, - RealtimeDictation, + SelectedRealtimeAssistant, + DefaultRealtimeDictation, } impl OpenAiVoiceCredential { @@ -20,8 +20,8 @@ impl OpenAiVoiceCredential { match self { Self::SpeechToText => Some(VoiceEndpointKind::Stt), Self::TextToSpeech => Some(VoiceEndpointKind::Tts), - Self::Realtime => Some(VoiceEndpointKind::Realtime), - Self::RealtimeDictation => None, + Self::SelectedRealtimeAssistant => Some(VoiceEndpointKind::Realtime), + Self::DefaultRealtimeDictation => None, } } @@ -33,10 +33,10 @@ impl OpenAiVoiceCredential { Self::TextToSpeech => { "OpenAI text-to-speech is not configured. Add an API key for the selected playback URL in Voice settings, then try again." } - Self::Realtime => { + Self::SelectedRealtimeAssistant => { "OpenAI Realtime voice is not configured. Add an API key for the selected Realtime URL in Voice settings, then try again." } - Self::RealtimeDictation => { + Self::DefaultRealtimeDictation => { "OpenAI Realtime dictation needs an API key for the default OpenAI endpoint" } } @@ -146,11 +146,11 @@ mod tests { Some(VoiceEndpointKind::Tts) )); assert!(matches!( - OpenAiVoiceCredential::Realtime.selected_kind(), + OpenAiVoiceCredential::SelectedRealtimeAssistant.selected_kind(), Some(VoiceEndpointKind::Realtime) )); assert_eq!( - account(OpenAiVoiceCredential::RealtimeDictation).unwrap(), + account(OpenAiVoiceCredential::DefaultRealtimeDictation).unwrap(), KEYCHAIN_ACCOUNT ); } @@ -189,7 +189,7 @@ mod tests { for credential in [ OpenAiVoiceCredential::SpeechToText, OpenAiVoiceCredential::TextToSpeech, - OpenAiVoiceCredential::Realtime, + OpenAiVoiceCredential::SelectedRealtimeAssistant, ] { let message = credential.missing_message(); assert!(message.contains("selected")); diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index 5c2602335..76d4cda88 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -11,7 +11,7 @@ pub(crate) const STT_DEFAULT: &str = "wss://api.openai.com/v1/realtime?intent=tr pub(crate) const TTS_DEFAULT: &str = "https://api.openai.com/v1/audio/speech"; pub(crate) const SETTINGS_CHANGED_EVENT: &str = "openai-voice:settings-changed"; const DEFAULT_BASE_URL: &str = "https://api.openai.com/v1"; -const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; +pub(crate) const BASE_URL_ENV: &str = "BERD_OPENAI_VOICE_BASE_URL"; static SETTINGS_UPDATE_LOCK: std::sync::Mutex<()> = std::sync::Mutex::new(()); #[derive(Clone, Copy, Debug, Deserialize, Serialize)] @@ -257,7 +257,7 @@ mod tests { } #[test] - fn legacy_base_url_keeps_the_v1_root_and_custom_path() { + fn base_url_environment_override_keeps_the_v1_root_and_custom_path() { assert_eq!( normalize_base_url("https://proxy.example".into()).unwrap(), "https://proxy.example/v1" From 3ee461161bc8427fb43107506dabc55265125e41 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 14:08:39 -0400 Subject: [PATCH 24/34] refactor(voice): name key-status freshness and update test fixture --- .../ui/OpenAiEndpointField.tsx | 21 ++++++++++--------- .../ui/VoiceSettings.test.tsx | 6 +++++- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index e961a41a3..8b97a0777 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -38,7 +38,7 @@ export function OpenAiEndpointField({ const [localKeyStatus, setLocalKeyStatus] = useState<{ url: string; configured: boolean; - observedConfigured: boolean; + parentConfiguredAtObservation: boolean; } | null>(null); const [apiKey, setApiKey] = useState(""); const [loaded, setLoaded] = useState(false); @@ -66,12 +66,13 @@ export function OpenAiEndpointField({ }, [kind]); const changed = url.trim() !== savedUrl; - const keyConfigured = - !changed && - (localKeyStatus?.url === savedUrl && - localKeyStatus.observedConfigured === configured + // A parent status change supersedes a local save or clear observation. + const observedKeyConfigured = + localKeyStatus?.url === savedUrl && + localKeyStatus.parentConfiguredAtObservation === configured ? localKeyStatus.configured - : configured); + : configured; + const keyConfigured = !changed && observedKeyConfigured; const save = async () => { setSaving(true); @@ -90,7 +91,7 @@ export function OpenAiEndpointField({ setLocalKeyStatus({ url: targetUrl, configured: false, - observedConfigured: configured, + parentConfiguredAtObservation: configured, }); } if (savingKey) { @@ -99,7 +100,7 @@ export function OpenAiEndpointField({ setLocalKeyStatus({ url: targetUrl, configured: true, - observedConfigured: configured, + parentConfiguredAtObservation: configured, }); } else if (changed) { // Metadata-only lookup; never request the Keychain secret to render settings. @@ -107,7 +108,7 @@ export function OpenAiEndpointField({ setLocalKeyStatus({ url: targetUrl, configured: status[`${kind}Configured`], - observedConfigured: configured, + parentConfiguredAtObservation: configured, }); } } catch (cause) { @@ -136,7 +137,7 @@ export function OpenAiEndpointField({ setLocalKeyStatus({ url: savedUrl, configured: false, - observedConfigured: configured, + parentConfiguredAtObservation: configured, }); } catch (cause) { setError(cause instanceof Error ? cause.message : String(cause)); diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 149e119e2..087f2727b 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -99,7 +99,11 @@ const openAiStatusState = vi.hoisted(() => ({ })); const openAiApiMocks = vi.hoisted(() => ({ getStatus: vi.fn(() => - Promise.resolve({ sttKeySaved: false, ttsKeySaved: false }), + Promise.resolve({ + sttConfigured: false, + ttsConfigured: false, + realtimeConfigured: false, + }), ), getEndpoints: vi.fn(() => Promise.resolve({ From fb0c7684d37ce39b7b6b188dc978beb984e3ab8f Mon Sep 17 00:00:00 2001 From: John Tennant Date: Tue, 29 Sep 2026 14:24:55 -0400 Subject: [PATCH 25/34] test(voice): describe endpoint save concurrency invariant --- src-tauri/src/commands/openai_voice_endpoints.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index 76d4cda88..2eaf54f58 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -235,7 +235,7 @@ mod tests { update_settings_file(&first_path, |settings| { settings.set(VoiceEndpointKind::Stt, Some("wss://stt.example".into())); first_started_tx.send(()).unwrap(); - // An unlocked update lets the second save complete against stale settings. + // Hold the first update while a concurrent save attempts to run; both must survive. let _ = second_finished_rx.recv_timeout(Duration::from_millis(300)); }) .unwrap(); From f675b876ce3342e47355fae40aa9cca29c3e4d3d Mon Sep 17 00:00:00 2001 From: John Tennant Date: Wed, 30 Sep 2026 07:02:46 -0400 Subject: [PATCH 26/34] chore: retrigger automated review From 3ebd16c5b676d15e5a3e1f81534c9864a8f0e50b Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 2 Oct 2026 14:42:29 -0400 Subject: [PATCH 27/34] fix(voice): reject key mutations for stale endpoint selections --- src-tauri/src/commands/openai_audio.rs | 28 +++++++-- src-tauri/src/commands/openai_realtime.rs | 38 ++++++++++-- .../src/commands/openai_voice_endpoints.rs | 61 +++++++++++++++++++ .../voice-conversation/api/openAiVoice.ts | 33 ++++++---- .../ui/OpenAiEndpointField.test.tsx | 46 +++++++++++++- .../ui/OpenAiEndpointField.tsx | 12 ++-- .../ui/RealtimeVoiceSettings.test.tsx | 5 +- .../ui/VoiceSettings.test.tsx | 6 +- 8 files changed, 196 insertions(+), 33 deletions(-) diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index d7a928b7b..a7c09df18 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -372,6 +372,7 @@ pub async fn set_openai_stt_api_key( state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, capture: State<'_, VoiceCaptureState>, + expected_url: String, api_key: String, ) -> Result<(), String> { let api_key = api_key.trim(); @@ -381,7 +382,11 @@ pub async fn set_openai_stt_api_key( native_voice .stop_active_then(&app, &capture, || { stop_openai_voice_inner(&state)?; - openai_voice_credentials::store(OpenAiVoiceCredential::SpeechToText, api_key)?; + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Stt, + &expected_url, + || openai_voice_credentials::store(OpenAiVoiceCredential::SpeechToText, api_key), + )?; state.credential_revision.fetch_add(1, Ordering::AcqRel); state.configured.store(true, Ordering::Release); app.emit(SETTINGS_CHANGED_EVENT, ()) @@ -396,11 +401,16 @@ pub async fn clear_openai_stt_api_key( state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, capture: State<'_, VoiceCaptureState>, + expected_url: String, ) -> Result<(), String> { native_voice .stop_active_then(&app, &capture, || { stop_openai_voice_inner(&state)?; - openai_voice_credentials::clear(OpenAiVoiceCredential::SpeechToText)?; + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Stt, + &expected_url, + || openai_voice_credentials::clear(OpenAiVoiceCredential::SpeechToText), + )?; state.credential_revision.fetch_add(1, Ordering::AcqRel); state.configured.store(false, Ordering::Release); app.emit(SETTINGS_CHANGED_EVENT, ()) @@ -415,6 +425,7 @@ pub async fn set_openai_tts_api_key( state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, capture: State<'_, VoiceCaptureState>, + expected_url: String, api_key: String, ) -> Result<(), String> { let api_key = api_key.trim(); @@ -424,7 +435,11 @@ pub async fn set_openai_tts_api_key( native_voice .stop_active_then(&app, &capture, || { stop_openai_voice_inner(&state)?; - openai_voice_credentials::store(OpenAiVoiceCredential::TextToSpeech, api_key)?; + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Tts, + &expected_url, + || openai_voice_credentials::store(OpenAiVoiceCredential::TextToSpeech, api_key), + )?; state.credential_revision.fetch_add(1, Ordering::AcqRel); state.configured.store( openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText)?, @@ -442,11 +457,16 @@ pub async fn clear_openai_tts_api_key( state: State<'_, OpenAiVoiceState>, native_voice: State<'_, NativeVoiceState>, capture: State<'_, VoiceCaptureState>, + expected_url: String, ) -> Result<(), String> { native_voice .stop_active_then(&app, &capture, || { stop_openai_voice_inner(&state)?; - openai_voice_credentials::clear(OpenAiVoiceCredential::TextToSpeech)?; + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Tts, + &expected_url, + || openai_voice_credentials::clear(OpenAiVoiceCredential::TextToSpeech), + )?; state.credential_revision.fetch_add(1, Ordering::AcqRel); state.configured.store( openai_voice_credentials::is_present(OpenAiVoiceCredential::SpeechToText)?, diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index cef522d4e..515507b26 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -85,19 +85,47 @@ pub async fn get_openai_realtime_status() -> Result Result<(), String> { - let api_key = api_key.trim(); +pub async fn set_openai_realtime_api_key( + app: AppHandle, + api_key: String, + expected_url: String, +) -> Result<(), String> { + let api_key = api_key.trim().to_string(); if api_key.is_empty() { return Err("Realtime API key cannot be empty".into()); } - openai_voice_credentials::store(OpenAiVoiceCredential::SelectedRealtimeAssistant, api_key)?; + tauri::async_runtime::spawn_blocking(move || { + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Realtime, + &expected_url, + || { + openai_voice_credentials::store( + OpenAiVoiceCredential::SelectedRealtimeAssistant, + &api_key, + ) + }, + ) + }) + .await + .map_err(|error| format!("Could not save Realtime key: {error}"))??; app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } #[tauri::command] -pub fn clear_openai_realtime_api_key(app: AppHandle) -> Result<(), String> { - openai_voice_credentials::clear(OpenAiVoiceCredential::SelectedRealtimeAssistant)?; +pub async fn clear_openai_realtime_api_key( + app: AppHandle, + expected_url: String, +) -> Result<(), String> { + tauri::async_runtime::spawn_blocking(move || { + openai_voice_endpoints::with_selected_endpoint( + VoiceEndpointKind::Realtime, + &expected_url, + || openai_voice_credentials::clear(OpenAiVoiceCredential::SelectedRealtimeAssistant), + ) + }) + .await + .map_err(|error| format!("Could not clear Realtime key: {error}"))??; app.emit(openai_voice_endpoints::SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh Realtime settings: {error}")) } diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index 2eaf54f58..68dd1c7f3 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -208,6 +208,32 @@ pub(crate) fn set_openai_voice_endpoint( .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) } +/// Keep endpoint selection stable while a credential mutation uses it. +pub(crate) fn with_selected_endpoint( + kind: VoiceEndpointKind, + expected_url: &str, + mutate: impl FnOnce() -> Result, +) -> Result { + with_selected_endpoint_at(&settings_path()?, kind, expected_url, mutate) +} + +fn with_selected_endpoint_at( + path: &std::path::Path, + kind: VoiceEndpointKind, + expected_url: &str, + mutate: impl FnOnce() -> Result, +) -> Result { + let expected = validate(kind, expected_url)?.filter(|url| url != kind.default_url()); + let _guard = SETTINGS_UPDATE_LOCK + .lock() + .map_err(|_| "OpenAI voice settings lock is poisoned".to_string())?; + let settings = read_settings_from(path)?; + if settings.get(kind) != expected.as_deref() { + return Err("The endpoint changed in another window. Reopen Voice settings before changing its key.".into()); + } + mutate() +} + pub(crate) fn reset() -> Result<(), String> { persist(&VoiceEndpointSettings::default()) } @@ -222,6 +248,41 @@ mod tests { use std::sync::mpsc; use std::time::Duration; + #[test] + fn credential_mutations_reject_a_stale_displayed_endpoint() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("endpoints.json"); + let mut settings = VoiceEndpointSettings::default(); + settings.set( + VoiceEndpointKind::Stt, + Some("wss://second.test/realtime".into()), + ); + persist_to(&path, &settings).unwrap(); + let mut mutated = false; + let result = with_selected_endpoint_at( + &path, + VoiceEndpointKind::Stt, + "wss://first.test/realtime", + || { + mutated = true; + Ok(()) + }, + ); + assert!(result.unwrap_err().contains("another window")); + assert!(!mutated); + with_selected_endpoint_at( + &path, + VoiceEndpointKind::Stt, + " wss://second.test/realtime ", + || { + mutated = true; + Ok(()) + }, + ) + .unwrap(); + assert!(mutated); + } + #[test] fn concurrent_endpoint_updates_preserve_both_services() { let directory = tempfile::tempdir().unwrap(); diff --git a/src/features/voice-conversation/api/openAiVoice.ts b/src/features/voice-conversation/api/openAiVoice.ts index e626a6ece..ee5534154 100644 --- a/src/features/voice-conversation/api/openAiVoice.ts +++ b/src/features/voice-conversation/api/openAiVoice.ts @@ -35,8 +35,11 @@ export const getOpenAiVoiceStatus = shareInFlight( (): Promise => invoke("get_openai_voice_status"), ); -export function setOpenAiTtsApiKey(apiKey: string): Promise { - return invoke("set_openai_tts_api_key", { apiKey }); +export function setOpenAiTtsApiKey( + apiKey: string, + expectedUrl = "", +): Promise { + return invoke("set_openai_tts_api_key", { apiKey, expectedUrl }); } export type OpenAiVoiceEndpointKind = "realtime" | "stt" | "tts"; @@ -58,24 +61,30 @@ export function setOpenAiVoiceEndpoint( return invoke("set_openai_voice_endpoint", { kind, url }); } -export function setOpenAiRealtimeApiKey(apiKey: string): Promise { - return invoke("set_openai_realtime_api_key", { apiKey }); +export function setOpenAiRealtimeApiKey( + apiKey: string, + expectedUrl = "", +): Promise { + return invoke("set_openai_realtime_api_key", { apiKey, expectedUrl }); } -export function clearOpenAiRealtimeApiKey(): Promise { - return invoke("clear_openai_realtime_api_key"); +export function clearOpenAiRealtimeApiKey(expectedUrl = ""): Promise { + return invoke("clear_openai_realtime_api_key", { expectedUrl }); } -export function setOpenAiSttApiKey(apiKey: string): Promise { - return invoke("set_openai_stt_api_key", { apiKey }); +export function setOpenAiSttApiKey( + apiKey: string, + expectedUrl = "", +): Promise { + return invoke("set_openai_stt_api_key", { apiKey, expectedUrl }); } -export function clearOpenAiSttApiKey(): Promise { - return invoke("clear_openai_stt_api_key"); +export function clearOpenAiSttApiKey(expectedUrl = ""): Promise { + return invoke("clear_openai_stt_api_key", { expectedUrl }); } -export function clearOpenAiTtsApiKey(): Promise { - return invoke("clear_openai_tts_api_key"); +export function clearOpenAiTtsApiKey(expectedUrl = ""): Promise { + return invoke("clear_openai_tts_api_key", { expectedUrl }); } export function listenToOpenAiVoiceSettings( diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx index 10094178c..1f496297c 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.test.tsx @@ -15,7 +15,14 @@ vi.mock("../api/openAiVoice", () => ({ setOpenAiVoiceEndpoint: vi.fn(), })); -beforeEach(() => vi.clearAllMocks()); +beforeEach(() => { + vi.clearAllMocks(); + voiceApi.getEndpoints.mockResolvedValue({ + realtime: null, + stt: null, + tts: null, + }); +}); it("updates a locally saved key indicator when another default service clears the shared key", async () => { const user = userEvent.setup(); @@ -33,8 +40,43 @@ it("updates a locally saved key indicator when another default service clears th await screen.findByText("Key saved for this URL in macOS Keychain."); await user.type(screen.getByLabelText("STT key"), "new-key"); await user.click(screen.getByRole("button", { name: "Save" })); - expect(onSaveKey).toHaveBeenCalledWith("new-key"); + expect(onSaveKey).toHaveBeenCalledWith("new-key", ""); view.rerender(); expect(await screen.findByText(/No key set/)).toBeInTheDocument(); }); + +it("binds key save and removal to the displayed endpoint", async () => { + voiceApi.getEndpoints.mockResolvedValue({ + realtime: null, + stt: "wss://first.test/realtime", + tts: null, + } as never); + const user = userEvent.setup(); + const onSaveKey = vi.fn(async () => {}); + const onClearKey = vi.fn(async () => {}); + renderWithProviders( + , + ); + await screen.findByDisplayValue("wss://first.test/realtime"); + voiceApi.getEndpoints.mockResolvedValue({ + realtime: null, + stt: "wss://second.test/realtime", + tts: null, + } as never); + await user.type(screen.getByLabelText("STT key"), "first-key"); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(onSaveKey).toHaveBeenCalledWith( + "first-key", + "wss://first.test/realtime", + ); + await user.click(screen.getByRole("button", { name: "Remove" })); + expect(onClearKey).toHaveBeenCalledWith("wss://first.test/realtime"); +}); diff --git a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx index 8b97a0777..9970ec312 100644 --- a/src/features/voice-conversation/ui/OpenAiEndpointField.tsx +++ b/src/features/voice-conversation/ui/OpenAiEndpointField.tsx @@ -27,8 +27,8 @@ export function OpenAiEndpointField({ label: string; keyLabel: string; configured: boolean; - onSaveKey: (apiKey: string) => Promise; - onClearKey: () => Promise; + onSaveKey: (apiKey: string, expectedUrl: string) => Promise; + onClearKey: (expectedUrl: string) => Promise; }) { const { t } = useTranslation("settings"); const id = useId(); @@ -81,11 +81,11 @@ export function OpenAiEndpointField({ let urlSaved = false; const savingKey = Boolean(apiKey.trim()); try { - // The key command uses the persisted endpoint; commit the displayed URL first. + // Commit the draft URL first, then bind the key mutation to that same URL. if (changed) { await setOpenAiVoiceEndpoint(kind, url); urlSaved = true; - targetUrl = (await getOpenAiVoiceEndpoints())[kind] ?? ""; + targetUrl = url.trim(); setUrl(targetUrl); setSavedUrl(targetUrl); setLocalKeyStatus({ @@ -95,7 +95,7 @@ export function OpenAiEndpointField({ }); } if (savingKey) { - await onSaveKey(apiKey); + await onSaveKey(apiKey, targetUrl); setApiKey(""); setLocalKeyStatus({ url: targetUrl, @@ -132,7 +132,7 @@ export function OpenAiEndpointField({ setSaving(true); setError(null); try { - await onClearKey(); + await onClearKey(savedUrl); setApiKey(""); setLocalKeyStatus({ url: savedUrl, diff --git a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx index bf27e8273..0a9ab47bf 100644 --- a/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/RealtimeVoiceSettings.test.tsx @@ -88,7 +88,10 @@ describe("RealtimeVoiceSettings", () => { "realtime", "ws://127.0.0.1:18870/v1/realtime", ); - expect(openAiVoiceMocks.setApiKey).toHaveBeenCalledWith(" sk-shared "); + expect(openAiVoiceMocks.setApiKey).toHaveBeenCalledWith( + " sk-shared ", + "ws://127.0.0.1:18870/v1/realtime", + ); expect( openAiVoiceMocks.setEndpoint.mock.invocationCallOrder[0], ).toBeLessThan(openAiVoiceMocks.setApiKey.mock.invocationCallOrder[0]); diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 087f2727b..36d8b7568 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -661,7 +661,7 @@ describe("VoiceSettings", () => { ); await user.click(screen.getAllByRole("button", { name: "Save" })[0]); - expect(openAiApiMocks.setSttApiKey).toHaveBeenCalledWith("stt-secret"); + expect(openAiApiMocks.setSttApiKey).toHaveBeenCalledWith("stt-secret", ""); }); it.each([ @@ -686,7 +686,7 @@ describe("VoiceSettings", () => { ? openAiApiMocks.setSttApiKey : openAiApiMocks.setTtsApiKey; expect(openAiApiMocks.setEndpoint).toHaveBeenCalledWith(kind, url); - expect(saveKey).toHaveBeenCalledWith("local-test"); + expect(saveKey).toHaveBeenCalledWith("local-test", url); expect(openAiApiMocks.setEndpoint.mock.invocationCallOrder[0]).toBeLessThan( saveKey.mock.invocationCallOrder[0], ); @@ -736,7 +736,7 @@ describe("VoiceSettings", () => { ); await user.click(screen.getByRole("button", { name: "Save" })); - expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret"); + expect(openAiApiMocks.setTtsApiKey).toHaveBeenCalledWith("tts-secret", ""); }); it("uses OpenAI guidance when only the selected OpenAI input is not ready", async () => { From c2f14aa31a826f1520138b38a240e6fe16857a82 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Fri, 2 Oct 2026 14:48:57 -0400 Subject: [PATCH 28/34] test(voice): verify call-scoped endpoint routing with local services --- scripts/verify-voice-endpoint-routing.py | 126 +++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 scripts/verify-voice-endpoint-routing.py diff --git a/scripts/verify-voice-endpoint-routing.py b/scripts/verify-voice-endpoint-routing.py new file mode 100644 index 000000000..a8ab1ec76 --- /dev/null +++ b/scripts/verify-voice-endpoint-routing.py @@ -0,0 +1,126 @@ +# /// script +# dependencies = ["websockets>=16,<17"] +# /// +"""Exercise berd-call URL overrides with loopback services and a PCM test host.""" +import argparse +import asyncio +import json +import os +import socket +import struct +from pathlib import Path +from tempfile import TemporaryDirectory +from urllib.parse import urlsplit +from websockets.exceptions import ConnectionClosed +from websockets.asyncio.server import serve + +async def main(binary): + observed = [] + sandbox = TemporaryDirectory(prefix="berd-voice-routing-") + config = Path(sandbox.name) / "config" / "openai-voice-endpoints.json" + config.parent.mkdir() + saved_settings = b'{"stt":"wss://saved.example/stt","tts":"https://saved.example/tts","realtime":"wss://saved.example/realtime"}' + config.write_bytes(saved_settings) + async def websocket(ws): + observed.append(urlsplit(ws.request.path).path) + assert ws.request.headers["Authorization"] == "Bearer disposable-routing-test" + try: + async for raw in ws: + event = json.loads(raw) + if event.get('type') == 'session.update': + session = event['session'] + await ws.send(json.dumps({'type': 'session.updated', 'session': { + 'model': 'test-model', 'audio': session.get('audio', {}) + }})) + except ConnectionClosed: + pass + async def synthesis(reader, writer): + headers = await reader.readuntil(b'\r\n\r\n') + length = next(int(line.split(b':', 1)[1]) for line in headers.split(b'\r\n') if line.lower().startswith(b'content-length:')) + body = json.loads(await reader.readexactly(length)) + assert body['input'] == 'A lighthouse guides the boat home.' + observed.append(headers.split(b' ')[1].decode()) + pcm = b'\x00\x00' * 2400 + writer.write(b'HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\nContent-Length: ' + str(len(pcm)).encode() + b'\r\nConnection: close\r\n\r\n' + pcm) + await writer.drain() + writer.close() + async with serve(websocket, '127.0.0.1', 0) as ws_server: + http = await asyncio.start_server(synthesis, '127.0.0.1', 0) + ws_port = ws_server.sockets[0].getsockname()[1] + http_port = http.sockets[0].getsockname()[1] + async def run(options, speak=False, defaults=False): + child_audio, host_audio = socket.socketpair() + env = dict(os.environ, OPENAI_API_KEY='disposable-routing-test', OPENAI_REALTIME_MODEL='test-model', OPENAI_REALTIME_ENDPOINT='ws://127.0.0.1:1/unused', OPENAI_BASE_URL='http://127.0.0.1:1/unused') + env['GOOSE_PATH_ROOT'] = sandbox.name + if defaults: + env['OPENAI_REALTIME_ENDPOINT'] = f'ws://127.0.0.1:{ws_port}/default-stt' + env['OPENAI_BASE_URL'] = f'http://127.0.0.1:{http_port}/default' + process = await asyncio.create_subprocess_exec(str(binary), 'session', '--pcm-output-fd', str(child_audio.fileno()), *options, env=env, pass_fds=(child_audio.fileno(),), stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE) + child_audio.close() + async def send(value): + payload = json.dumps(value).encode() + process.stdin.write(b'BV\x03\x01' + struct.pack(' Date: Fri, 2 Oct 2026 15:00:36 -0400 Subject: [PATCH 29/34] Keep voice endpoint credentials paired during startup --- src-tauri/src/commands/native_voice.rs | 18 +++-- src-tauri/src/commands/openai_audio.rs | 20 ++--- src-tauri/src/commands/openai_realtime.rs | 7 +- .../src/commands/openai_voice_credentials.rs | 12 +++ .../src/commands/openai_voice_endpoints.rs | 74 ++++++++++++++++++- 5 files changed, 106 insertions(+), 25 deletions(-) diff --git a/src-tauri/src/commands/native_voice.rs b/src-tauri/src/commands/native_voice.rs index fda95c3c8..421292d53 100644 --- a/src-tauri/src/commands/native_voice.rs +++ b/src-tauri/src/commands/native_voice.rs @@ -1465,12 +1465,12 @@ pub async fn start_native_voice_conversation( "Download the macOS speech recognition model before starting a call.".to_string(), ); } - let openai_api_key = if input_backend == VoiceInputBackend::Openai { - Some(super::openai_audio::stt_api_key()?) + let openai_endpoint_and_key = if input_backend == VoiceInputBackend::Openai { + Some(super::openai_audio::stt_endpoint_and_key()?) } else { None }; - if openai_api_key.is_some() { + if openai_endpoint_and_key.is_some() { let deadline = tokio::time::Instant::now() + Duration::from_secs(1); while !webview_window.is_focused().unwrap_or(false) && tokio::time::Instant::now() < deadline @@ -1525,13 +1525,15 @@ pub async fn start_native_voice_conversation( return Err("macOS speech recognition requires macOS 26 or later.".to_string()); } } - VoiceInputBackend::Openai => super::openai_audio::realtime_endpoint().map(|endpoint| { - berd_call::input::VoiceInputEngineConfig::OpenAi { + VoiceInputBackend::Openai => { + let (endpoint, api_key) = + openai_endpoint_and_key.expect("OpenAI endpoint and key resolved for OpenAI input"); + Ok(berd_call::input::VoiceInputEngineConfig::OpenAi { endpoint, - api_key: openai_api_key.expect("OpenAI key resolved for OpenAI input"), + api_key, model: super::openai_audio::transcription_model(), - } - }), + }) + } }; let engine = match engine { Ok(engine) => engine, diff --git a/src-tauri/src/commands/openai_audio.rs b/src-tauri/src/commands/openai_audio.rs index a7c09df18..0bb86a1a0 100644 --- a/src-tauri/src/commands/openai_audio.rs +++ b/src-tauri/src/commands/openai_audio.rs @@ -174,17 +174,8 @@ fn env_trimmed(name: &str) -> Option { .filter(|value| !value.is_empty()) } -#[cfg(target_os = "macos")] -fn tts_api_key() -> Result { - openai_voice_credentials::require(OpenAiVoiceCredential::TextToSpeech) -} - -pub(crate) fn stt_api_key() -> Result { - openai_voice_credentials::require(OpenAiVoiceCredential::SpeechToText) -} - -pub(crate) fn realtime_endpoint() -> Result { - openai_voice_endpoints::effective_url(VoiceEndpointKind::Stt) +pub(crate) fn stt_endpoint_and_key() -> Result<(String, String), String> { + openai_voice_credentials::require_endpoint(OpenAiVoiceCredential::SpeechToText) } pub(crate) fn transcription_model() -> String { @@ -531,7 +522,8 @@ pub fn start_openai_voice_stream( else { return Ok(false); }; - let key = tts_api_key()?; + let (endpoint, key) = + openai_voice_credentials::require_endpoint(OpenAiVoiceCredential::TextToSpeech)?; { let mut playback = state .playback @@ -564,6 +556,7 @@ pub fn start_openai_voice_stream( let result = run_openai_voice_stream( &app, &stream_id, + endpoint, key, active.clone(), receiver, @@ -779,6 +772,7 @@ impl From for StreamFailure { fn run_openai_voice_stream( app: &AppHandle, stream_id: &str, + endpoint: String, key: String, active: Arc, receiver: mpsc::Receiver, @@ -791,7 +785,7 @@ fn run_openai_voice_stream( voice: String, ) -> Result { let tts = ConfiguredTtsSlot::new(TtsConfiguration::openai( - openai_voice_endpoints::effective_url(VoiceEndpointKind::Tts)?, + endpoint, key, speech_model(), voice, diff --git a/src-tauri/src/commands/openai_realtime.rs b/src-tauri/src/commands/openai_realtime.rs index 515507b26..11922daea 100644 --- a/src-tauri/src/commands/openai_realtime.rs +++ b/src-tauri/src/commands/openai_realtime.rs @@ -169,10 +169,11 @@ pub fn start_openai_realtime_spokesperson_runtime( return Err("This window already owns an OpenAI Realtime runtime session".into()); } - let api_key = - openai_voice_credentials::require(OpenAiVoiceCredential::SelectedRealtimeAssistant)?; + let (endpoint, api_key) = openai_voice_credentials::require_endpoint( + OpenAiVoiceCredential::SelectedRealtimeAssistant, + )?; let mut config = OpenAiSpokespersonConfig::new(api_key, options, Vec::new()); - config.endpoint = openai_voice_endpoints::effective_url(VoiceEndpointKind::Realtime)?; + config.endpoint = endpoint; let semantic_revision = Arc::new(AtomicU64::new(0)); let event_window = webview_window.clone(); let event_session_id = session_id.clone(); diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index a7ea8a1cb..05a08a9aa 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -132,6 +132,18 @@ pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result Result<(String, String), String> { + let kind = credential + .selected_kind() + .ok_or_else(|| "Dictation uses the fixed default endpoint".to_string())?; + openai_voice_endpoints::resolve_with_url(kind, |url| { + read_account(&account_for_url(kind, url))? + .ok_or_else(|| credential.missing_message().to_string()) + }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index 68dd1c7f3..d8a19b67b 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -166,7 +166,14 @@ fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { } pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { - if let Some(saved) = read_settings()?.get(kind) { + effective_url_from(&read_settings()?, kind) +} + +fn effective_url_from( + settings: &VoiceEndpointSettings, + kind: VoiceEndpointKind, +) -> Result { + if let Some(saved) = settings.get(kind) { return Ok(saved.to_string()); } if !matches!(kind, VoiceEndpointKind::Realtime) @@ -189,6 +196,27 @@ pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { Ok(kind.default_url().to_string()) } +/// Resolve an endpoint and its credential without allowing a concurrent selection change. +pub(crate) fn resolve_with_url( + kind: VoiceEndpointKind, + resolve: impl FnOnce(&str) -> Result, +) -> Result<(String, T), String> { + resolve_with_url_at(&settings_path()?, kind, resolve) +} + +fn resolve_with_url_at( + path: &std::path::Path, + kind: VoiceEndpointKind, + resolve: impl FnOnce(&str) -> Result, +) -> Result<(String, T), String> { + let _guard = SETTINGS_UPDATE_LOCK + .lock() + .map_err(|_| "OpenAI voice settings lock is poisoned".to_string())?; + let destination = effective_url_from(&read_settings_from(path)?, kind)?; + let credential = resolve(&destination)?; + Ok((destination, credential)) +} + #[tauri::command] pub(crate) fn get_openai_voice_endpoints() -> Result { read_settings() @@ -248,6 +276,50 @@ mod tests { use std::sync::mpsc; use std::time::Duration; + #[test] + fn startup_keeps_the_credential_and_destination_paired_during_endpoint_changes() { + for kind in [ + VoiceEndpointKind::Stt, + VoiceEndpointKind::Tts, + VoiceEndpointKind::Realtime, + ] { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("endpoints.json"); + let (first, second) = if matches!(kind, VoiceEndpointKind::Tts) { + ("https://first.test/speech", "https://second.test/speech") + } else { + ("wss://first.test/realtime", "wss://second.test/realtime") + }; + update_settings_file(&path, |settings| settings.set(kind, Some(first.into()))).unwrap(); + let update_path = path.clone(); + let (started_tx, started_rx) = mpsc::channel(); + let (finished_tx, finished_rx) = mpsc::channel(); + let mut update = None; + let (destination, credential_url) = resolve_with_url_at(&path, kind, |url| { + update = Some(std::thread::spawn(move || { + started_tx.send(()).unwrap(); + update_settings_file(&update_path, |settings| { + settings.set(kind, Some(second.into())) + }) + .unwrap(); + finished_tx.send(()).unwrap(); + })); + started_rx.recv().unwrap(); + // A concurrent save must not change the destination while its key is being read. + let _ = finished_rx.recv_timeout(Duration::from_millis(100)); + Ok(url.to_string()) + }) + .unwrap(); + update.unwrap().join().unwrap(); + assert_eq!( + destination, credential_url, + "a key must only be sent to its own endpoint" + ); + assert_eq!(destination, first); + assert_eq!(read_settings_from(&path).unwrap().get(kind), Some(second)); + } + } + #[test] fn credential_mutations_reject_a_stale_displayed_endpoint() { let directory = tempfile::tempdir().unwrap(); From 476361023d84a72edf54861c4d72828f4de5f729 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 5 Oct 2026 08:31:00 -0400 Subject: [PATCH 30/34] Preserve legacy environment-routed voice credentials --- .../src/commands/openai_voice_credentials.rs | 77 ++++++++++++++++++- .../src/commands/openai_voice_endpoints.rs | 6 ++ 2 files changed, 81 insertions(+), 2 deletions(-) diff --git a/src-tauri/src/commands/openai_voice_credentials.rs b/src-tauri/src/commands/openai_voice_credentials.rs index 05a08a9aa..270c59b56 100644 --- a/src-tauri/src/commands/openai_voice_credentials.rs +++ b/src-tauri/src/commands/openai_voice_credentials.rs @@ -48,7 +48,16 @@ fn account(credential: OpenAiVoiceCredential) -> Result { return Ok(KEYCHAIN_ACCOUNT.to_string()); }; let url = openai_voice_endpoints::effective_url(kind)?; - Ok(account_for_url(kind, &url)) + account_for_selected_url(kind, &url) +} + +fn account_for_selected_url(kind: VoiceEndpointKind, url: &str) -> Result { + // Environment routing predates URL-scoped keys and retains its existing shared credential. + // Saved URLs never inherit this account, even while the environment override is present. + if openai_voice_endpoints::uses_legacy_environment_credential(kind)? { + return Ok(KEYCHAIN_ACCOUNT.to_string()); + } + Ok(account_for_url(kind, url)) } fn account_for_url(kind: VoiceEndpointKind, url: &str) -> String { @@ -134,12 +143,19 @@ pub(crate) fn require(credential: OpenAiVoiceCredential) -> Result Result<(String, String), String> { + require_endpoint_with(credential, read_account) +} + +fn require_endpoint_with( + credential: OpenAiVoiceCredential, + read: impl FnOnce(&str) -> Result, String>, ) -> Result<(String, String), String> { let kind = credential .selected_kind() .ok_or_else(|| "Dictation uses the fixed default endpoint".to_string())?; openai_voice_endpoints::resolve_with_url(kind, |url| { - read_account(&account_for_url(kind, url))? + read(&account_for_selected_url(kind, url)?)? .ok_or_else(|| credential.missing_message().to_string()) }) } @@ -147,6 +163,63 @@ pub(crate) fn require_endpoint( #[cfg(test)] mod tests { use super::*; + #[test] + fn environment_endpoint_upgrade_preserves_the_legacy_shared_credential() { + const CHILD: &str = "BERD_TEST_ENVIRONMENT_CREDENTIAL_UPGRADE"; + if std::env::var_os(CHILD).is_none() { + let root = tempfile::tempdir().unwrap(); + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "commands::openai_voice_credentials::tests::environment_endpoint_upgrade_preserves_the_legacy_shared_credential", "--nocapture"]) + .env(CHILD, "1") + .env("GOOSE_PATH_ROOT", root.path()) + .env(openai_voice_endpoints::BASE_URL_ENV, "https://legacy.test/openai") + .output().unwrap(); + assert!( + output.status.success(), + "{}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + return; + } + for (credential, destination) in [ + ( + OpenAiVoiceCredential::SpeechToText, + "wss://legacy.test/openai/realtime?intent=transcription", + ), + ( + OpenAiVoiceCredential::TextToSpeech, + "https://legacy.test/openai/audio/speech", + ), + ] { + let (url, key) = require_endpoint_with(credential, |account| { + Ok((account == KEYCHAIN_ACCOUNT).then(|| "legacy-disposable-key".to_string())) + }) + .unwrap(); + assert_eq!(url, destination); + assert_eq!(key, "legacy-disposable-key"); + } + let custom = openai_voice_endpoints::VoiceEndpointSettings { + stt: Some("wss://custom.test/realtime".into()), + tts: Some("https://custom.test/speech".into()), + realtime: Some("wss://custom.test/assistant".into()), + }; + openai_voice_endpoints::restore(&custom).unwrap(); + for credential in [ + OpenAiVoiceCredential::SpeechToText, + OpenAiVoiceCredential::TextToSpeech, + OpenAiVoiceCredential::SelectedRealtimeAssistant, + ] { + assert!( + require_endpoint_with(credential, |account| { + Ok((account == KEYCHAIN_ACCOUNT).then(|| "legacy-disposable-key".to_string())) + }) + .is_err(), + "a saved custom endpoint must not inherit the legacy shared key" + ); + } + } + #[test] fn speech_services_use_independent_endpoints() { assert!(matches!( diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index d8a19b67b..f64bf8c70 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -165,6 +165,12 @@ fn endpoint_for_base_url(base_url: &str, path: &str) -> Result { Ok(url.to_string()) } +pub(crate) fn uses_legacy_environment_credential(kind: VoiceEndpointKind) -> Result { + Ok(!matches!(kind, VoiceEndpointKind::Realtime) + && std::env::var_os(BASE_URL_ENV).is_some() + && read_settings()?.get(kind).is_none()) +} + pub(crate) fn effective_url(kind: VoiceEndpointKind) -> Result { effective_url_from(&read_settings()?, kind) } From f7f81f40142c5a43c05ae3b9cbc3e6ad904174ed Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 5 Oct 2026 08:31:38 -0400 Subject: [PATCH 31/34] Refresh Realtime endpoint controls after voice reset --- .../ui/VoiceSettings.test.tsx | 37 ++++++++++++++++++- .../voice-conversation/ui/VoiceSettings.tsx | 2 +- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/features/voice-conversation/ui/VoiceSettings.test.tsx b/src/features/voice-conversation/ui/VoiceSettings.test.tsx index 36d8b7568..13609db47 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.test.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.test.tsx @@ -112,6 +112,7 @@ const openAiApiMocks = vi.hoisted(() => ({ tts: null as string | null, }), ), + setRealtimeApiKey: vi.fn(() => Promise.resolve()), setEndpoint: vi.fn(() => Promise.resolve()), setSttApiKey: vi.fn(() => Promise.resolve()), clearSttApiKey: vi.fn(() => Promise.resolve()), @@ -124,7 +125,7 @@ const openAiApiMocks = vi.hoisted(() => ({ })); vi.mock("../api/openAiVoice", () => ({ - setOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), + setOpenAiRealtimeApiKey: openAiApiMocks.setRealtimeApiKey, clearOpenAiRealtimeApiKey: vi.fn(() => Promise.resolve()), getOpenAiVoiceEndpoints: openAiApiMocks.getEndpoints, getOpenAiVoiceStatus: openAiApiMocks.getStatus, @@ -441,6 +442,40 @@ describe("VoiceSettings", () => { expect(preferenceMocks.setRealtimePreference).toHaveBeenCalledOnce(); }); + it("refreshes the mounted Realtime endpoint after resetting all voice settings", async () => { + modeState.mode = "openai-realtime"; + openAiApiMocks.getEndpoints.mockResolvedValue({ + realtime: "wss://previous.test/realtime", + stt: null, + tts: null, + }); + openAiApiMocks.resetAll.mockImplementationOnce(async () => { + openAiApiMocks.getEndpoints.mockResolvedValue({ + realtime: null, + stt: null, + tts: null, + }); + }); + renderWithProviders(); + const user = userEvent.setup(); + expect( + await screen.findByDisplayValue("wss://previous.test/realtime"), + ).toBeInTheDocument(); + await user.click(screen.getByRole("button", { name: "Reset to defaults" })); + await user.click( + within( + screen.getByRole("dialog", { name: "Reset all voice settings?" }), + ).getByRole("button", { name: "Reset to defaults" }), + ); + expect(screen.getByLabelText("Realtime endpoint URL")).toHaveValue(""); + await user.type(screen.getByLabelText("OpenAI API key"), "disposable-key"); + await user.click(screen.getByRole("button", { name: "Save" })); + expect(openAiApiMocks.setRealtimeApiKey).toHaveBeenCalledWith( + "disposable-key", + "", + ); + }); + it("waits for Apple capability detection before offering reset", () => { macSpeechSetupState.current = { ...macSpeechSetupState.current, diff --git a/src/features/voice-conversation/ui/VoiceSettings.tsx b/src/features/voice-conversation/ui/VoiceSettings.tsx index f83c11a64..ea1782d0b 100644 --- a/src/features/voice-conversation/ui/VoiceSettings.tsx +++ b/src/features/voice-conversation/ui/VoiceSettings.tsx @@ -614,7 +614,7 @@ export function VoiceSettings() {
) : ( - + )}
Date: Mon, 5 Oct 2026 13:16:53 -0400 Subject: [PATCH 32/34] Move spoken reply card fix to its dedicated PR Revert b5d06ea35cf24218b94401a74a6dc457ffaa7be0 here.\nThe isolated transcript fix is preserved in PR #375. --- .../projection/buildTranscriptItems.ts | 5 ---- .../transcriptProjectionCache.test.ts | 27 ------------------- 2 files changed, 32 deletions(-) diff --git a/src/features/chat/transcript/projection/buildTranscriptItems.ts b/src/features/chat/transcript/projection/buildTranscriptItems.ts index 2c0d0a686..0b24f6686 100644 --- a/src/features/chat/transcript/projection/buildTranscriptItems.ts +++ b/src/features/chat/transcript/projection/buildTranscriptItems.ts @@ -1297,11 +1297,6 @@ function canProjectAssistantTextFragments( if (visibleContent.length !== 1 || visibleContent[0]?.type !== "text") { return false; } - // Speech belongs to the whole assistant reply. Fragmenting its text would - // duplicate the playback status and draw each paragraph as a separate card. - if (visibleContent[0].speech) { - return false; - } if ( message.metadata?.attachments?.length || message.metadata?.chips?.length diff --git a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts index 629415887..7a5db3617 100644 --- a/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts +++ b/src/features/chat/transcript/projection/transcriptProjectionCache.test.ts @@ -123,33 +123,6 @@ describe("transcript projection cache", () => { ); }); - it("keeps a spoken multi-paragraph reply in one assistant card", () => { - const assistant = messageWithContent( - "assistant-voice-story", - "assistant", - [ - { - type: "text", - text: multiParagraphText("story paragraph", 3, 20), - speech: { status: "speaking" }, - }, - ], - utc(2026, 6, 4, 10), - { completionStatus: "completed" }, - ); - - const snapshot = update(createTranscriptProjectionCache(), [assistant]); - - expect( - snapshot.rows.filter((row) => row.messageId === assistant.id), - ).toHaveLength(1); - expect(messageRow(snapshot, assistant.id)).toMatchObject({ - kind: "message", - rowId: "message:assistant-voice-story", - }); - expect(snapshot.fragmentRowCount).toBe(0); - }); - it("keeps long markdown tables on whole-message rows", () => { const cache = createTranscriptProjectionCache(); const assistant = message( From 967fad513adc5454bb13b34cf859644526ba5757 Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 5 Oct 2026 13:58:27 -0400 Subject: [PATCH 33/34] Keep transient voice endpoint URLs out of restart preferences --- .../crates/berd-call/src/host_session.rs | 66 ++++++++++++++++++- src-tauri/crates/berd-call/src/main.rs | 11 +--- .../crates/berd-call/src/saved_settings.rs | 14 ++++ 3 files changed, 80 insertions(+), 11 deletions(-) diff --git a/src-tauri/crates/berd-call/src/host_session.rs b/src-tauri/crates/berd-call/src/host_session.rs index cd6820707..d827ae2fc 100644 --- a/src-tauri/crates/berd-call/src/host_session.rs +++ b/src-tauri/crates/berd-call/src/host_session.rs @@ -27,6 +27,7 @@ use berd_call::PocketAudioPlayer; use crate::codex::{self, CodexRecord, CodexRelay, CodexTarget}; use crate::host_control::{ControlServer, HostControl}; +use crate::saved_settings; use crate::session_audio::{ AUDIO_BEGIN_KIND, AUDIO_CANCEL_KIND, AUDIO_CHUNK_KIND, AUDIO_END_KIND, AUDIO_FRAME_HEADER_BYTES, AUDIO_FRAME_MAGIC, AUDIO_FRAME_MARKER, @@ -340,7 +341,7 @@ impl SessionActor { ) -> SessionStart { SessionStart { saved: self.saved.take().map(|(path, mut saved)| { - saved.arguments = restart.arguments[1..].to_vec(); + saved.arguments = saved_settings::persistable_arguments(&restart.arguments[1..]); saved.tts = None; (path, saved) }), @@ -2687,6 +2688,69 @@ mod tests { ); } + #[test] + fn restart_keeps_endpoint_overrides_out_of_saved_preferences() { + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("settings.json"); + let mut child = Command::new("/bin/cat") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .spawn() + .unwrap(); + let writer = Arc::new(Mutex::new(child.stdin.take().unwrap())); + let (_events_tx, events) = mpsc::sync_channel(1); + let (_commands_tx, commands) = mpsc::sync_channel(1); + let (audio, _audio_rx) = mpsc::sync_channel(1); + let mut actor = test_actor(writer, events, commands, audio); + actor.saved = Some(( + path.clone(), + crate::saved_settings::SavedSettings::default(), + )); + let arguments = [ + "session", + "--mode", + "chained", + "--realtime-url", + "ws://127.0.0.1:18870/realtime", + "--stt-backend", + "openai", + "--stt-url", + "ws://127.0.0.1:18870/stt", + "--tts-backend", + "openai", + "--tts-url", + "http://127.0.0.1:18870/tts", + ] + .map(str::to_string) + .to_vec(); + let (response, _reply) = mpsc::sync_channel(1); + let start = actor.restart_start( + PendingRestart { + arguments: arguments.clone(), + expert_spokesperson: false, + response, + }, + InputDuringTtsPolicy::AllowBargeIn, + ); + assert_eq!(start.arguments, arguments); + let (_, saved) = start.saved.unwrap(); + crate::saved_settings::save(&path, &saved).unwrap(); + assert_eq!( + crate::saved_settings::load(&path).unwrap().arguments, + [ + "--mode", + "chained", + "--stt-backend", + "openai", + "--tts-backend", + "openai", + ] + .map(str::to_string) + ); + drop(actor); + assert!(child.wait().unwrap().success()); + } + #[test] fn restart_preserves_native_intent_before_child_acknowledgement() { for requested in [true, false] { diff --git a/src-tauri/crates/berd-call/src/main.rs b/src-tauri/crates/berd-call/src/main.rs index 8d3078a78..c1a2083ba 100644 --- a/src-tauri/crates/berd-call/src/main.rs +++ b/src-tauri/crates/berd-call/src/main.rs @@ -812,16 +812,7 @@ fn parse_saved_start_args_at(args: &[String], path: PathBuf) -> Result Vec Vec { + arguments + .chunks_exact(2) + .filter(|pair| { + !matches!( + pair[0].as_str(), + "--realtime-url" | "--stt-url" | "--tts-url" + ) + }) + .flatten() + .cloned() + .collect() +} + pub(crate) fn without_tts(arguments: &[String]) -> Vec { arguments .chunks_exact(2) From e2b182b7f6d747356d4609951be5c81705033efc Mon Sep 17 00:00:00 2001 From: John Tennant Date: Mon, 5 Oct 2026 14:11:19 -0400 Subject: [PATCH 34/34] Preserve explicit default voice endpoints over legacy environment routing --- .../src/commands/openai_voice_endpoints.rs | 64 +++++++++++++++++-- 1 file changed, 59 insertions(+), 5 deletions(-) diff --git a/src-tauri/src/commands/openai_voice_endpoints.rs b/src-tauri/src/commands/openai_voice_endpoints.rs index f64bf8c70..dfca2eede 100644 --- a/src-tauri/src/commands/openai_voice_endpoints.rs +++ b/src-tauri/src/commands/openai_voice_endpoints.rs @@ -234,14 +234,22 @@ pub(crate) fn set_openai_voice_endpoint( kind: VoiceEndpointKind, url: String, ) -> Result<(), String> { - let selected = validate(kind, &url)?; - update_settings_file(&settings_path()?, |settings| { - settings.set(kind, selected.filter(|url| url != kind.default_url())); - })?; + set_endpoint_at(&settings_path()?, kind, &url)?; app.emit(SETTINGS_CHANGED_EVENT, ()) .map_err(|error| format!("Could not refresh OpenAI voice settings: {error}")) } +fn set_endpoint_at( + path: &std::path::Path, + kind: VoiceEndpointKind, + url: &str, +) -> Result<(), String> { + let selected = validate(kind, url)?; + update_settings_file(path, |settings| { + settings.set(kind, selected); + }) +} + /// Keep endpoint selection stable while a credential mutation uses it. pub(crate) fn with_selected_endpoint( kind: VoiceEndpointKind, @@ -257,7 +265,7 @@ fn with_selected_endpoint_at( expected_url: &str, mutate: impl FnOnce() -> Result, ) -> Result { - let expected = validate(kind, expected_url)?.filter(|url| url != kind.default_url()); + let expected = validate(kind, expected_url)?; let _guard = SETTINGS_UPDATE_LOCK .lock() .map_err(|_| "OpenAI voice settings lock is poisoned".to_string())?; @@ -361,6 +369,52 @@ mod tests { assert!(mutated); } + #[test] + fn explicit_default_url_overrides_legacy_environment_routing() { + const CHILD: &str = "BERD_TEST_EXPLICIT_OPENAI_URL"; + if std::env::var_os(CHILD).is_none() { + let output = std::process::Command::new(std::env::current_exe().unwrap()) + .args(["--exact", "commands::openai_voice_endpoints::tests::explicit_default_url_overrides_legacy_environment_routing", "--nocapture"]) + .env(CHILD, "1") + .env(BASE_URL_ENV, "https://legacy.test/openai") + .output() + .unwrap(); + assert!( + output.status.success(), + "{}", + String::from_utf8_lossy(&output.stderr) + ); + return; + } + let directory = tempfile::tempdir().unwrap(); + let path = directory.path().join("endpoints.json"); + for kind in [VoiceEndpointKind::Stt, VoiceEndpointKind::Tts] { + assert_ne!( + effective_url_from(&read_settings_from(&path).unwrap(), kind).unwrap(), + kind.default_url() + ); + set_endpoint_at(&path, kind, kind.default_url()).unwrap(); + let settings = read_settings_from(&path).unwrap(); + assert_eq!(settings.get(kind), Some(kind.default_url())); + assert_eq!( + effective_url_from(&settings, kind).unwrap(), + kind.default_url() + ); + with_selected_endpoint_at(&path, kind, kind.default_url(), || Ok(())).unwrap(); + } + set_endpoint_at(&path, VoiceEndpointKind::Stt, "").unwrap(); + assert_ne!( + effective_url_from(&read_settings_from(&path).unwrap(), VoiceEndpointKind::Stt) + .unwrap(), + STT_DEFAULT + ); + assert_eq!( + effective_url_from(&read_settings_from(&path).unwrap(), VoiceEndpointKind::Tts) + .unwrap(), + TTS_DEFAULT + ); + } + #[test] fn concurrent_endpoint_updates_preserve_both_services() { let directory = tempfile::tempdir().unwrap();