From 2261ab5582b977244362791b00665fe7d0f5fed9 Mon Sep 17 00:00:00 2001 From: adavila0703 Date: Fri, 13 Nov 2026 20:24:49 -0600 Subject: [PATCH] Fix dictation freezing after short phrases and sentence ends never committing. - Clamp Whisper word timestamps to the audio actually transcribed. On short clips Whisper (which pads to 30s) can time the last word far past the real audio; that bogus end outlived the trim, so the timestamp fallback discarded every new word until the buffer caught up (~18s of lost speech, and buffers growing to 30s+ with 2s passes). - Commit the newest word when two passes heard it identically as a sentence end. Holding it back meant Whisper often rewrote "years." to "years, and" first, so no sentence end committed and the buffer hit the 15s cap and was cut mid-sentence. Co-Authored-By: Claude --- src/renderer/src/audio/streaming.ts | 30 ++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/renderer/src/audio/streaming.ts b/src/renderer/src/audio/streaming.ts index 96c70d0..5c1a3a6 100644 --- a/src/renderer/src/audio/streaming.ts +++ b/src/renderer/src/audio/streaming.ts @@ -7,9 +7,10 @@ import { SAMPLE_RATE } from './vad' * Macháček et al., "Turning Whisper into Real-Time Transcription System" (2023). * * Whisper is re-run over a growing audio buffer every half second or so. A word - * is committed only once two consecutive passes agree on it, and never while it + * is committed only once two consecutive passes agree on it, and not while it * is the newest word heard: Whisper's guess at the trailing word (its casing, - * punctuation, and timing) is the least reliable. Committed text is stable; + * punctuation, and timing) is the least reliable. The exception is a trailing + * word both passes heard identically as a sentence end. Committed text is stable; * everything after it is shown as an unconfirmed "live" tail. * * Whisper sees whole sentences rather than pause-delimited fragments, so it @@ -141,6 +142,10 @@ export class StreamingTranscriber { } const base = this.offset / SAMPLE_RATE + // Whisper pads input to 30 s, so on short clips it can time the last word + // well past the real audio. A bogus end would outlive the trim and make the + // timestamp fallback in `unseen` discard everything said afterwards. + const limit = base + end / SAMPLE_RATE const overdue = finalizeAt === undefined && this.length >= MAX_BUFFER this.passStartedAt = Date.now() this.passes++ @@ -148,8 +153,8 @@ export class StreamingTranscriber { try { heard = (await this.transcribe(this.audio(end), this.prompt())).map((word) => ({ ...word, - start: word.start + base, - end: word.end + base + start: Math.min(word.start + base, limit), + end: Math.min(word.end + base, limit) })) } finally { this.passStartedAt = 0 @@ -159,7 +164,7 @@ export class StreamingTranscriber { let agreed: number if (finalizeAt !== undefined) agreed = words.length else if (overdue) agreed = Math.max(0, words.length - 1) - else agreed = Math.min(agreedPrefix(this.tentative, words), words.length - 1) + else agreed = this.agreed(words) agreed = Math.max(0, agreed) const committed = words.slice(0, agreed) @@ -183,6 +188,21 @@ export class StreamingTranscriber { } else this.trimAtSentence() } + /** + * How many of this pass's words both passes agree on. The newest word is held + * back, unless both passes heard it identically as the end of a sentence: + * otherwise the sentence end only commits once more speech follows, and by + * then Whisper has often rewritten it ("years." → "years, and"), so the + * buffer never trims at a sentence and grows to MAX_BUFFER. + */ + private agreed(words: TranscriptWord[]): number { + const n = agreedPrefix(this.tentative, words) + if (n === 0 || n < words.length) return n + const last = words[n - 1] + const sameEnd = SENTENCE_END.test(last.text) && this.tentative[n - 1].text === last.text + return sameEnd ? n : n - 1 + } + private commit(words: TranscriptWord[]): void { if (!words.length) return this.committed.push(...words)