Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 25 additions & 5 deletions src/renderer/src/audio/streaming.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,10 @@ import { SAMPLE_RATE } from './vad'
* Macháček et al., "Turning Whisper into Real-Time Transcription System" (2023).
*
* Whisper is re-run over a growing audio buffer every half second or so. A word
* is committed only once two consecutive passes agree on it, and never while it
* is committed only once two consecutive passes agree on it, and not while it
* is the newest word heard: Whisper's guess at the trailing word (its casing,
* punctuation, and timing) is the least reliable. Committed text is stable;
* punctuation, and timing) is the least reliable. The exception is a trailing
* word both passes heard identically as a sentence end. Committed text is stable;
* everything after it is shown as an unconfirmed "live" tail.
*
* Whisper sees whole sentences rather than pause-delimited fragments, so it
Expand Down Expand Up @@ -141,15 +142,19 @@ export class StreamingTranscriber {
}

const base = this.offset / SAMPLE_RATE
// Whisper pads input to 30 s, so on short clips it can time the last word
// well past the real audio. A bogus end would outlive the trim and make the
// timestamp fallback in `unseen` discard everything said afterwards.
const limit = base + end / SAMPLE_RATE
const overdue = finalizeAt === undefined && this.length >= MAX_BUFFER
this.passStartedAt = Date.now()
this.passes++
let heard: TranscriptWord[]
try {
heard = (await this.transcribe(this.audio(end), this.prompt())).map((word) => ({
...word,
start: word.start + base,
end: word.end + base
start: Math.min(word.start + base, limit),
end: Math.min(word.end + base, limit)
}))
} finally {
this.passStartedAt = 0
Expand All @@ -159,7 +164,7 @@ export class StreamingTranscriber {
let agreed: number
if (finalizeAt !== undefined) agreed = words.length
else if (overdue) agreed = Math.max(0, words.length - 1)
else agreed = Math.min(agreedPrefix(this.tentative, words), words.length - 1)
else agreed = this.agreed(words)
agreed = Math.max(0, agreed)

const committed = words.slice(0, agreed)
Expand All @@ -183,6 +188,21 @@ export class StreamingTranscriber {
} else this.trimAtSentence()
}

/**
* How many of this pass's words both passes agree on. The newest word is held
* back, unless both passes heard it identically as the end of a sentence:
* otherwise the sentence end only commits once more speech follows, and by
* then Whisper has often rewritten it ("years." → "years, and"), so the
* buffer never trims at a sentence and grows to MAX_BUFFER.
*/
private agreed(words: TranscriptWord[]): number {
const n = agreedPrefix(this.tentative, words)
if (n === 0 || n < words.length) return n
const last = words[n - 1]
const sameEnd = SENTENCE_END.test(last.text) && this.tentative[n - 1].text === last.text
return sameEnd ? n : n - 1
}

private commit(words: TranscriptWord[]): void {
if (!words.length) return
this.committed.push(...words)
Expand Down
Loading