diff --git a/.claude/skills/porting-1-intake/SKILL.md b/.claude/skills/porting-1-intake/SKILL.md index 0b7a5f987..0da7b7a89 100644 --- a/.claude/skills/porting-1-intake/SKILL.md +++ b/.claude/skills/porting-1-intake/SKILL.md @@ -55,6 +55,12 @@ field semantics. 2. **Architecture pattern.** One of `encoder-transducer`, `encoder-decoder`, `audio-llm`, `encoder-ctc`. The script's `config.architecture_candidates` is a heuristic starting point. If it doesn't fit, propose a new pattern and have the user accept it based on your research of the architecture. + **Role.** ASR (the product is a transcript) or DIARIZE (the product is + who spoke when; see `docs/roles.md`). A DIARIZE port implements + `DiarizeOps` (`src/transcribe-diarize.h`) instead of the ASR hooks, sets + `"role": "diarize"` in its catalog record, and is accepted on DER + (`scripts/diar/`), not WER. + 3. **Acceptance dataset.** Default: LibriSpeech test-clean. Capture any publisher-reported score in `upstream_benchmarks` when available for context, but downstream gates use the measured Oracle reference diff --git a/.claude/skills/porting-7-wer/SKILL.md b/.claude/skills/porting-7-wer/SKILL.md index 3b27c3111..4005e5e55 100644 --- a/.claude/skills/porting-7-wer/SKILL.md +++ b/.claude/skills/porting-7-wer/SKILL.md @@ -42,7 +42,7 @@ WER progress: ### Step 1: Acceptance manifest (execute or ask-point) -Read the acceptance dataset and metric from `upstream_benchmarks[0].{dataset, metric}`. Slugify: lowercase, spaces → hyphens. `"LibriSpeech test-clean"` → `librispeech-test-clean`. Resolve to `$MANIFEST`; every later step uses `$MANIFEST`, never a reconstructed path. Confirm `metric` is `wer` or `cer`; anything else is out of scope. Do not use any publisher-reported score for pass/fail; the measured Oracle reference score is the gate target. +Read the acceptance dataset and metric from `upstream_benchmarks[0].{dataset, metric}`. Slugify: lowercase, spaces → hyphens. `"LibriSpeech test-clean"` → `librispeech-test-clean`. Resolve to `$MANIFEST`; every later step uses `$MANIFEST`, never a reconstructed path. Confirm `metric` is `wer` or `cer` (`der` for a DIARIZE-role model: score with `scripts/diar/` instead; anything else is out of scope). Do not use any publisher-reported score for pass/fail; the measured Oracle reference score is the gate target. If the intake's dataset is not covered by `scripts/wer/ingest.py`, extend that script before running this step. diff --git a/.github/actions/fetch-canary/action.yml b/.github/actions/fetch-canary/action.yml index 5a6ee6cd1..9cdeabf43 100644 --- a/.github/actions/fetch-canary/action.yml +++ b/.github/actions/fetch-canary/action.yml @@ -6,7 +6,8 @@ description: > secret): the model tests then skip, exactly as before this action existed. Always fetches ALL CI canaries (whisper-tiny + moonshine-streaming ~95 MB, Canary+SenseVoice text-control canaries ~290 MB, plus the two parakeet 0.6b - family-extension canaries — cache-aware + buffered — at Q4_K_M ~0.95 GB) + family-extension canaries — cache-aware + buffered — at Q4_K_M ~0.95 GB, + plus the Sortformer DIARIZE canary at Q8_0 ~140 MB) under one cache key — a per-consumer subset would let one job save the shared key with only its subset in it, and every other consumer would re-download forever (exact-key hits are never re-saved), which is also why the key carries @@ -35,7 +36,7 @@ runs: uses: actions/cache@v5 with: path: canary - key: canary-models-v3 + key: canary-models-v4 - name: Fetch canary models (cache miss only) if: inputs.hf-token != '' shell: bash @@ -67,6 +68,12 @@ runs: uvx --from huggingface_hub hf download \ handy-computer/parakeet-unified-en-0.6b-gguf \ parakeet-unified-en-0.6b-Q4_K_M.gguf --local-dir canary + # Sortformer DIARIZE canary (Q8_0, ~140 MB) for the binding diarize + # tests. + [ -f canary/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf ] || \ + uvx --from huggingface_hub hf download \ + handy-computer/diar_streaming_sortformer_4spk-v2.1-gguf \ + diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf --local-dir canary # GITHUB_WORKSPACE (not bash's $PWD) so Windows exports a path # Python can open rather than an MSYS one. prefix='${{ inputs.model-path-prefix }}' @@ -77,3 +84,4 @@ runs: echo "TRANSCRIBE_SMOKE_ITN_MODEL=$prefix/canary/SenseVoiceSmall-Q4_K_M.gguf" >> "$GITHUB_ENV" echo "TRANSCRIBE_SMOKE_PARAKEET_STREAM_MODEL=$prefix/canary/nemotron-speech-streaming-en-0.6b-Q4_K_M.gguf" >> "$GITHUB_ENV" echo "TRANSCRIBE_SMOKE_PARAKEET_BUFFERED_MODEL=$prefix/canary/parakeet-unified-en-0.6b-Q4_K_M.gguf" >> "$GITHUB_ENV" + echo "TRANSCRIBE_SMOKE_SORTFORMER_MODEL=$prefix/canary/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf" >> "$GITHUB_ENV" diff --git a/AGENTS.md b/AGENTS.md index 3c547af58..89a74ccf6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -44,7 +44,7 @@ scripts/ci/clang-format.sh --check # verify, no changes No C++ exception may escape a public entry point. - A new public entry point must either route through an `api_guard_*` - wrapper (`src/transcribe.cpp`) or be nothrow by construction. Device and + wrapper (`src/transcribe-api-guard.h`) or be nothrow by construction. Device and registry queries are not pure reads; guard them. - Entry points with ownership out-params enforce "non-OK => `*out == NULL`, nothing leaked" in their forwarders on every error return. diff --git a/README.md b/README.md index 19c924c49..1e9309777 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ C/C++ speech-to-text inference library. Runs diverse STT model families via [GGU **Speaker diarization models** (no transcription; verified by DER/JER rather than WER): - + | Family | Variants | Available capabilities | Docs | | --- | --- | --- | --- | | Streaming Sortformer Diarizer 4spk v2.1 | `diar_streaming_sortformer_4spk-v2.1` | diarize, streaming | [docs/models/diar_streaming_sortformer_4spk-v2.1.md](docs/models/diar_streaming_sortformer_4spk-v2.1.md) | diff --git a/bindings/python/README.md b/bindings/python/README.md index c24be3453..9422adf99 100644 --- a/bindings/python/README.md +++ b/bindings/python/README.md @@ -70,6 +70,24 @@ Long transcriptions can be cancelled from another thread with `session.cancel()` — the run raises `Aborted` with the partial transcript on `exc.partial_result` (same for `OutputTruncated`). +### Diarization + +Models whose `model.roles` include `Role.DIARIZE` (Sortformer) answer "who +spoke when" through a diarize session. `run()` returns `SpeakerSegment` +rows (`speaker_id` in `1..model.diarize_info.max_speakers`), grouped by +speaker. Locking, `Busy`, `cancel()` and `close()` work as on `Session`. A +model without the role raises `UnsupportedRole`, as `model.session()` and +`model.capabilities` do on a model without `Role.ASR` (Sortformer serves only +`Role.DIARIZE`). + +```python +with model.diarize_session() as diarizer: + turns = diarizer.run(pcm, family=transcribe_cpp.SortformerDiarizeOptions( + preset="very_high_latency")) + for turn in turns: + print(turn.speaker_id, turn.t0_ms, turn.t1_ms) +``` + ## Backends `Model(backend=...)` applies a backend policy (`"auto"` uses the best @@ -120,9 +138,10 @@ TRANSCRIBE_LIBRARY=../../build-shared/src/libtranscribe.dylib \ ## Notes -- One run/stream at a time per `Model` in 0.x: sessions share the model's - compute backend, so serialize runs across sessions (or load one model per - worker). See the `Model` docstring. +- One compute call at a time per `Model`: the binding serializes calls across + all sessions of a model with a model-wide lock (load one model per worker + for parallelism). While a stream is active, other runs and stream begins on + that model raise `transcribe_cpp.Busy`. See the `Model` docstring. - Import package: `transcribe_cpp` - Distribution: `transcribe-cpp` - License: MIT diff --git a/bindings/python/src/transcribe_cpp/__init__.py b/bindings/python/src/transcribe_cpp/__init__.py index 5b431cb2e..a77f20225 100644 --- a/bindings/python/src/transcribe_cpp/__init__.py +++ b/bindings/python/src/transcribe_cpp/__init__.py @@ -16,16 +16,20 @@ Long-running native calls (model load, run) release the GIL — ctypes does this for every foreign call — so other Python threads make progress during inference. +Compute locking and ``Busy`` rules: see ``Model``. """ from __future__ import annotations +import collections import ctypes +import enum import os import threading import weakref +from contextlib import contextmanager from dataclasses import dataclass, field -from typing import Literal, Optional, Sequence, Union +from typing import Literal, Optional, Sequence, TypeVar, Union from . import _abi, _generated from ._library import _base_version, artifact_dir, load_library, selected_provider @@ -33,6 +37,7 @@ AbiError, Aborted, BackendError, + Busy, InputTooLong, InvalidArgument, ModelFileNotFound, @@ -43,6 +48,7 @@ OutputTruncated, TranscribeError, UnsupportedRequest, + UnsupportedRole, exception_for_status, raise_for_status, ) @@ -70,6 +76,9 @@ "transcribe", "Model", "Session", + "DiarizeSession", + "DiarizeInfo", + "Role", "Result", "Segment", "SpeakerSegment", @@ -86,7 +95,7 @@ "MoonshineStreamingOptions", "ParakeetStreamOptions", "ParakeetBufferedStreamOptions", - "SortformerStreamOptions", + "SortformerDiarizeOptions", "VoxtralRealtimeStreamOptions", "Backend", "SortformerPreset", @@ -106,8 +115,10 @@ "OutOfMemory", "BackendError", "UnsupportedRequest", + "UnsupportedRole", "AbiError", "Aborted", + "Busy", "InputTooLong", "OutputTruncated", "OutputRepetition", @@ -243,6 +254,7 @@ _EXT_SLOTS = { "run": _generated.TRANSCRIBE_EXT_SLOT_RUN, "stream": _generated.TRANSCRIBE_EXT_SLOT_STREAM, + "diarize_run": _generated.TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, } _FEATURES = { "initial_prompt": _generated.TRANSCRIBE_FEATURE_INITIAL_PROMPT, @@ -391,12 +403,14 @@ def set_log_callback(handler) -> None: contract for transcribe_log_set). The handler may be invoked from ggml worker threads, so it must be thread-safe — route to the ``logging`` module or a queue rather than doing heavy work inline. Levels mirror - TRANSCRIBE_LOG_LEVEL_* (1=info, 2=warn, 3=error, 4=debug).""" + TRANSCRIBE_LOG_LEVEL_* (1=info, 2=warn, 3=error, 4=debug). + + The handler may fire while its model's compute lock is held, so it must + not call compute APIs on that model (raises, or deadlocks off-thread).""" global _log_handler, _log_trampoline _log_handler = handler if _log_trampoline is None: - # Install the trampoline once; later calls just swap the Python handler - # behind it, so transcribe_log_set is never re-invoked after threads run. + # Installed once; later calls only swap the Python handler behind it. _log_trampoline = _LOG_CFUNC(_log_thunk) _lib.transcribe_log_set(_log_trampoline, None) @@ -542,6 +556,20 @@ class Capabilities: translate_target_languages: tuple[str, ...] +class Role(enum.Enum): + """What a model serves (``Model.roles``): ASR is ``Model.session()``, + DIARIZE is ``Model.diarize_session()``.""" + + ASR = _generated.TRANSCRIBE_ROLE_ASR + DIARIZE = _generated.TRANSCRIBE_ROLE_DIARIZE + + +@dataclass(frozen=True) +class DiarizeInfo: + sample_rate: int + max_speakers: int # speaker_id is in [1, max_speakers] + + @dataclass(frozen=True) class SessionLimits: """Effective per-session limits (model bound narrowed by session params). @@ -869,10 +897,10 @@ def _apply(self, ext) -> None: ext.min_decode_interval_ms = self.min_decode_interval_ms -class SortformerStreamOptions(FamilyExtension): - """Sortformer streaming operating-point options (run slot). +class SortformerDiarizeOptions(FamilyExtension): + """Sortformer operating-point options for ``DiarizeSession.run()`` + (diarize-run slot). - Sortformer is a diarizer: a run produces speaker segments, no text. ``preset`` selects the latency / accuracy trade-off from the model's published menu; ``"default"`` keeps the GGUF-shipped checkpoint configuration. ``"very_high_latency"`` (~30 s lookahead) is the @@ -880,10 +908,10 @@ class SortformerStreamOptions(FamilyExtension): real-time point and costs substantially more compute per audio second.""" - _slot = "run" - _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM - _struct = _generated.transcribe_sortformer_stream_ext - _init = "transcribe_sortformer_stream_ext_init" + _slot = "diarize_run" + _kind = _generated.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE + _struct = _generated.transcribe_sortformer_diarize_ext + _init = "transcribe_sortformer_diarize_ext_init" _presets = { "default": _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT, @@ -906,14 +934,34 @@ def _apply(self, ext) -> None: # --- high-level handles --------------------------------------------------- -class Model: - """A loaded model. Sharing it across threads for queries and session - creation is safe; it must outlive its sessions. +class _StreamLease: + """One stream's claim on its model's stream lease (identity token for + ``Model._stream_owner``). ``handle`` is the session handle it began on.""" + + __slots__ = ("handle",) + + def __init__(self, handle: ctypes.c_void_p): + self.handle = handle + - Known 0.x limitation: at most one run/stream may be IN FLIGHT across all - sessions of a model at a time — sessions share the model's compute - backend, so overlapping runs race. Run sessions serially (a pool behind - a lock is fine), or load one Model per worker for true parallelism. +class Model: + """A loaded model. Sharing it across threads is safe: queries + (``arch``, ``capabilities``, ``supports`` ...) and session creation run + concurrently. It must outlive its sessions (``close()`` enforces that). + + The native library allows one compute call in flight across ALL sessions + of a model, so ``Session.run()`` / ``run_batch()`` / ``stream()``, + ``Stream.feed()`` / ``finalize()`` / ``reset()`` and ``DiarizeSession.run()`` + hold a model-wide lock for the native call and its copy-out. Calls from + other threads wait; a ``cancel()`` on a queued call is kept, and it + aborts at its first poll after acquiring the lock. Load one Model per + worker for parallelism. An active stream holds the model's stream + lease from ``stream()`` until it is finalized, reset, fails, is + garbage-collected or its session/model is closed. Meanwhile ``run()``, + ``run_batch()``, ``stream()`` and diarize ``run()`` on any session raise + :class:`Busy` once they hold the lock; the stream's own calls proceed. + ``close()`` and GC never wait for the lock: the handle is closed at once + and the native free runs right after any in-flight call. ``backend="auto"`` (the default) picks the best available device. The ``TRANSCRIBE_BACKEND`` environment variable overrides that *default* — @@ -929,6 +977,7 @@ def __init__(self, path: str | os.PathLike, *, # derived session is gone (use-after-free otherwise). Created before # the load call so the failure path of __del__ finds it. self._sessions = weakref.WeakSet() + self._init_compute_state() backend_source = "backend" if backend == "auto" and os.environ.get("TRANSCRIBE_BACKEND"): backend = os.environ["TRANSCRIBE_BACKEND"] # type: ignore[assignment] @@ -956,6 +1005,122 @@ def _h(self) -> ctypes.c_void_p: raise TranscribeError("model is closed") return self._handle + # --- compute exclusion ------------------------------------------------- + # One plain Lock per model serializes native compute calls and copy-out. + # Not reentrant: the log callback can fire inside a native call, so a + # nested compute is detected via _compute_owner and raises instead. + # Native frees never wait: they queue FIFO on _deferred and the lock + # holder runs them before releasing. close() queues every live session's + # free before the model's, from _live (strong records, not the weak + # _sessions, which the cycle collector clears before finalizers run). + # _free_order makes "pop a session's _live record + queue its free" one + # step against close() queuing the model's free, so a session closed on + # another thread can never land behind the model free. An RLock: a GC + # finalizer can close a session on the thread already holding it. + # _stream_owner (the active stream's _StreamLease) is written only under it. + + def _init_compute_state(self) -> None: + self._compute_lock = threading.Lock() + self._compute_owner: Optional[int] = None # thread ident while held + self._deferred: collections.deque = collections.deque() + self._stream_owner: Optional[_StreamLease] = None + # id(handle) -> (handle, native free name, pin) per open session. + self._live: dict = {} + self._free_order = threading.RLock() + + @contextmanager + def _exclusive(self, kind: str, busy: Optional[str] = None): + """Hold the compute lock for one call of *kind* and check the model + is open; run the native call and its copy-out inside the block. With + *busy* set, raise Busy(*busy*) if a stream holds the lease.""" + me = threading.get_ident() + if self._compute_owner == me: + raise TranscribeError( + f"cannot start {kind}: this thread is already inside a " + "compute call on this model (a re-entrant call from a " + "callback?)") + try: + # `with`: no gap between acquire and the release guard in which + # a KeyboardInterrupt could leak the lock. + with self._compute_lock: + try: + self._compute_owner = me + self._h # raises if the model is closed + if busy is not None and self._stream_owner is not None: + raise Busy(busy) + yield + finally: + try: + self._drain_deferred_locked() + finally: + # Owner stays set while draining: a native free can + # log, and the re-entrancy check must still see this + # thread. + self._compute_owner = None + finally: + self._try_drain() + + def _drain_deferred_locked(self) -> None: + """Run queued native frees. Caller holds the compute lock.""" + while self._deferred: + try: + self._deferred.popleft()() + except Exception: + pass # a native free cannot meaningfully fail + + def _try_drain(self) -> None: + """Run queued frees now if the lock is free, else leave them to the + holder. Never blocks; loops so a free queued during release is run.""" + while self._deferred: + got = False + try: + # Acquired inside the try, so the release guard covers all but + # the call's own return (no non-blocking `with` exists). + got = self._compute_lock.acquire(False) + if not got: + return + self._compute_owner = threading.get_ident() + self._drain_deferred_locked() + finally: + if got: + self._compute_owner = None + self._compute_lock.release() + + def _free_or_defer(self, free) -> None: + """Run *free* behind any in-flight compute call without waiting for + it. *free* must not reference the Session (it may already be gone).""" + self._deferred.append(free) + self._try_drain() + + def _track(self, handle: ctypes.c_void_p, free_fn: str, pin=None) -> None: + """Record an open session's native handle (and *pin*, kept alive + until its free) so close() frees it even if its wrapper is gone.""" + self._live[id(handle)] = (handle, free_fn, pin) + + def _free_session(self, handle: ctypes.c_void_p) -> None: + """Free a tracked session behind any in-flight call. The record is + popped first, so the session's own close() and the model's close() + free it exactly once, whichever runs first.""" + with self._free_order: + self._queue_session_free(handle) + self._try_drain() + + def _queue_session_free(self, handle: ctypes.c_void_p) -> None: + """Pop *handle*'s record and queue its free. Caller holds _free_order.""" + record = self._live.pop(id(handle), None) + if record is None: + return + handle, free_fn, pin = record + + def free(_pin=pin): + getattr(_lib, free_fn)(handle) + # Release this session's stream lease only after the free, in the + # same locked step, so a queued run cannot start before it. + if getattr(self._stream_owner, "handle", None) is handle: + self._stream_owner = None + + self._deferred.append(free) + @property def arch(self) -> str: return _decode(_lib.transcribe_model_arch_string(self._h)) @@ -1043,15 +1208,46 @@ def session(self, *, n_threads: int = 0, kv_type: KVType = "auto", n_ctx: int = 0) -> "Session": return Session(self, n_threads=n_threads, kv_type=kv_type, n_ctx=n_ctx) + @property + def roles(self) -> frozenset[Role]: + mask = _lib.transcribe_model_roles(self._h) + return frozenset(r for r in Role if mask & r.value) + + @property + def diarize_info(self) -> DiarizeInfo: + """Raises :class:`UnsupportedRole` without the DIARIZE role.""" + info = _generated.transcribe_diarize_info() + _lib.transcribe_diarize_info_init(_byref(info)) + _check(_lib.transcribe_diarize_get_info(self._h, _byref(info)), + "reading diarize info") + return DiarizeInfo(sample_rate=info.sample_rate, + max_speakers=info.max_speakers) + + def diarize_session(self, *, n_threads: int = 0) -> "DiarizeSession": + """Raises :class:`UnsupportedRole` without the DIARIZE role.""" + return DiarizeSession(self, n_threads=n_threads) + def close(self) -> None: """Free the model. Any session still open on it is closed first — the C contract forbids freeing a model before its sessions, so this - keeps explicit close()/context-manager exit safe in any order.""" - if getattr(self, "_handle", None) is not None: - for session in list(getattr(self, "_sessions", ()) or ()): - session.close() - _lib.transcribe_model_free(self._handle) - self._handle = None + keeps explicit close()/context-manager exit safe in any order. + Never waits for the compute lock (see above).""" + handle = getattr(self, "_handle", None) + if handle is None: + return + # Closed from here on: a call queued on the lock sees it once it gets + # the lock; an in-flight call already captured its handles. + self._handle = None + for session in list(getattr(self, "_sessions", ()) or ()): + session.close() + # One step under _free_order: every session free still pending + # (wrappers already collected, or a close() racing on another thread) + # is queued ahead of the model's. + with self._free_order: + for record in list(self._live.values()): + self._queue_session_free(record[0]) + self._deferred.append(lambda: _lib.transcribe_model_free(handle)) + self._try_drain() def __enter__(self) -> "Model": return self @@ -1066,41 +1262,28 @@ def __del__(self): pass -class Session: - """A single transcription context bound to one Model. Not thread-safe. +_S = TypeVar("_S", bound="_SessionBase") - Sessions of one model must also not RUN concurrently with each other in - 0.x (they share the model's compute backend — see the Model docstring); - interleave or serialize their runs, or use one Model per worker.""" - def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", - n_ctx: int = 0): - self._model = model # keep the model alive for the session's lifetime - params = _SessionParams() - _lib.transcribe_session_params_init(_byref(params)) - params.n_threads = n_threads - params.kv_type = _enum(_KV_TYPES, kv_type, "kv_type") - params.n_ctx = n_ctx +class _SessionBase: + """Handle, cancellation and close plumbing shared by Session and + DiarizeSession. Subclasses name their native free in ``_free_fn``.""" - handle = ctypes.c_void_p() - status = _lib.transcribe_session_init(model._h, _byref(params), _byref(handle)) - _check(status, "opening session") - if not handle.value: - raise TranscribeError("session init returned a null handle") - self._handle = handle + _free_fn = "" - # Cancellation: an abort callback polled at chunk/decode boundaries - # returns True once cancel() is requested, so another thread can abort an - # in-flight run/stream. Bind the Event (not self) into the trampoline to - # avoid a reference cycle that would defer __del__. + def _arm_abort(self, set_cb) -> None: + """Install the abort callback (polled at chunk/decode boundaries; + True once cancel() is requested) and register with the model. Binds + the Event, not self, into the trampoline to avoid a reference cycle.""" self._cancel = threading.Event() - _event = self._cancel - self._abort_trampoline = _ABORT_CFUNC(lambda _ud: _event.is_set()) - _lib.transcribe_set_abort_callback(self._handle, self._abort_trampoline, None) - - # Registered last, once the session is fully constructed: Model.close() - # closes any session still tracked here before freeing the model. - model._sessions.add(self) + event = self._cancel + self._abort_trampoline = _ABORT_CFUNC(lambda _ud: event.is_set()) + # The record pins the trampoline the native session points at. + self._model._track(self._handle, self._free_fn, self._abort_trampoline) + set_cb(self._handle, self._abort_trampoline, None) + # Registered last, once fully constructed: Model.close() closes any + # session still tracked here before freeing the model. + self._model._sessions.add(self) @property def _h(self) -> ctypes.c_void_p: @@ -1108,9 +1291,17 @@ def _h(self) -> ctypes.c_void_p: raise TranscribeError("session is closed") return self._handle + def _attach_family(self, family: "FamilyExtension | None", slot: str, params) -> None: + """Resolve *family* for *slot* and point ``params.family`` at it (the + params struct keeps the built extension alive).""" + if family is not None: + ext = self._resolve_family(family, slot) + params.family = ctypes.cast(_byref(ext), ctypes.POINTER(_generated.transcribe_ext)) + params._family_keepalive = ext + def _resolve_family(self, family: "FamilyExtension", slot: str): """Validate + probe a family extension and return its built ctypes - struct (which the caller keeps alive and points params.family at).""" + struct.""" if not isinstance(family, FamilyExtension): raise InvalidArgument("family must be a FamilyExtension instance") if family._slot != slot: @@ -1124,6 +1315,63 @@ def _resolve_family(self, family: "FamilyExtension", slot: str): f"{slot!r} slot") return family._build() + def cancel(self) -> None: + """Request cancellation of an in-flight call from another thread: it + aborts at the next chunk/decode boundary and raises ``Aborted`` (with + ``partial_result`` where available). The flag is cleared when the next + run / run_batch / stream / diarize run starts, before it waits for the + lock, so a cancel() while queued still aborts it. Lock-free.""" + self._cancel.set() + + def close(self) -> None: + """Free the session. Idempotent; never waits for the compute lock + (see ``Model``).""" + handle = getattr(self, "_handle", None) + if handle is None: + return + self._handle = None + self._model._free_session(handle) # a no-op if the model freed it + + def __enter__(self: _S) -> _S: + return self + + def __exit__(self, *exc) -> None: + self.close() + + def __del__(self): + try: + self.close() + except Exception: + pass + + +class Session(_SessionBase): + """A single transcription context bound to one Model. Compute locking and + ``Busy`` rules: see ``Model``. Reads (``limits``, ``was_aborted``, Stream + ``text()`` ...) are unlocked; don't read a session while another thread + computes on it.""" + + _free_fn = "transcribe_session_free" + + def __init__(self, model: Model, *, n_threads: int = 0, kv_type: KVType = "auto", + n_ctx: int = 0): + self._model = model # keep the model alive for the session's lifetime + # Lease of the newest Stream begun here; an older Stream is superseded. + self._stream_lease: Optional[_StreamLease] = None + params = _SessionParams() + _lib.transcribe_session_params_init(_byref(params)) + params.n_threads = n_threads + params.kv_type = _enum(_KV_TYPES, kv_type, "kv_type") + params.n_ctx = n_ctx + + handle = ctypes.c_void_p() + status = _lib.transcribe_session_init(model._h, _byref(params), _byref(handle)) + _check(status, "opening session") + if not handle.value: + raise TranscribeError("session init returned a null handle") + self._handle = handle + self._arm_abort(_lib.transcribe_set_abort_callback) + def run(self, pcm: PCMLike, *, task: Task = "transcribe", language: str | None = None, target_language: str | None = None, @@ -1157,26 +1405,31 @@ def run(self, pcm: PCMLike, *, task: Task = "transcribe", On ``Aborted`` (via :meth:`cancel`) and ``OutputTruncated`` (including its ``OutputRepetition`` subclass) the partial transcript is preserved - and attached to the exception as ``partial_result``.""" + and attached to the exception as ``partial_result``. + + Raises :class:`Busy` if a stream is active on this model.""" + # Cleared BEFORE waiting for the compute lock, so a cancel() issued + # while this call is queued behind another still aborts it. self._cancel.clear() array, n_samples = _pcm_to_carray(pcm) params = _build_run_params(task, language, target_language, timestamps, keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt, prefix) - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - try: - _check(_lib.transcribe_run(self._h, array, n_samples, _byref(params)), - "transcribe_run") - except (Aborted, OutputTruncated) as exc: - # The C API preserves the partial transcript on the session for - # these statuses (OutputRepetition included, as an OutputTruncated - # subclass); surface it rather than discard it. - exc.partial_result = self._materialize() - raise - return self._materialize() + with self._model._exclusive( + "run", busy="a stream is active on this model; " + "finish or drop it before run()"): + self._attach_family(family, "run", params) + h = self._h # captured under the lock; close() defers its free + try: + _check(_lib.transcribe_run(h, array, n_samples, _byref(params)), + "transcribe_run") + except (Aborted, OutputTruncated) as exc: + # The C API preserves the partial transcript on the session for + # these statuses (OutputRepetition included, as an OutputTruncated + # subclass); surface it rather than discard it. + exc.partial_result = self._materialize(h) + raise + return self._materialize(h) def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", language: str | None = None, @@ -1210,8 +1463,10 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", directly (the ``asyncio.gather`` convention). ``vocabulary`` / ``prompt`` apply to every utterance (see :meth:`run`); - a transcript prefix is per-utterance and so is not accepted here.""" - self._cancel.clear() + a transcript prefix is per-utterance and so is not accepted here. + + Raises :class:`Busy` if a stream is active on this model.""" + self._cancel.clear() # before the lock wait, as in run() pcms = list(pcms) if not pcms: raise InvalidArgument("run_batch requires at least one PCM buffer") @@ -1228,44 +1483,46 @@ def run_batch(self, pcms: Sequence[PCMLike], *, task: Task = "transcribe", params = _build_run_params(task, language, target_language, timestamps, keep_special_tags, spec_k_drafts, diarize, pnc, itn, vocabulary, prompt) - ext = self._resolve_family(family, "run") if family is not None else None - if ext is not None: - params.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - batch_abort = None - try: - _check( - _lib.transcribe_run_batch(self._h, ptrs, counts, len(pcms), _byref(params)), - "transcribe_run_batch", - ) - except Aborted as exc: - # Cancellation surfaces at the BATCH level, but the native side - # pads the per-utterance result set so clips completed before the - # abort survive. Fall through to the per-utterance loop, which - # restores utterance context instead of discarding that work. - batch_abort = exc - - out: list = [] - first_exc = None - for i in range(_lib.transcribe_batch_n_results(self._h)): - status = _lib.transcribe_batch_status(self._h, i) - if status == 0: - out.append(self._materialize(i)) - continue - exc = exception_for_status(status, _status_string(status), - f"utterance {i} in batch") - exc.utterance_index = i - if isinstance(exc, (Aborted, OutputTruncated)): - # Attach the partial transcript only when the native layer - # actually snapshotted one for this slot — some batch paths - # record failed slots as empty, and None is more honest than - # a confidently empty Result. - partial = self._materialize(i) - if partial.text or partial.tokens or partial.segments: - exc.partial_result = partial - out.append(exc) - if first_exc is None: - first_exc = exc + # The native call AND the per-utterance copy-out run under the lock. + with self._model._exclusive( + "run_batch", busy="a stream is active on this model; " + "finish or drop it before run_batch()"): + self._attach_family(family, "run", params) + h = self._h # captured under the lock; close() defers its free + batch_abort = None + try: + _check( + _lib.transcribe_run_batch(h, ptrs, counts, len(pcms), _byref(params)), + "transcribe_run_batch", + ) + except Aborted as exc: + # Cancellation surfaces at the BATCH level, but the native side + # pads the per-utterance result set so clips completed before the + # abort survive. Fall through to the per-utterance loop, which + # restores utterance context instead of discarding that work. + batch_abort = exc + + out: list = [] + first_exc = None + for i in range(_lib.transcribe_batch_n_results(h)): + status = _lib.transcribe_batch_status(h, i) + if status == 0: + out.append(self._materialize(h, i)) + continue + exc = exception_for_status(status, _status_string(status), + f"utterance {i} in batch") + exc.utterance_index = i + if isinstance(exc, (Aborted, OutputTruncated)): + # Attach the partial transcript only when the native layer + # actually snapshotted one for this slot — some batch paths + # record failed slots as empty, and None is more honest than + # a confidently empty Result. + partial = self._materialize(h, i) + if partial.text or partial.tokens or partial.segments: + exc.partial_result = partial + out.append(exc) + if first_exc is None: + first_exc = exc if first_exc is None and batch_abort is not None: # Anomalous: an abort with no per-utterance trace. Re-raise loud # (even under return_exceptions) rather than swallow a cancel. @@ -1290,11 +1547,13 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, Requires a model whose capabilities advertise ``supports_streaming``; otherwise raises NotImplementedByModel. ``family`` is an optional family-specific stream extension (e.g. MoonshineStreamingOptions). The - session is single-threaded and runs at most one stream at a time. Use - the Stream as a context manager so it is reset when you are done. - ``vocabulary`` and ``prompt`` are as in :meth:`run`; ``task="instruct"`` - raises.""" - self._cancel.clear() + session is single-threaded. Use the Stream as a context manager so it + is reset when you are done. ``vocabulary`` and ``prompt`` are as in + :meth:`run`; ``task="instruct"`` raises. + + Takes the model's stream lease (see ``Model``); raises :class:`Busy` + if another stream is active on this model.""" + self._cancel.clear() # before the lock wait, as in run() # spec_k_drafts is an offline-decode knob; streaming always uses the # family default (-1). run_params = _build_run_params(task, language, target_language, timestamps, @@ -1304,26 +1563,34 @@ def stream(self, *, task: Task = "transcribe", language: str | None = None, _lib.transcribe_stream_params_init(_byref(sp)) sp.commit_policy = _enum(_COMMIT_POLICIES, commit_policy, "commit_policy") sp.stable_prefix_agreement_n = stable_prefix_agreement_n - ext = self._resolve_family(family, "stream") if family is not None else None - if ext is not None: - sp.family = ctypes.cast( - _byref(ext), ctypes.POINTER(_generated.transcribe_ext)) - _check( - _lib.transcribe_stream_begin(self._h, _byref(run_params), _byref(sp)), - "transcribe_stream_begin", - ) - # The C contract says everything passed to begin may be freed once it - # returns (strings are copied into session-owned storage). The Stream - # still pins the params structs until reset() as defense in depth — - # it costs nothing and keeps the binding safe even against an older - # or out-of-tree native library that predates that contract. - return Stream(self, _keepalive=(run_params, sp, ext)) - - def _materialize(self, utt: int | None = None) -> Result: - """Copy out one result. utt is None for the single-result accessors, or - an utterance index for the batch accessors (index 0 aliases the single - result after a plain run, so both paths share this code).""" - h = self._h + with self._model._exclusive( + "stream_begin", busy="a stream is already active on this model"): + self._attach_family(family, "stream", sp) + h = self._h + _check( + _lib.transcribe_stream_begin(h, _byref(run_params), _byref(sp)), + "transcribe_stream_begin", + ) + # Claim the lease only now that the native begin succeeded. + lease = _StreamLease(h) + self._model._stream_owner = lease + self._stream_lease = lease + # The C contract says everything passed to begin may be freed once + # it returns (strings are copied into session-owned storage). The + # Stream still pins the params structs until reset() as defense in + # depth — it costs nothing and keeps the binding safe even against + # an older or out-of-tree native library that predates that + # contract. Built under the lock, so the lease always has a Stream. + return Stream(self, _keepalive=(run_params, sp), _lease=lease) + + def _materialize(self, h: ctypes.c_void_p, utt: int | None = None) -> Result: + """Copy out one result from session handle *h*. The run paths call + this under the compute lock with *h* captured there, so a concurrent + close() cannot pull it away mid-copy; ``Stream.snapshot()`` is an + unlocked read (see ``Session``). utt is None for the single-result + accessors, or an utterance index for the batch accessors (index 0 + aliases the single result after a plain run, so both paths share + this code).""" if utt is None: n_seg = lambda: _lib.transcribe_n_segments(h) get_seg = lambda j, out: _lib.transcribe_get_segment(h, j, out) @@ -1412,56 +1679,51 @@ def limits(self) -> SessionLimits: max_kv_bytes=lm.max_kv_bytes, ) - def cancel(self) -> None: - """Request cancellation of an in-flight run/stream from another thread. - The active call aborts at the next chunk/decode boundary and raises - ``Aborted`` (with ``partial_result`` attached); ``was_aborted`` then - reports True. The flag is cleared at the start of the next - run/stream.""" - self._cancel.set() - @property def was_aborted(self) -> bool: """True if the most recent call was ended by cancel().""" return bool(_lib.transcribe_was_aborted(self._h)) - def close(self) -> None: - if getattr(self, "_handle", None) is not None: - _lib.transcribe_session_free(self._handle) - self._handle = None - - def __enter__(self) -> "Session": - return self - - def __exit__(self, *exc) -> None: - self.close() - - def __del__(self): - try: - self.close() - except Exception: - pass - class Stream: """An active streaming transcription on a Session. Feed audio in chunks with ``feed()``, read the committed/tentative text with ``text()``, and call ``finalize()`` when the audio ends. The session is - returned to idle on context-manager exit (or ``reset()``).""" + returned to idle on context-manager exit (or ``reset()``). - def __init__(self, session: Session, *, _keepalive=None): + While active it holds the model's stream lease (see ``Model``); a feed + rejected before the model sees it (e.g. NaN/Inf samples) keeps it.""" + + def __init__(self, session: Session, *, _lease: _StreamLease, _keepalive=None): self._session = session # keep the session (and its model) alive # Pins the ctypes params structs (and any family ext) passed to # transcribe_stream_begin until reset(). The native library copies # what it needs at begin; this is belt-and-braces for the FFI layer. self._keepalive = _keepalive self._active = True + self._lease = _lease # holds the lease while model._stream_owner is this + + def _release_lease_locked(self) -> None: + """Release the model's stream lease if THIS stream still holds it + (never a later stream's). Caller holds the compute lock.""" + model = self._session._model + if model._stream_owner is self._lease: + model._stream_owner = None + + def _release_lease_if_ended_locked(self, h: ctypes.c_void_p) -> None: + """After a feed/finalize: release the lease once the native stream + is no longer ACTIVE (FINISHED or FAILED). A call rejected before the + model saw it (e.g. NaN/Inf samples) leaves it ACTIVE and keeps it.""" + if _lib.transcribe_stream_get_state(h) != _generated.TRANSCRIBE_STREAM_ACTIVE: + self._release_lease_locked() @property def _h(self) -> ctypes.c_void_p: if not self._active: raise TranscribeError("stream has been reset") + if self._session._stream_lease is not self._lease: + raise TranscribeError("stream superseded by a newer stream on this session") return self._session._h def feed(self, pcm: PCMLike) -> StreamUpdate: @@ -1469,17 +1731,23 @@ def feed(self, pcm: PCMLike) -> StreamUpdate: array, n_samples = _pcm_to_carray(pcm) update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) - _check(_lib.transcribe_stream_feed(self._h, array, n_samples, _byref(update)), - "transcribe_stream_feed") - return _stream_update_from(update) + with self._session._model._exclusive("stream_feed"): # no busy check + h = self._h + status = _lib.transcribe_stream_feed(h, array, n_samples, _byref(update)) + self._release_lease_if_ended_locked(h) + _check(status, "transcribe_stream_feed") + return _stream_update_from(update) def finalize(self) -> StreamUpdate: """Signal end of audio and flush the final hypothesis.""" update = _StreamUpdate() _lib.transcribe_stream_update_init(_byref(update)) - _check(_lib.transcribe_stream_finalize(self._h, _byref(update)), - "transcribe_stream_finalize") - return _stream_update_from(update) + with self._session._model._exclusive("stream_finalize"): + h = self._h + status = _lib.transcribe_stream_finalize(h, _byref(update)) + self._release_lease_if_ended_locked(h) + _check(status, "transcribe_stream_finalize") + return _stream_update_from(update) def text(self) -> StreamText: """Current committed / tentative / full text views (owned copies).""" @@ -1495,8 +1763,7 @@ def text(self) -> StreamText: def snapshot(self) -> Result: """Full structured snapshot of the current hypothesis (owned copies).""" - _ = self._h # validate that this stream has not been reset - return self._session._materialize() + return self._session._materialize(self._h) @property def state(self) -> str: @@ -1520,17 +1787,105 @@ def last_status(self) -> TranscribeError | None: return exception_for_status(status, _status_string(status), "stream") def reset(self) -> None: - """Return the session to idle, discarding stream state. Idempotent.""" + """Return the session to idle, discarding stream state, and release + the model's stream lease. Idempotent; a no-op once a newer stream has + begun on the session.""" if self._active: - _lib.transcribe_stream_reset(self._session._h) + with self._session._model._exclusive("stream_reset"): + if self._session._stream_lease is self._lease: + _lib.transcribe_stream_reset(self._session._h) + self._release_lease_locked() self._active = False self._keepalive = None def __enter__(self) -> "Stream": return self - def __exit__(self, *exc) -> None: - self.reset() + def __exit__(self, exc_type, *exc) -> None: + try: + self.reset() + except TranscribeError: + # e.g. the session or model was closed in the block: never mask + # the exception that is already propagating. + if exc_type is None: + raise + + def __del__(self): + # Abandoned while holding the lease: reset and release it behind any + # in-flight call (never waits). "Not the owner" is final; re-checked locked. + try: + lease, model = self._lease, self._session._model + if model._stream_owner is not lease: + return + + def reset_and_release(): + if model._stream_owner is lease: # so the session is still open + _lib.transcribe_stream_reset(lease.handle) + model._stream_owner = None + + model._free_or_defer(reset_and_release) + except Exception: + pass + + +class DiarizeSession(_SessionBase): + """A diarization (who spoke when) context on a model with the DIARIZE + role. Compute locking and ``Busy`` rules: see ``Model``.""" + + _free_fn = "transcribe_diarize_session_free" + + def __init__(self, model: Model, *, n_threads: int = 0): + self._model = model # keep the model alive for the session's lifetime + params = _generated.transcribe_diarize_session_params() + _lib.transcribe_diarize_session_params_init(_byref(params)) + params.n_threads = n_threads + + handle = ctypes.c_void_p() + _check(_lib.transcribe_diarize_session_init(model._h, _byref(params), _byref(handle)), + "opening diarize session") + if not handle.value: + raise TranscribeError("diarize session init returned a null handle") + self._handle = handle + self._arm_abort(_lib.transcribe_diarize_set_abort_callback) + + def run(self, pcm: PCMLike, *, + family: FamilyExtension | None = None) -> list[SpeakerSegment]: + """Diarize one recording (16 kHz mono float32 PCM) and return its + speaker turns: grouped by speaker, time-ordered within a speaker, + possibly overlapping across speakers. ``family`` is an optional + diarize-run extension (e.g. SortformerDiarizeOptions). + + Raises :class:`Aborted` after :meth:`cancel`, and :class:`Busy` if a + stream is active on this model.""" + self._cancel.clear() # before the lock wait, as in Session.run() + array, n_samples = _pcm_to_carray(pcm) + params = _generated.transcribe_diarize_params() + _lib.transcribe_diarize_params_init(_byref(params)) + with self._model._exclusive( + "diarize_run", busy="a stream is active on this model; " + "finish or drop it before diarize run()"): + self._attach_family(family, "diarize_run", params) + h = self._h # captured under the lock; close() defers its free + _check(_lib.transcribe_diarize_run(h, array, n_samples, _byref(params)), + "transcribe_diarize_run") + rows = [] + for i in range(_lib.transcribe_diarize_n_segments(h)): + s = _SpeakerSegment() + _lib.transcribe_speaker_segment_init(_byref(s)) + _check(_lib.transcribe_diarize_get_segment(h, i, _byref(s)), + "transcribe_diarize_get_segment") + rows.append(_speaker_segment_from(s)) + return rows + + @property + def timings(self) -> Timings: + """Load time plus the last run's mel / encode time. Not locked, like + ``Session.limits``.""" + tm = _Timings() + _lib.transcribe_timings_init(_byref(tm)) + _check(_lib.transcribe_diarize_get_timings(self._h, _byref(tm)), + "transcribe_diarize_get_timings") + return _timings_from(tm) def transcribe( diff --git a/bindings/python/src/transcribe_cpp/_generated.py b/bindings/python/src/transcribe_cpp/_generated.py index bf84e145c..f908877ae 100644 --- a/bindings/python/src/transcribe_cpp/_generated.py +++ b/bindings/python/src/transcribe_cpp/_generated.py @@ -13,7 +13,7 @@ # Stable digest of the ABI surface below (structs, enums, macros, layout, # prototypes). A native provider package echoes this back so the API # package can reject an ABI-mismatched provider before dlopen. -PUBLIC_HEADER_HASH = "57b1af43650f195d" +PUBLIC_HEADER_HASH = "bd3273dabb25a1fe" # === enum constants === TRANSCRIBE_OK = 0 @@ -36,6 +36,7 @@ TRANSCRIBE_ERR_INPUT_TOO_LONG = 17 TRANSCRIBE_ERR_OUTPUT_TRUNCATED = 18 TRANSCRIBE_ERR_OUTPUT_REPETITION = 19 +TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20 TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0 TRANSCRIBE_ABI_SESSION_PARAMS = 1 TRANSCRIBE_ABI_RUN_PARAMS = 2 @@ -52,6 +53,9 @@ TRANSCRIBE_ABI_DEVICE_INFO = 13 TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14 TRANSCRIBE_ABI_BACKEND_INIT_PARAMS = 15 +TRANSCRIBE_ABI_DIARIZE_INFO = 16 +TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 17 +TRANSCRIBE_ABI_DIARIZE_PARAMS = 18 TRANSCRIBE_LOG_LEVEL_NONE = 0 TRANSCRIBE_LOG_LEVEL_INFO = 1 TRANSCRIBE_LOG_LEVEL_WARN = 2 @@ -80,6 +84,7 @@ TRANSCRIBE_DIARIZE_MODE_ON = 2 TRANSCRIBE_EXT_SLOT_RUN = 0 TRANSCRIBE_EXT_SLOT_STREAM = 1 +TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2 TRANSCRIBE_BACKEND_AUTO = 0 TRANSCRIBE_BACKEND_CPU = 1 TRANSCRIBE_BACKEND_METAL = 2 @@ -91,6 +96,8 @@ TRANSCRIBE_DEVICE_TYPE_GPU = 1 TRANSCRIBE_DEVICE_TYPE_IGPU = 2 TRANSCRIBE_DEVICE_TYPE_ACCEL = 3 +TRANSCRIBE_ROLE_ASR = 1 +TRANSCRIBE_ROLE_DIARIZE = 2 TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0 TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1 TRANSCRIBE_FEATURE_LONG_FORM = 2 @@ -127,7 +134,7 @@ TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM = 1414746957 TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584 TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912 -TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635 +TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163 TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710 TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319 @@ -164,13 +171,19 @@ class transcribe_token(_c.Structure): pass class transcribe_speaker_segment(_c.Structure): pass +class transcribe_diarize_info(_c.Structure): + pass +class transcribe_diarize_session_params(_c.Structure): + pass +class transcribe_diarize_params(_c.Structure): + pass class transcribe_moonshine_streaming_stream_ext(_c.Structure): pass class transcribe_parakeet_stream_ext(_c.Structure): pass class transcribe_parakeet_buffered_stream_ext(_c.Structure): pass -class transcribe_sortformer_stream_ext(_c.Structure): +class transcribe_sortformer_diarize_ext(_c.Structure): pass class transcribe_voxtral_realtime_stream_ext(_c.Structure): pass @@ -195,10 +208,13 @@ class transcribe_whisper_chunk_trace(_c.Structure): transcribe_word._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("seg_index", _c.c_int), ("first_token", _c.c_int), ("n_tokens", _c.c_int), ("text", _c.c_char_p)] transcribe_token._fields_ = [("struct_size", _c.c_uint64), ("id", _c.c_int), ("p", _c.c_float), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("seg_index", _c.c_int), ("word_index", _c.c_int), ("text", _c.c_char_p)] transcribe_speaker_segment._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("speaker_id", _c.c_int32), ("p", _c.c_float)] +transcribe_diarize_info._fields_ = [("struct_size", _c.c_uint64), ("sample_rate", _c.c_int32), ("max_speakers", _c.c_int32)] +transcribe_diarize_session_params._fields_ = [("struct_size", _c.c_uint64), ("n_threads", _c.c_int32)] +transcribe_diarize_params._fields_ = [("struct_size", _c.c_uint64), ("family", _c.POINTER(transcribe_ext))] transcribe_moonshine_streaming_stream_ext._fields_ = [("ext", transcribe_ext), ("min_decode_interval_ms", _c.c_int32)] transcribe_parakeet_stream_ext._fields_ = [("ext", transcribe_ext), ("att_context_right", _c.c_int32)] transcribe_parakeet_buffered_stream_ext._fields_ = [("ext", transcribe_ext), ("left_ms", _c.c_int32), ("chunk_ms", _c.c_int32), ("right_ms", _c.c_int32)] -transcribe_sortformer_stream_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] +transcribe_sortformer_diarize_ext._fields_ = [("ext", transcribe_ext), ("preset", _c.c_int)] transcribe_voxtral_realtime_stream_ext._fields_ = [("ext", transcribe_ext), ("num_delay_tokens", _c.c_int32), ("min_decode_interval_ms", _c.c_int32)] transcribe_whisper_run_ext._fields_ = [("ext", transcribe_ext), ("initial_prompt", _c.c_char_p), ("prompt_tokens", _c.POINTER(_c.c_int32)), ("n_prompt_tokens", _c.c_size_t), ("prompt_condition", _c.c_int), ("condition_on_prev_tokens", _c.c_bool), ("max_prev_context_tokens", _c.c_int32), ("temperature", _c.c_float), ("temperature_inc", _c.c_float), ("compression_ratio_thold", _c.c_float), ("logprob_thold", _c.c_float), ("no_speech_thold", _c.c_float), ("seed", _c.c_uint32), ("max_initial_timestamp", _c.c_float)] transcribe_whisper_chunk_trace._fields_ = [("struct_size", _c.c_uint64), ("t0_ms", _c.c_int64), ("t1_ms", _c.c_int64), ("temperature_used", _c.c_float), ("compression_ratio", _c.c_float), ("avg_logprob", _c.c_float), ("no_speech_prob", _c.c_float), ("no_speech_triggered", _c.c_bool), ("n_fallbacks", _c.c_int32)] @@ -222,6 +238,9 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_word': 7, 'transcribe_token': 8, 'transcribe_speaker_segment': 14, + 'transcribe_diarize_info': 16, + 'transcribe_diarize_session_params': 17, + 'transcribe_diarize_params': 18, } # C-compiler layout captured at generation (for offset self-check). @@ -242,10 +261,13 @@ class transcribe_whisper_chunk_trace(_c.Structure): 'transcribe_word': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'seg_index': 24, 'first_token': 28, 'n_tokens': 32, 'text': 40}}, 'transcribe_token': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 'id': 8, 'p': 12, 't0_ms': 16, 't1_ms': 24, 'seg_index': 32, 'word_index': 36, 'text': 40}}, 'transcribe_speaker_segment': {'size': 32, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'speaker_id': 24, 'p': 28}}, + 'transcribe_diarize_info': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'sample_rate': 8, 'max_speakers': 12}}, + 'transcribe_diarize_session_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'n_threads': 8}}, + 'transcribe_diarize_params': {'size': 16, 'align': 8, 'offsets': {'struct_size': 0, 'family': 8}}, 'transcribe_moonshine_streaming_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'min_decode_interval_ms': 16}}, 'transcribe_parakeet_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'att_context_right': 16}}, 'transcribe_parakeet_buffered_stream_ext': {'size': 32, 'align': 8, 'offsets': {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24}}, - 'transcribe_sortformer_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, + 'transcribe_sortformer_diarize_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'preset': 16}}, 'transcribe_voxtral_realtime_stream_ext': {'size': 24, 'align': 8, 'offsets': {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20}}, 'transcribe_whisper_run_ext': {'size': 80, 'align': 8, 'offsets': {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76}}, 'transcribe_whisper_chunk_trace': {'size': 48, 'align': 8, 'offsets': {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'temperature_used': 24, 'compression_ratio': 28, 'avg_logprob': 32, 'no_speech_prob': 36, 'no_speech_triggered': 40, 'n_fallbacks': 44}}, @@ -308,6 +330,28 @@ def configure(lib): lib.transcribe_device_get_info.argtypes = [_c.c_void_p, _c.POINTER(transcribe_device_info)] lib.transcribe_device_info_init.restype = None lib.transcribe_device_info_init.argtypes = [_c.POINTER(transcribe_device_info)] + lib.transcribe_diarize_get_info.restype = _c.c_int + lib.transcribe_diarize_get_info.argtypes = [_c.c_void_p, _c.POINTER(transcribe_diarize_info)] + lib.transcribe_diarize_get_segment.restype = _c.c_int + lib.transcribe_diarize_get_segment.argtypes = [_c.c_void_p, _c.c_int, _c.POINTER(transcribe_speaker_segment)] + lib.transcribe_diarize_get_timings.restype = _c.c_int + lib.transcribe_diarize_get_timings.argtypes = [_c.c_void_p, _c.POINTER(transcribe_timings)] + lib.transcribe_diarize_info_init.restype = None + lib.transcribe_diarize_info_init.argtypes = [_c.POINTER(transcribe_diarize_info)] + lib.transcribe_diarize_n_segments.restype = _c.c_int + lib.transcribe_diarize_n_segments.argtypes = [_c.c_void_p] + lib.transcribe_diarize_params_init.restype = None + lib.transcribe_diarize_params_init.argtypes = [_c.POINTER(transcribe_diarize_params)] + lib.transcribe_diarize_run.restype = _c.c_int + lib.transcribe_diarize_run.argtypes = [_c.c_void_p, _c.POINTER(_c.c_float), _c.c_int, _c.POINTER(transcribe_diarize_params)] + lib.transcribe_diarize_session_free.restype = None + lib.transcribe_diarize_session_free.argtypes = [_c.c_void_p] + lib.transcribe_diarize_session_init.restype = _c.c_int + lib.transcribe_diarize_session_init.argtypes = [_c.c_void_p, _c.POINTER(transcribe_diarize_session_params), _c.POINTER(_c.c_void_p)] + lib.transcribe_diarize_session_params_init.restype = None + lib.transcribe_diarize_session_params_init.argtypes = [_c.POINTER(transcribe_diarize_session_params)] + lib.transcribe_diarize_set_abort_callback.restype = None + lib.transcribe_diarize_set_abort_callback.argtypes = [_c.c_void_p, _c.CFUNCTYPE(_c.c_bool, _c.c_void_p), _c.c_void_p] lib.transcribe_ext_check.restype = _c.c_int lib.transcribe_ext_check.argtypes = [_c.POINTER(transcribe_ext), _c.c_uint32, _c.c_uint64] lib.transcribe_full_text.restype = _c.c_char_p @@ -354,6 +398,8 @@ def configure(lib): lib.transcribe_model_load_params_init.argtypes = [_c.POINTER(transcribe_model_load_params)] lib.transcribe_model_meta_val_str.restype = _c.c_char_p lib.transcribe_model_meta_val_str.argtypes = [_c.c_void_p, _c.c_char_p] + lib.transcribe_model_roles.restype = _c.c_uint32 + lib.transcribe_model_roles.argtypes = [_c.c_void_p] lib.transcribe_model_supports.restype = _c.c_bool lib.transcribe_model_supports.argtypes = [_c.c_void_p, _c.c_int] lib.transcribe_model_variant_string.restype = _c.c_char_p @@ -402,8 +448,8 @@ def configure(lib): lib.transcribe_session_params_init.argtypes = [_c.POINTER(transcribe_session_params)] lib.transcribe_set_abort_callback.restype = None lib.transcribe_set_abort_callback.argtypes = [_c.c_void_p, _c.CFUNCTYPE(_c.c_bool, _c.c_void_p), _c.c_void_p] - lib.transcribe_sortformer_stream_ext_init.restype = None - lib.transcribe_sortformer_stream_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_stream_ext)] + lib.transcribe_sortformer_diarize_ext_init.restype = None + lib.transcribe_sortformer_diarize_ext_init.argtypes = [_c.POINTER(transcribe_sortformer_diarize_ext)] lib.transcribe_speaker_segment_init.restype = None lib.transcribe_speaker_segment_init.argtypes = [_c.POINTER(transcribe_speaker_segment)] lib.transcribe_status_string.restype = _c.c_char_p diff --git a/bindings/python/src/transcribe_cpp/errors.py b/bindings/python/src/transcribe_cpp/errors.py index c92e3f16f..236e94c67 100644 --- a/bindings/python/src/transcribe_cpp/errors.py +++ b/bindings/python/src/transcribe_cpp/errors.py @@ -29,6 +29,7 @@ TRANSCRIBE_ERR_UNSUPPORTED_ITN as ERR_UNSUPPORTED_ITN, TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE as ERR_UNSUPPORTED_LANGUAGE, TRANSCRIBE_ERR_UNSUPPORTED_PNC as ERR_UNSUPPORTED_PNC, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE as ERR_UNSUPPORTED_ROLE, TRANSCRIBE_ERR_UNSUPPORTED_TASK as ERR_UNSUPPORTED_TASK, TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS as ERR_UNSUPPORTED_TIMESTAMPS, TRANSCRIBE_ERR_UNSUPPORTED_VARIANT as ERR_UNSUPPORTED_VARIANT, @@ -83,6 +84,13 @@ class UnsupportedRequest(TranscribeError): """Task / language / timestamp granularity the model does not support.""" +class UnsupportedRole(TranscribeError): + """The model does not serve the role the call needs, e.g. opening a + transcription session on, or reading the (ASR) capabilities of, a model + without the ASR role. A property of the loaded model, not of the request + options, so retrying with different run options cannot succeed.""" + + class AbiError(TranscribeError): """Caller-owned struct layout did not match the library (struct_size).""" @@ -91,6 +99,12 @@ class InputTooLong(TranscribeError): pass +class Busy(TranscribeError): + """A stream is active on this model, so the call was refused instead of + started (see ``Model``). Finalize or reset the stream first, or use one + Model per worker. Raised by the binding, so ``status`` is 0.""" + + class Aborted(TranscribeError): """The run/stream was ended by ``Session.cancel()``. @@ -145,6 +159,7 @@ class OutputRepetition(OutputTruncated): ERR_INPUT_TOO_LONG: InputTooLong, ERR_OUTPUT_TRUNCATED: OutputTruncated, ERR_OUTPUT_REPETITION: OutputRepetition, + ERR_UNSUPPORTED_ROLE: UnsupportedRole, } diff --git a/bindings/python/tests/conftest.py b/bindings/python/tests/conftest.py index aeb0036df..392ef2935 100644 --- a/bindings/python/tests/conftest.py +++ b/bindings/python/tests/conftest.py @@ -59,6 +59,11 @@ REPO / "models/Voxtral-Mini-4B-Realtime-2602/Voxtral-Mini-4B-Realtime-2602-Q4_K_M.gguf" ) +SORTFORMER_MODEL = ( + REPO + / "models/diar_streaming_sortformer_4spk-v2.1" + / "diar_streaming_sortformer_4spk-v2.1-F32.gguf" +) PNC_MODEL = REPO / "models/canary-180m-flash/canary-180m-flash-Q8_0.gguf" ITN_MODEL = REPO / "models/SenseVoiceSmall/SenseVoiceSmall-Q8_0.gguf" @@ -161,6 +166,12 @@ def voxtral_model_path() -> Path: return _family_model("TRANSCRIBE_SMOKE_VOXTRAL_MODEL", VOXTRAL_MODEL) +@pytest.fixture(scope="session") +def sortformer_model_path() -> Path: + """Sortformer diarizer (serves only the DIARIZE role).""" + return _family_model("TRANSCRIBE_SMOKE_SORTFORMER_MODEL", SORTFORMER_MODEL) + + @pytest.fixture(scope="session") def pnc_model_path() -> Path: """Canary model whose generic PNC run parameter changes the prompt.""" diff --git a/bindings/python/tests/test_abi.py b/bindings/python/tests/test_abi.py index 3ec807c16..894dddd7e 100644 --- a/bindings/python/tests/test_abi.py +++ b/bindings/python/tests/test_abi.py @@ -66,9 +66,9 @@ def test_base_version_strips_suffixes(version, base): def test_status_constants_discovered(): - # transcribe_status has OK + 18 error codes as of this writing; never let + # transcribe_status has OK + 20 error codes as of this writing; never let # this collapse to a near-empty set that would make the checks vacuous. - assert len(_STATUS_NAMES) >= 19 + assert len(_STATUS_NAMES) >= 21 def test_every_generated_status_is_aliased_and_mapped(): diff --git a/bindings/python/tests/test_compute_lock.py b/bindings/python/tests/test_compute_lock.py new file mode 100644 index 000000000..ef65ddf8a --- /dev/null +++ b/bindings/python/tests/test_compute_lock.py @@ -0,0 +1,573 @@ +"""Model-wide compute lock and stream lease (see ``Model``), with NO model: +Model/Session objects wrap fake handles and the native entry points are +replaced by Python probes. The fixture closes every fake before monkeypatch +restores the real functions, so a fake handle never reaches the library. +""" + +from __future__ import annotations + +import ctypes +import gc +import itertools +import threading +import time +import weakref +from types import SimpleNamespace + +import pytest + +import transcribe_cpp as t +from transcribe_cpp import _generated + +TIMEOUT = 10.0 # generous upper bound for "must not deadlock" joins + + +def _in_thread(fn, *args): + """Run fn in a daemon thread; returns (thread, box) where box collects the + return value or the exception.""" + box: dict = {} + + def target(): + try: + box["value"] = fn(*args) + except BaseException as exc: # surfaced by the test, never swallowed + box["error"] = exc + + th = threading.Thread(target=target, daemon=True) + th.start() + return th, box + + +def _join(th, what): + th.join(TIMEOUT) + assert not th.is_alive(), f"{what} did not finish (deadlock?)" + + +@pytest.fixture +def fake(monkeypatch): + ids = itertools.count(0x1000, 0x10) + frees: list = [] # ("session" | "model", handle) in native-free order + made: list = [] + + monkeypatch.setattr(t._lib, "transcribe_session_free", + lambda h: frees.append(("session", h.value))) + monkeypatch.setattr(t._lib, "transcribe_model_free", + lambda h: frees.append(("model", h.value))) + monkeypatch.setattr(t._lib, "transcribe_diarize_session_free", + lambda h: frees.append(("diarize", h.value))) + resets: list = [] # session handles transcribe_stream_reset was given + monkeypatch.setattr(t._lib, "transcribe_stream_reset", + lambda h: resets.append(h.value)) + state = {"value": _generated.TRANSCRIBE_STREAM_ACTIVE} + monkeypatch.setattr(t._lib, "transcribe_stream_get_state", + lambda h: state["value"]) + # The copy-out reads many native accessors; stub it to report which + # handle it was given (proves the in-flight call kept its handle). + monkeypatch.setattr(t.Session, "_materialize", + lambda self, h, utt=None: ("result", h.value, utt)) + + def model(): + m = t.Model.__new__(t.Model) + m._sessions = weakref.WeakSet() + m._init_compute_state() # the same init Model.__init__ runs + m._handle = ctypes.c_void_p(next(ids)) + made.append(m) + return m + + def session(m, cls=t.Session, track=True): + s = cls.__new__(cls) + s._model = m + s._handle = ctypes.c_void_p(next(ids)) + s._cancel = threading.Event() + s._abort_trampoline = None + m._track(s._handle, cls._free_fn) # as _arm_abort does + m._sessions.add(s) + if track: + made.append(s) + return s + + yield SimpleNamespace(model=model, session=session, + frees=frees, resets=resets, state=state) + + for obj in reversed(made): + obj.close() + made.clear() + gc.collect() + + +PCM = [0.0] * 160 + + +class _LockSpy: + """Wraps a model's compute lock and signals when a thread starts a + BLOCKING acquire (i.e. is parked waiting for it). The non-blocking + acquires of the deferred-free path do not count. Install it while the + real lock is held: release() is looked up on the attribute at call + time, so the holder releases the same underlying lock.""" + + def __init__(self, model): + self._lock = model._compute_lock + self.waiting = threading.Event() + model._compute_lock = self + + def acquire(self, blocking=True, timeout=-1): + if blocking: + self.waiting.set() + return self._lock.acquire(blocking, timeout) + + def release(self): + self._lock.release() + + def __enter__(self): + return self.acquire() + + def __exit__(self, *exc): + self.release() + + def locked(self): + return self._lock.locked() + + +class _Gate: + """A native probe that parks inside the 'native call' until released.""" + + def __init__(self): + self.entered = threading.Event() + self.release = threading.Event() + + def __call__(self, *args): + self.entered.set() + assert self.release.wait(TIMEOUT), "gate never released" + return 0 + + +# --- serialization ------------------------------------------------------------ + + +def test_calls_on_two_sessions_never_overlap(fake, monkeypatch): + m = fake.model() + sessions = [fake.session(m) for _ in range(3)] + + guard = threading.Lock() + state = {"active": 0, "max": 0, "calls": 0} + + def probe(*args): + with guard: + state["active"] += 1 + state["calls"] += 1 + state["max"] = max(state["max"], state["active"]) + time.sleep(0.02) # widen the window an unlocked overlap would hit + with guard: + state["active"] -= 1 + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + + start = threading.Barrier(3) + n = 5 + + def runs(session): + start.wait(TIMEOUT) + return [session.run(PCM) for _ in range(n)] + + workers = [_in_thread(runs, s) for s in sessions] + for th, _ in workers: + _join(th, "worker") + assert state["calls"] == 3 * n + assert state["max"] == 1, "compute calls on one model overlapped" + for s, (_, box) in zip(sessions, workers): + assert box.get("value") == [("result", s._handle.value, None)] * n, box + + +def test_every_compute_site_holds_the_lock(fake, monkeypatch): + m = fake.model() + s = fake.session(m) + d = fake.session(m, cls=t.DiarizeSession) + seen: list = [] + + def holds(name, ret=0): + def probe(*args): + seen.append((name, m._compute_lock.locked() + and m._compute_owner == threading.get_ident())) + return ret + return probe + + for name in ("transcribe_run", "transcribe_run_batch", + "transcribe_stream_begin", "transcribe_stream_feed", + "transcribe_stream_finalize", "transcribe_stream_reset", + "transcribe_batch_status", "transcribe_diarize_run", + "transcribe_diarize_n_segments"): + monkeypatch.setattr(t._lib, name, holds(name)) + monkeypatch.setattr(t._lib, "transcribe_batch_n_results", + holds("transcribe_batch_n_results", ret=1)) + real_materialize = t.Session._materialize + + def materialize(self, h, utt=None): + seen.append(("copy-out", m._compute_lock.locked())) + return real_materialize(self, h, utt) + + monkeypatch.setattr(t.Session, "_materialize", materialize) + + s.run(PCM) + assert s.run_batch([PCM]) == [("result", s._handle.value, 0)] + stream = s.stream() + stream.feed(PCM) + stream.finalize() + stream.reset() + assert d.run(PCM) == [] + + names = [n for n, _ in seen] + for site in ("transcribe_run", "transcribe_run_batch", + "transcribe_batch_n_results", "transcribe_batch_status", + "transcribe_stream_begin", "transcribe_stream_feed", + "transcribe_stream_finalize", "transcribe_stream_reset", + "transcribe_diarize_run", "transcribe_diarize_n_segments", + "copy-out"): + assert site in names, f"{site} never reached" + assert all(ok for _, ok in seen), [n for n, ok in seen if not ok] + assert not m._compute_lock.locked() + + +# --- cancellation ------------------------------------------------------------- + + +def test_cancel_while_queued_is_kept(fake, monkeypatch): + # cancel() is lock-free and the flag is cleared BEFORE the lock wait, so + # a cancel issued while a call is queued is still set when it starts. + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + gate = _Gate() + flag_at_start: list = [] + + def probe(h, *rest): + if h.value == s1._handle.value: + return gate(h) + flag_at_start.append(s2._cancel.is_set()) + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, _ = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + b, _ = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) # B cleared its flag and is parked + s2.cancel() # must not block although the lock is held + finally: + gate.release.set() + _join(a, "first run") + _join(b, "queued run") + assert flag_at_start == [True] + + +# --- close / GC during an in-flight call ------------------------------------------ + + +def test_session_close_during_in_flight_call_defers_free(fake, monkeypatch): + m = fake.model() + s = fake.session(m) + handle = s._handle.value + gate = _Gate() + monkeypatch.setattr(t._lib, "transcribe_run", gate) + try: + a, a_box = _in_thread(s.run, PCM) + assert gate.entered.wait(TIMEOUT) + closer, _ = _in_thread(s.close) + _join(closer, "session.close() during an in-flight call") + assert fake.frees == [], "session freed under an in-flight call" + finally: + gate.release.set() + _join(a, "in-flight run") + # The in-flight call finished its copy-out on the handle it captured... + assert a_box.get("value") == ("result", handle, None), a_box + # ...and the deferred free ran exactly once, after it. + assert fake.frees == [("session", handle)] + with pytest.raises(t.TranscribeError, match="closed"): + s.run(PCM) + + +def test_model_close_during_in_flight_call_defers_frees(fake, monkeypatch): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + h1, h2, hm = s1._handle.value, s2._handle.value, m._handle.value + gate = _Gate() + + def probe(h, *rest): + return gate(h) if h.value == h1 else 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + try: + a, a_box = _in_thread(s1.run, PCM) + assert gate.entered.wait(TIMEOUT) + spy = _LockSpy(m) + queued, q_box = _in_thread(s2.run, PCM) + assert spy.waiting.wait(TIMEOUT) # parked on the lock before close + closer, _ = _in_thread(m.close) + _join(closer, "model.close() during an in-flight call") + assert fake.frees == [], "freed under an in-flight call" + finally: + gate.release.set() + _join(a, "in-flight run") + _join(queued, "queued run") + assert a_box.get("value") == ("result", h1, None), a_box + # The queued call sees the closed model once it gets the lock. + assert isinstance(q_box.get("error"), t.TranscribeError), q_box + assert "closed" in str(q_box["error"]) + # Sessions first (either order), then the model; each exactly once. + assert sorted(fake.frees[:2]) == sorted([("session", h1), ("session", h2)]) + assert fake.frees[2:] == [("model", hm)] + + +def test_model_close_racing_session_close_frees_session_first(fake): + # Model.close() on another thread lands between a session close() popping + # its _live record and queuing its free: the model free must still queue + # behind the session's. + m = fake.model() + s = fake.session(m) + hs, hm = s._handle.value, m._handle.value + closer: dict = {} + + class _RacingLive(dict): + def pop(self, key, *default): + record = super().pop(key, *default) + if not closer: + closer["thread"], _ = _in_thread(m.close) + closer["thread"].join(0.2) # let it run as far as it can + return record + + m._live = _RacingLive(m._live) + s.close() + _join(closer["thread"], "model.close() racing session.close()") + assert fake.frees == [("session", hs), ("model", hm)] + + +def test_gc_on_holder_thread_defers_free(fake, monkeypatch): + # A finalizer can run on the thread that holds the lock (GC mid copy-out). + # It must neither deadlock nor free under the in-flight call. + m = fake.model() + s = fake.session(m) + doomed: list = [] + during: dict = {} + + def probe(*args): + victim = fake.session(m, track=False) + victim._cycle = victim # only the cyclic GC can reclaim it + doomed.append(victim._handle.value) + del victim + gc.collect() # runs Session.__del__ -> close() on THIS thread + during["frees"] = list(fake.frees) + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + th, box = _in_thread(s.run, PCM) + _join(th, "run with GC on the holder thread") + assert "error" not in box, box + assert doomed, "probe never ran" + victim = ("session", doomed[0]) + assert victim not in during["frees"], "freed while the lock was held" + assert fake.frees.count(victim) == 1 + + +def test_reentrant_compute_raises_instead_of_deadlocking(fake, monkeypatch): + # e.g. a log handler that calls run() from inside a native call. + m = fake.model() + s, other = fake.session(m), fake.session(m) + inner: dict = {} + + def probe(h, *rest): + if h.value == s._handle.value: + try: + other.run(PCM) + except t.TranscribeError as exc: + inner["error"] = exc + return 0 + + monkeypatch.setattr(t._lib, "transcribe_run", probe) + th, box = _in_thread(s.run, PCM) + _join(th, "re-entrant run") + assert "error" not in box, box + assert "re-entrant" in str(inner.get("error")), inner + assert not m._compute_lock.locked() + + +# --- stream lease --------------------------------------------------------------- + +RUN_BUSY = "a stream is active on this model; finish or drop it before run()" +BATCH_BUSY = ("a stream is active on this model; " + "finish or drop it before run_batch()") +BEGIN_BUSY = "a stream is already active on this model" + + +@pytest.fixture +def native(fake, monkeypatch): + """Probes for the native compute entry points. ``calls`` records + (name, session handle) for every native compute call that ran; + ``status`` sets what each one returns. A successful begin makes the + stream ACTIVE; finalize ends it (FINISHED, or FAILED on error).""" + calls: list = [] + status = {"run": 0, "begin": 0, "feed": 0, "finalize": 0} + ends = {"begin": (_generated.TRANSCRIBE_STREAM_ACTIVE, None), + "finalize": (_generated.TRANSCRIBE_STREAM_FINISHED, + _generated.TRANSCRIBE_STREAM_FAILED)} + + def probe(name, key): + def fn(h, *rest): + calls.append((name, h.value)) + ok, failed = ends.get(key, (None, None)) + new = ok if status[key] == 0 else failed + if new is not None: + fake.state["value"] = new + return status[key] + return fn + + monkeypatch.setattr(t._lib, "transcribe_run", probe("run", "run")) + monkeypatch.setattr(t._lib, "transcribe_run_batch", probe("run_batch", "run")) + monkeypatch.setattr(t._lib, "transcribe_batch_n_results", lambda h: 1) + monkeypatch.setattr(t._lib, "transcribe_batch_status", lambda h, i: 0) + monkeypatch.setattr(t._lib, "transcribe_stream_begin", probe("begin", "begin")) + monkeypatch.setattr(t._lib, "transcribe_stream_feed", probe("feed", "feed")) + monkeypatch.setattr(t._lib, "transcribe_stream_finalize", + probe("finalize", "finalize")) + return SimpleNamespace(calls=calls, status=status) + + +def _assert_busy(session, *, ran: list): + """Every lease-checked call on *session* raises Busy without reaching + native code.""" + before = list(ran) + with pytest.raises(t.Busy) as ei: + session.run(PCM) + assert str(ei.value) == RUN_BUSY + assert ei.value.status == 0 and isinstance(ei.value, t.TranscribeError) + with pytest.raises(t.Busy) as ei: + session.run_batch([PCM]) + assert str(ei.value) == BATCH_BUSY + with pytest.raises(t.Busy) as ei: + session.stream() + assert str(ei.value) == BEGIN_BUSY + assert ran == before, "a refused call reached native code" + assert not session._model._compute_lock.locked() + + +def _assert_free(session): + """The model is free again: a run on *session* goes through.""" + assert session.run(PCM) == ("result", session._handle.value, None) + + +def test_active_stream_refuses_sibling_and_same_session_calls(fake, native): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + stream = s1.stream() + _assert_busy(s2, ran=native.calls) # sibling session + _assert_busy(s1, ran=native.calls) # the stream's own session + # The lease holder itself feeds and finalizes without a busy check. + stream.feed(PCM) + stream.feed(PCM) + stream.finalize() + assert [n for n, _ in native.calls] == ["begin", "feed", "feed", "finalize"] + _assert_free(s2) + _assert_free(s1) + + +def test_failed_begin_takes_no_lease(fake, native): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + native.status["begin"] = _generated.TRANSCRIBE_ERR_NOT_IMPLEMENTED + with pytest.raises(t.NotImplementedByModel): + s1.stream() + assert m._stream_owner is None + _assert_free(s2) + + +@pytest.mark.parametrize("ending", [ + "finalize", "finalize_error", "reset", "gc", "session_close", +]) +def test_lease_released_when_stream_ends(fake, native, ending): + m = fake.model() + s1, s2 = fake.session(m), fake.session(m) + h1 = s1._handle.value + stream = s1.stream() + stream.feed(PCM) + _assert_busy(s2, ran=native.calls) + + if ending == "finalize": + stream.finalize() + elif ending == "finalize_error": + native.status["finalize"] = _generated.TRANSCRIBE_ERR_BACKEND + with pytest.raises(t.BackendError): + stream.finalize() + elif ending == "reset": + stream.reset() + assert fake.resets == [h1] + elif ending == "gc": + stream._cycle = stream # only the cyclic GC can reclaim it + del stream + gc.collect() + assert fake.resets == [h1], "GC'd active stream was not reset once" + elif ending == "session_close": + s1.close() + assert fake.frees == [("session", h1)] + + assert m._stream_owner is None + _assert_free(s2) + if ending == "session_close": + # The stream outlived its session: dropping it now must not reset + # a freed native session. + del stream + gc.collect() + assert fake.resets == [] + + +def test_ended_stream_never_clears_a_later_lease(fake, native): + m = fake.model() + sa, sb, sc = fake.session(m), fake.session(m), fake.session(m) + a = sa.stream() + a.finalize() + b = sb.stream() # B now holds the lease + _assert_busy(sc, ran=native.calls) + + # A, already ended, fails a feed (not ACTIVE), finalizes again, resets + # and is garbage-collected: none of that may touch B's lease. + native.status["feed"] = _generated.TRANSCRIBE_ERR_INVALID_ARG + fake.state["value"] = _generated.TRANSCRIBE_STREAM_FINISHED + with pytest.raises(t.InvalidArgument): + a.feed(PCM) + _assert_busy(sc, ran=native.calls) + a.finalize() + _assert_busy(sc, ran=native.calls) + a.reset() + _assert_busy(sc, ran=native.calls) + a._cycle = a + del a + gc.collect() + _assert_busy(sc, ran=native.calls) + sa.close() # A's session closing leaves B's lease alone too + _assert_busy(sc, ran=native.calls) + + native.status["feed"] = 0 + fake.state["value"] = _generated.TRANSCRIBE_STREAM_ACTIVE + b.feed(PCM) + b.finalize() + _assert_free(sc) + + +# --- DiarizeSession ------------------------------------------------------------- + +DIARIZE_BUSY = ("a stream is active on this model; " + "finish or drop it before diarize run()") + + +def test_diarize_run_busy_while_stream_active(fake, native, monkeypatch): + m = fake.model() + s, d = fake.session(m), fake.session(m, cls=t.DiarizeSession) + calls: list = [] + monkeypatch.setattr(t._lib, "transcribe_diarize_run", + lambda *a: calls.append("run") or 0) + monkeypatch.setattr(t._lib, "transcribe_diarize_n_segments", lambda h: 0) + stream = s.stream() + with pytest.raises(t.Busy) as ei: + d.run(PCM) + assert str(ei.value) == DIARIZE_BUSY + assert calls == [] and not m._compute_lock.locked() + stream.finalize() + assert d.run(PCM) == [] and calls == ["run"] diff --git a/bindings/python/tests/test_diarize.py b/bindings/python/tests/test_diarize.py new file mode 100644 index 000000000..e5fe9deb6 --- /dev/null +++ b/bindings/python/tests/test_diarize.py @@ -0,0 +1,83 @@ +"""DIARIZE role against real models: roles, diarize info, DiarizeSession.run +and its Sortformer preset extension. The locking / Busy / close / cancel +rules are pinned model-free in test_compute_lock.py.""" + +from __future__ import annotations + +import gc +import math + +import pytest + +import transcribe_cpp as t +from conftest import SAMPLES, load_wav + + +@pytest.fixture(scope="module") +def mix_pcm(): + return load_wav(SAMPLES / "sortformer-2spk-mix.wav") + + +def _turns(rows): + return [(r.t0_ms, r.t1_ms, r.speaker_id) for r in rows] + + +def test_sortformer_roles_and_info(sortformer_model_path): + with t.Model(sortformer_model_path) as model: + assert model.roles == {t.Role.DIARIZE} + assert model.diarize_info == t.DiarizeInfo(sample_rate=16000, max_speakers=4) + + +def test_sortformer_rejects_asr(sortformer_model_path): + with t.Model(sortformer_model_path) as model: + with pytest.raises(t.UnsupportedRole): + model.capabilities + with pytest.raises(t.UnsupportedRole): + model.session() + with model.diarize_session(): # the DIARIZE role still opens + pass + + +def test_asr_only_model_rejects_diarize(model_path): + with t.Model(model_path) as model: + assert model.roles == {t.Role.ASR} + with pytest.raises(t.UnsupportedRole): + model.diarize_info + with pytest.raises(t.UnsupportedRole): + model.diarize_session() + + +def test_diarize_run_returns_turns_per_preset(sortformer_model_path, mix_pcm): + # Exact segments are pinned in C (tests/sortformer_diarize_unit.cpp); here + # only that rows come back well-formed and the preset reaches the run. + with t.Model(sortformer_model_path, backend="cpu") as model: + with model.diarize_session() as d: + default = d.run(mix_pcm) + low = d.run(mix_pcm, family=t.SortformerDiarizeOptions(preset="low_latency")) + timings = d.timings + assert default + for r in default: + assert 1 <= r.speaker_id <= 4 and r.t0_ms < r.t1_ms and math.isnan(r.p) + assert _turns(low) != _turns(default) + assert timings.encode_ms > 0 + + +def test_bad_preset_rejected(sortformer_model_path, mix_pcm): + class OutOfRange(t.SortformerDiarizeOptions): + _presets = {**t.SortformerDiarizeOptions._presets, "bogus": 99} + + with t.Model(sortformer_model_path) as model, model.diarize_session() as d: + with pytest.raises(t.InvalidArgument): + d.run(mix_pcm, family=OutOfRange(preset="bogus")) # type: ignore[arg-type] + with pytest.raises(t.InvalidArgument, match="slot"): + d.run(mix_pcm, family=t.WhisperRunOptions()) + assert d.run(mix_pcm) # still usable + + +def test_diarize_session_keeps_model_alive(sortformer_model_path, mix_pcm): + d = t.Model(sortformer_model_path).diarize_session() + gc.collect() + assert d.run(mix_pcm) + d.close() + with pytest.raises(t.TranscribeError, match="closed"): + d.run(mix_pcm) diff --git a/bindings/python/tests/test_errors.py b/bindings/python/tests/test_errors.py index 3aaee5d2b..1c7b64d71 100644 --- a/bindings/python/tests/test_errors.py +++ b/bindings/python/tests/test_errors.py @@ -42,6 +42,7 @@ def test_every_status_maps_to_documented_subclass(): errors.ERR_INPUT_TOO_LONG: t.InputTooLong, errors.ERR_OUTPUT_TRUNCATED: t.OutputTruncated, errors.ERR_OUTPUT_REPETITION: t.OutputRepetition, + errors.ERR_UNSUPPORTED_ROLE: t.UnsupportedRole, } # The mapping table covers every non-OK status the header defines, and # nothing else (a new C status must be mapped deliberately, not by @@ -73,6 +74,12 @@ def test_output_repetition_is_an_output_truncated(): assert exc.partial_result is None +def test_unsupported_role_is_its_own_type(): + # A role mismatch is a property of the loaded model, not of a run option: + # it must not be caught by an UnsupportedRequest handler. + assert not issubclass(t.UnsupportedRole, t.UnsupportedRequest) + + def test_exception_for_status_builds_without_raising(): exc = errors.exception_for_status(errors.ERR_ABORTED, "aborted", "run") assert isinstance(exc, t.Aborted) diff --git a/bindings/python/tests/test_family_ext.py b/bindings/python/tests/test_family_ext.py index 3b0a433ee..62cd8a2a0 100644 --- a/bindings/python/tests/test_family_ext.py +++ b/bindings/python/tests/test_family_ext.py @@ -22,7 +22,7 @@ t.MoonshineStreamingOptions, t.ParakeetStreamOptions, t.ParakeetBufferedStreamOptions, - t.SortformerStreamOptions, + t.SortformerDiarizeOptions, t.VoxtralRealtimeStreamOptions, ] @@ -35,7 +35,7 @@ def test_build_stamps_kind_and_size(cls): ext = cls()._build() assert ext.ext.kind == cls._kind assert ext.ext.size == ctypes.sizeof(cls._struct) - assert cls._slot in ("run", "stream") + assert cls._slot in t._EXT_SLOTS @pytest.mark.parametrize("cls", ALL_OPTION_TYPES) @@ -74,19 +74,19 @@ def test_parakeet_buffered_partial_overrides(): assert built.right_ms == fresh.right_ms -def test_sortformer_preset_maps_to_enum_value(): - built = t.SortformerStreamOptions(preset="very_high_latency")._build() +def test_sortformer_unknown_preset_rejected(): + with pytest.raises(ValueError, match="preset"): + t.SortformerDiarizeOptions(preset="ultra_low_latency") # type: ignore[arg-type] + + +def test_sortformer_diarize_preset_maps_to_enum_value(): + built = t.SortformerDiarizeOptions(preset="very_high_latency")._build() assert built.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY # None keeps the init default (DEFAULT = GGUF-shipped cfg). - default = t.SortformerStreamOptions()._build() + default = t.SortformerDiarizeOptions()._build() assert default.preset == _generated.TRANSCRIBE_SORTFORMER_PRESET_DEFAULT -def test_sortformer_unknown_preset_rejected(): - with pytest.raises(ValueError, match="preset"): - t.SortformerStreamOptions(preset="ultra_low_latency") # type: ignore[arg-type] - - # --- model-gated: resolve_family validation + a real extension run ---------- diff --git a/bindings/python/tests/test_lifetime.py b/bindings/python/tests/test_lifetime.py index e094b5f91..fb6dffbef 100644 --- a/bindings/python/tests/test_lifetime.py +++ b/bindings/python/tests/test_lifetime.py @@ -76,18 +76,14 @@ def test_gc_order_session_keeps_model_alive(model_path, audio_pcm): # --- multi-session use of one model ------------------------------------------ # -# SUPPORTED in 0.x: many sessions on one model, run serially (from any -# threads). NOT yet supported: overlapping runs across sessions of one model -# — they share the model's compute backend and some family state, so -# concurrent runs race (observed: corrupted whisper decodes on CPU, -# command-buffer failures on Metal). The xfail below pins the limitation so -# the day a per-session backend architecture lands, the XPASS flags it for -# promotion to a hard test. +# Sessions of one model share its compute backend, so the binding serializes +# their calls with a model-wide lock (see Model). The model-free lock and +# stream lease tests live in test_compute_lock.py. def test_serial_sessions_across_threads(model_path, audio_pcm): - # Two threads, each with its own session, runs SERIALIZED by a lock: - # the supported session-pool pattern. + # Two threads, each with its own session, runs serialized by a caller + # lock: must keep working (it nests with the binding's lock). import threading lock = threading.Lock() @@ -114,12 +110,11 @@ def worker(tag: str, model: "t.Model") -> None: assert all("country" in text for text in results.values()), results -# Run INSIDE a subprocess: the known failure mode is NATIVE (corrupted -# decodes, Metal command-buffer failures, potentially a crash or a wedged -# thread). xfail can absorb a Python assert but not a segfault of the test -# runner, and a wedged native thread must never reach interpreter teardown -# in the CI process (freeing the session under an in-flight native call is -# a use-after-free). os._exit on the wedge path skips teardown on purpose. +# Run INSIDE a subprocess: a regression of the compute lock fails NATIVELY +# (corrupted decodes, Metal command-buffer failures, potentially a crash or a +# wedged thread). A segfault must fail this test, not kill the runner, and a +# wedged native thread must never reach interpreter teardown in the CI +# process. os._exit on the wedge path skips teardown on purpose. _CONCURRENT_RUNS_SNIPPET = """\ import array, os, sys, threading, wave @@ -136,7 +131,8 @@ def worker(tag: str, model: "t.Model") -> None: def worker(tag, model): try: with model.session() as session: - results[tag] = session.run(pcm).text.lower() + for k in range(2): # NO caller lock: the binding serializes + results[(tag, k)] = session.run(pcm).text.lower() except Exception as exc: errors.append((tag, repr(exc))) @@ -152,18 +148,14 @@ def worker(tag, model): os._exit(3) # never free handles under an in-flight native call assert not errors, errors -assert len(results) == 2 and all("country" in v for v in results.values()), results +assert len(results) == 4 and all("country" in v for v in results.values()), results print("CONCURRENT-OK") """ -@pytest.mark.xfail( - strict=False, - reason="known 0.x limitation: overlapping runs on sessions of one model " - "race on the shared backend/model state (see Model docstring and " - "notes/bindings-review-fixes.md); per-session backends planned", -) def test_concurrent_sessions_on_shared_model(model_path, audio_path): + # Unsynchronized callers on two sessions of one model wait for each + # other and both decode cleanly. import os import subprocess import sys diff --git a/bindings/python/tests/test_prompting.py b/bindings/python/tests/test_prompting.py index ca08cbc09..51e3ebd44 100644 --- a/bindings/python/tests/test_prompting.py +++ b/bindings/python/tests/test_prompting.py @@ -113,4 +113,4 @@ def test_control_token_literal_rejected(model_path, audio_pcm): first = session.run(audio_pcm) with pytest.raises(t.InvalidArgument): session.run(audio_pcm, prompt="hello <|endoftext|>") - assert session._materialize().text == first.text + assert session._materialize(session._h).text == first.text diff --git a/bindings/python/tests/test_streaming.py b/bindings/python/tests/test_streaming.py index 6c1695092..f26901a46 100644 --- a/bindings/python/tests/test_streaming.py +++ b/bindings/python/tests/test_streaming.py @@ -97,13 +97,16 @@ def test_cancellation_aborts_pending_feed(streaming_model_path, audio_pcm): # Pre-set the cancel flag, then feed: the abort callback fires at the # first poll inside the feed. (True cross-thread mid-run cancellation is # covered by TestCancellation in test_transcribe.py.) Must surface as - # the dedicated Aborted class, not a generic TranscribeError. + # the dedicated Aborted class, not a generic TranscribeError. The stream + # is then FAILED, so it releases the model's stream lease. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: session.cancel() with pytest.raises(t.Aborted): stream.feed(audio_pcm) assert session.was_aborted, "was_aborted should be True after cancel()" + assert stream.state == "failed" + sibling.run(audio_pcm[:16000]) # not Busy # --- stream lifecycle edges --------------------------------------------------- @@ -132,6 +135,31 @@ def test_stream_use_after_reset_rejected(streaming_model_path, audio_pcm): stream.snapshot() +def test_superseded_stream_cannot_touch_the_new_one(streaming_model_path, audio_pcm): + # A finalized Stream stays readable until the session begins another; then + # it is superseded: feed raises and reset is a no-op on the new stream. + with t.Model(streaming_model_path) as model, model.session() as session: + old = session.stream() + old.feed(audio_pcm[:16000]) + old.finalize() + assert old.state == "finished" + + new = session.stream() + new.feed(audio_pcm[:16000]) + revision = new.revision + with pytest.raises(t.TranscribeError, match="superseded"): + old.feed(audio_pcm[16000:32000]) + with pytest.raises(t.TranscribeError, match="superseded"): + old.text() + old.reset() + assert new.state == "active" + assert new.revision == revision + with model.session() as sibling, pytest.raises(t.Busy): + sibling.run(audio_pcm[:16000]) # the new stream still holds the lease + new.finalize() + new.reset() + + def test_stream_reset_idempotent_and_session_reusable( streaming_model_path, audio_pcm): with t.Model(streaming_model_path) as model, model.session() as session: @@ -151,10 +179,52 @@ def test_stream_reset_idempotent_and_session_reusable( def test_second_stream_while_active_rejected(streaming_model_path, audio_pcm): - # One stream at a time per session: the C dispatcher rejects a begin - # while ACTIVE, and the binding surfaces it as InvalidArgument. + # One active stream per MODEL: the stream lease refuses a second begin + # with Busy, on the same session and on a sibling, before native code. with t.Model(streaming_model_path) as model, model.session() as session: - with session.stream() as stream: + with model.session() as sibling, session.stream() as stream: stream.feed(audio_pcm[:16000]) - with pytest.raises(t.InvalidArgument): + with pytest.raises(t.Busy, match="already active"): session.stream() + with pytest.raises(t.Busy, match="already active"): + sibling.stream() + assert stream.state == "active" + + +def test_rejected_feed_keeps_stream_active(streaming_model_path, audio_pcm): + # Non-finite PCM is rejected BEFORE the family hook: the feed raises + # InvalidArgument but the native stream stays ACTIVE (only failures + # inside the hook move it to FAILED). The binding releases the compute + # lock on that error but KEEPS the stream lease: the stream stays + # usable and sibling calls stay Busy until it is finalized. + with t.Model(streaming_model_path) as model, model.session() as session: + with model.session() as sibling, session.stream() as stream: + stream.feed(audio_pcm[:16000]) + bad = list(audio_pcm[16000:32000]) + bad[100] = float("nan") + with pytest.raises(t.InvalidArgument): + stream.feed(bad) + assert stream.state == "active" + assert not model._compute_lock.locked() + with pytest.raises(t.Busy): + sibling.run(audio_pcm[:16000]) + for i in range(16000, len(audio_pcm), 16000): + stream.feed(audio_pcm[i : i + 16000]) + update = stream.finalize() + committed = stream.text().committed + ran = sibling.run(audio_pcm).text # finalize released the lease + assert update.is_final + assert "country" in committed.lower(), committed + assert "country" in ran.lower(), ran + + +def test_stream_begin_clears_pending_cancel(streaming_model_path, audio_pcm): + # A cancel() requested before stream() is cleared by the begin: the new + # stream's feeds are not aborted by a stale flag. (A cancel() AFTER begin + # is honoured on the next feed — test_cancellation_aborts_pending_feed.) + with t.Model(streaming_model_path) as model, model.session() as session: + session.cancel() + with session.stream() as stream: + stream.feed(audio_pcm[:16000]) + assert not session.was_aborted + assert stream.state == "active" diff --git a/bindings/python/tests/test_transcribe.py b/bindings/python/tests/test_transcribe.py index e242bcaa4..bc8779b39 100644 --- a/bindings/python/tests/test_transcribe.py +++ b/bindings/python/tests/test_transcribe.py @@ -193,6 +193,18 @@ def test_was_aborted_false_after_clean_run(self, model_path, audio_pcm): session.run(audio_pcm) assert session.was_aborted is False + def test_run_clears_pending_cancel(self, model_path, audio_pcm): + # A cancel() requested before run()/run_batch() is cleared when the + # call starts: the stale flag must not abort the new call. + with t.Model(model_path) as model, model.session() as session: + session.cancel() + assert "country" in session.run(audio_pcm).text.lower() + assert session.was_aborted is False + session.cancel() + results = session.run_batch([audio_pcm]) + assert "country" in results[0].text.lower() + assert session.was_aborted is False + def test_cross_thread_cancel_in_flight_run(self, model_path, audio_pcm): # Long input (tiled jfk) so the run is mid-flight when the timer # fires from another thread. The abort must surface as Aborted with diff --git a/bindings/rust/sys/src/transcribe_sys.rs b/bindings/rust/sys/src/transcribe_sys.rs index f679cb8ff..2c08ac549 100644 --- a/bindings/rust/sys/src/transcribe_sys.rs +++ b/bindings/rust/sys/src/transcribe_sys.rs @@ -1,11 +1,11 @@ // @generated by `cargo xtask bindgen` from include/transcribe/extensions.h // DO NOT EDIT BY HAND. Regenerate: `cargo xtask bindgen`. -// Pinned to include/transcribe.abihash = 57b1af43650f195d +// Pinned to include/transcribe.abihash = bd3273dabb25a1fe /// The public-ABI digest these bindings were generated against /// (sha256/16 over the normalized FFI surface). The load-time version /// gate and the CI drift check both anchor on this value. -pub const PUBLIC_HEADER_HASH: &str = "57b1af43650f195d"; +pub const PUBLIC_HEADER_HASH: &str = "bd3273dabb25a1fe"; /* automatically generated by rust-bindgen 0.72.1 */ @@ -19,7 +19,7 @@ pub const TRANSCRIBE_BACKEND_MASK_ALL: u32 = 4294967295; pub const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM: u32 = 1414746957; pub const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM: u32 = 1414744912; pub const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM: u32 = 1396853584; -pub const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM: u32 = 1414743635; +pub const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE: u32 = 1380206163; pub const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM: u32 = 1414746710; pub const TRANSCRIBE_EXT_KIND_WHISPER_RUN: u32 = 1314015319; impl transcribe_status { @@ -43,6 +43,7 @@ impl transcribe_status { pub const TRANSCRIBE_ERR_INPUT_TOO_LONG: transcribe_status = transcribe_status(17); pub const TRANSCRIBE_ERR_OUTPUT_TRUNCATED: transcribe_status = transcribe_status(18); pub const TRANSCRIBE_ERR_OUTPUT_REPETITION: transcribe_status = transcribe_status(19); + pub const TRANSCRIBE_ERR_UNSUPPORTED_ROLE: transcribe_status = transcribe_status(20); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -74,6 +75,10 @@ impl transcribe_abi_struct { pub const TRANSCRIBE_ABI_DEVICE_INFO: transcribe_abi_struct = transcribe_abi_struct(13); pub const TRANSCRIBE_ABI_SPEAKER_SEGMENT: transcribe_abi_struct = transcribe_abi_struct(14); pub const TRANSCRIBE_ABI_BACKEND_INIT_PARAMS: transcribe_abi_struct = transcribe_abi_struct(15); + pub const TRANSCRIBE_ABI_DIARIZE_INFO: transcribe_abi_struct = transcribe_abi_struct(16); + pub const TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: transcribe_abi_struct = + transcribe_abi_struct(17); + pub const TRANSCRIBE_ABI_DIARIZE_PARAMS: transcribe_abi_struct = transcribe_abi_struct(18); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -191,6 +196,7 @@ unsafe extern "C" { impl transcribe_ext_slot { pub const TRANSCRIBE_EXT_SLOT_RUN: transcribe_ext_slot = transcribe_ext_slot(0); pub const TRANSCRIBE_EXT_SLOT_STREAM: transcribe_ext_slot = transcribe_ext_slot(1); + pub const TRANSCRIBE_EXT_SLOT_DIARIZE_RUN: transcribe_ext_slot = transcribe_ext_slot(2); } #[repr(transparent)] #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] @@ -480,6 +486,16 @@ const _: () = { unsafe extern "C" { pub fn transcribe_capabilities_init(out: *mut transcribe_capabilities); } +impl transcribe_role { + pub const TRANSCRIBE_ROLE_ASR: transcribe_role = transcribe_role(1); + pub const TRANSCRIBE_ROLE_DIARIZE: transcribe_role = transcribe_role(2); +} +#[repr(transparent)] +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct transcribe_role(pub ::std::os::raw::c_uint); +unsafe extern "C" { + pub fn transcribe_model_roles(model: *const transcribe_model) -> u32; +} unsafe extern "C" { pub fn transcribe_model_get_capabilities( model: *const transcribe_model, @@ -1149,6 +1165,122 @@ unsafe extern "C" { } #[repr(C)] #[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_session { + _unused: [u8; 0], +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_info { + pub struct_size: u64, + pub sample_rate: i32, + pub max_speakers: i32, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_info"][::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_info"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_info::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_info, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_info::sample_rate"] + [::std::mem::offset_of!(transcribe_diarize_info, sample_rate) - 8usize]; + ["Offset of field: transcribe_diarize_info::max_speakers"] + [::std::mem::offset_of!(transcribe_diarize_info, max_speakers) - 12usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_session_params { + pub struct_size: u64, + pub n_threads: i32, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_session_params"] + [::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_session_params"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_session_params::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_session_params, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_session_params::n_threads"] + [::std::mem::offset_of!(transcribe_diarize_session_params, n_threads) - 8usize]; +}; +#[repr(C)] +#[derive(Debug, Copy, Clone)] +pub struct transcribe_diarize_params { + pub struct_size: u64, + pub family: *const transcribe_ext, +} +#[allow(clippy::unnecessary_operation, clippy::identity_op)] +const _: () = { + ["Size of transcribe_diarize_params"] + [::std::mem::size_of::() - 16usize]; + ["Alignment of transcribe_diarize_params"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_diarize_params::struct_size"] + [::std::mem::offset_of!(transcribe_diarize_params, struct_size) - 0usize]; + ["Offset of field: transcribe_diarize_params::family"] + [::std::mem::offset_of!(transcribe_diarize_params, family) - 8usize]; +}; +unsafe extern "C" { + pub fn transcribe_diarize_info_init(out: *mut transcribe_diarize_info); +} +unsafe extern "C" { + pub fn transcribe_diarize_session_params_init(params: *mut transcribe_diarize_session_params); +} +unsafe extern "C" { + pub fn transcribe_diarize_params_init(params: *mut transcribe_diarize_params); +} +unsafe extern "C" { + pub fn transcribe_diarize_get_info( + model: *const transcribe_model, + out: *mut transcribe_diarize_info, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_session_init( + model: *mut transcribe_model, + params: *const transcribe_diarize_session_params, + out_session: *mut *mut transcribe_diarize_session, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_session_free(session: *mut transcribe_diarize_session); +} +unsafe extern "C" { + pub fn transcribe_diarize_set_abort_callback( + session: *mut transcribe_diarize_session, + cb: transcribe_abort_callback, + user_data: *mut ::std::os::raw::c_void, + ); +} +unsafe extern "C" { + pub fn transcribe_diarize_run( + session: *mut transcribe_diarize_session, + pcm: *const f32, + n_samples: ::std::os::raw::c_int, + params: *const transcribe_diarize_params, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_n_segments( + session: *const transcribe_diarize_session, + ) -> ::std::os::raw::c_int; +} +unsafe extern "C" { + pub fn transcribe_diarize_get_segment( + session: *const transcribe_diarize_session, + i: ::std::os::raw::c_int, + out: *mut transcribe_speaker_segment, + ) -> transcribe_status; +} +unsafe extern "C" { + pub fn transcribe_diarize_get_timings( + session: *const transcribe_diarize_session, + out: *mut transcribe_timings, + ) -> transcribe_status; +} +#[repr(C)] +#[derive(Debug, Copy, Clone)] pub struct transcribe_moonshine_streaming_stream_ext { pub ext: transcribe_ext, pub min_decode_interval_ms: i32, @@ -1235,23 +1367,23 @@ impl transcribe_sortformer_preset { pub struct transcribe_sortformer_preset(pub ::std::os::raw::c_uint); #[repr(C)] #[derive(Debug, Copy, Clone)] -pub struct transcribe_sortformer_stream_ext { +pub struct transcribe_sortformer_diarize_ext { pub ext: transcribe_ext, pub preset: transcribe_sortformer_preset, } #[allow(clippy::unnecessary_operation, clippy::identity_op)] const _: () = { - ["Size of transcribe_sortformer_stream_ext"] - [::std::mem::size_of::() - 24usize]; - ["Alignment of transcribe_sortformer_stream_ext"] - [::std::mem::align_of::() - 8usize]; - ["Offset of field: transcribe_sortformer_stream_ext::ext"] - [::std::mem::offset_of!(transcribe_sortformer_stream_ext, ext) - 0usize]; - ["Offset of field: transcribe_sortformer_stream_ext::preset"] - [::std::mem::offset_of!(transcribe_sortformer_stream_ext, preset) - 16usize]; + ["Size of transcribe_sortformer_diarize_ext"] + [::std::mem::size_of::() - 24usize]; + ["Alignment of transcribe_sortformer_diarize_ext"] + [::std::mem::align_of::() - 8usize]; + ["Offset of field: transcribe_sortformer_diarize_ext::ext"] + [::std::mem::offset_of!(transcribe_sortformer_diarize_ext, ext) - 0usize]; + ["Offset of field: transcribe_sortformer_diarize_ext::preset"] + [::std::mem::offset_of!(transcribe_sortformer_diarize_ext, preset) - 16usize]; }; unsafe extern "C" { - pub fn transcribe_sortformer_stream_ext_init(ext: *mut transcribe_sortformer_stream_ext); + pub fn transcribe_sortformer_diarize_ext_init(ext: *mut transcribe_sortformer_diarize_ext); } #[repr(C)] #[derive(Debug, Copy, Clone)] diff --git a/bindings/rust/transcribe-cpp/README.md b/bindings/rust/transcribe-cpp/README.md index c8f037a63..dd12b78c7 100644 --- a/bindings/rust/transcribe-cpp/README.md +++ b/bindings/rust/transcribe-cpp/README.md @@ -76,6 +76,23 @@ let transcript = stream.snapshot(); // language, segments, words, tokens, timing # Ok::<(), transcribe_cpp::Error>(()) ``` +### Diarization (who spoke when) + +A model whose `roles()` contain `Role::Diarize` (e.g. Sortformer) opens a +`DiarizeSession` that returns speaker turns; calls for a role the model lacks +return `Error::UnsupportedRole`. + +```rust +use transcribe_cpp::{DiarizeOptions, Model, Role}; +let model = Model::load("diarizer.gguf")?; +assert!(model.roles().contains(Role::Diarize)); +let mut diarize = model.diarize_session()?; +for turn in diarize.run(&pcm, &DiarizeOptions::default())? { + println!("speaker {}: {}..{} ms", turn.speaker_id, turn.t0_ms, turn.t1_ms); +} +# Ok::<(), transcribe_cpp::Error>(()) +``` + Runnable examples: ```sh @@ -198,6 +215,7 @@ fn main() { - `Session` is `Send` but not `Sync`; mutating calls take `&mut self`. - In 0.x the C library allows at most one in-flight run across all sessions of a model; this crate enforces it with a per-model mutex, so concurrent calls - queue rather than race. For real parallelism, use one `Model` per worker. + queue rather than race. `DiarizeSession`s share the same lock. For real + parallelism, use one `Model` per worker. - License: MIT diff --git a/bindings/rust/transcribe-cpp/examples/streaming.rs b/bindings/rust/transcribe-cpp/examples/streaming.rs index 2afb3f7cc..4c2bbc0e6 100644 --- a/bindings/rust/transcribe-cpp/examples/streaming.rs +++ b/bindings/rust/transcribe-cpp/examples/streaming.rs @@ -27,7 +27,7 @@ fn main() -> Result<(), Box> { let pcm = common::load_wav(&audio_path); let model = Model::load(&model_path)?; - if !model.capabilities().supports_streaming { + if !model.capabilities()?.supports_streaming { // Not an error — the wrong-model case, stated plainly (a whisper model // would land here). Exit 0 so CI's no-canary path stays green. eprintln!("{} does not support streaming", model.arch()); diff --git a/bindings/rust/transcribe-cpp/examples/transcribe-file.rs b/bindings/rust/transcribe-cpp/examples/transcribe-file.rs index 0def25b0f..f5974ea4e 100644 --- a/bindings/rust/transcribe-cpp/examples/transcribe-file.rs +++ b/bindings/rust/transcribe-cpp/examples/transcribe-file.rs @@ -36,7 +36,7 @@ fn main() -> Result<(), Box> { let pcm = common::load_wav(&audio_path); let model = Model::load(&model_path)?; - let caps = model.capabilities(); + let caps = model.capabilities()?; println!( "model: {} on {} | max_ts={:?}", model.arch(), diff --git a/bindings/rust/transcribe-cpp/src/diarize.rs b/bindings/rust/transcribe-cpp/src/diarize.rs new file mode 100644 index 000000000..fe343bd1a --- /dev/null +++ b/bindings/rust/transcribe-cpp/src/diarize.rs @@ -0,0 +1,179 @@ +//! [`DiarizeSession`] — the DIARIZE role (who spoke when), from [`Model::diarize_session`]. +//! Threading, lifetime, and compute-lock rules are those of [`Session`](crate::Session). + +use std::os::raw::c_void; +use std::sync::atomic::AtomicBool; +use std::sync::Arc; + +use transcribe_cpp_sys as sys; + +use crate::cancel::{abort_trampoline, CancelToken}; +use crate::error::{check, Result}; +use crate::family::DiarizeExtension; +use crate::model::{Model, ModelInner}; +use crate::result::{SpeakerSegment, Timings}; +use crate::session::clamp_len; + +/// Static facts about a diarization model ([`Model::diarize_info`]). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct DiarizeInfo { + /// Input PCM rate (16000). + pub sample_rate: i32, + /// `speaker_id` is in `[1, max_speakers]`. + pub max_speakers: i32, +} + +/// Options for creating a diarization session. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default) +)] +pub struct DiarizeSessionOptions { + /// CPU threads for ops that run on CPU; 0 = library default. + pub n_threads: i32, +} + +/// Per-run diarization parameters. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +#[cfg_attr( + feature = "serde", + derive(serde::Serialize, serde::Deserialize), + serde(default) +)] +pub struct DiarizeOptions { + /// Optional family-specific extension (e.g. the Sortformer preset). + pub family: Option, +} + +/// A diarization session. +pub struct DiarizeSession { + ptr: *mut sys::transcribe_diarize_session, + // Keeps the native model alive and carries the per-model compute lock. + model: Arc, + // Keeps the abort callback's userdata alive while installed. + cancel: Option>, +} + +impl std::fmt::Debug for DiarizeSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DiarizeSession").finish_non_exhaustive() + } +} + +// SAFETY: as for `Session` — `&mut self` on the mutating calls keeps use to +// one thread at a time; deliberately NOT Sync. +unsafe impl Send for DiarizeSession {} + +impl Drop for DiarizeSession { + fn drop(&mut self) { + unsafe { sys::transcribe_diarize_session_free(self.ptr) }; + } +} + +impl DiarizeSession { + pub(crate) fn new(model: &Model, options: &DiarizeSessionOptions) -> Result { + let mut params: sys::transcribe_diarize_session_params = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_session_params_init(&mut params) }; + params.n_threads = options.n_threads; + + let mut out: *mut sys::transcribe_diarize_session = std::ptr::null_mut(); + let status = + unsafe { sys::transcribe_diarize_session_init(model.inner.ptr, ¶ms, &mut out) }; + check(status, "diarize session init")?; + debug_assert!(!out.is_null()); + + Ok(DiarizeSession { + ptr: out, + model: Arc::clone(&model.inner), + cancel: None, + }) + } + + /// Install a [`CancelToken`] so an in-flight run can be aborted from + /// another thread (the run then returns [`Error::Aborted`](crate::Error)). + /// Replaces any previously installed token. + pub fn set_cancel_token(&mut self, token: &CancelToken) { + let flag = Arc::clone(&token.flag); + let userdata = Arc::as_ptr(&flag) as *mut c_void; + unsafe { + sys::transcribe_diarize_set_abort_callback(self.ptr, Some(abort_trampoline), userdata) + }; + self.cancel = Some(flag); + } + + /// Remove any installed cancel token. + pub fn clear_cancel_token(&mut self) { + unsafe { sys::transcribe_diarize_set_abort_callback(self.ptr, None, std::ptr::null_mut()) }; + self.cancel = None; + } + + /// Diarize one recording of 16 kHz mono float32 PCM. Returns the speaker + /// segments, grouped by speaker and time-ordered within a speaker; + /// segments of different speakers may overlap. + pub fn run(&mut self, pcm: &[f32], options: &DiarizeOptions) -> Result> { + let n = clamp_len(pcm.len())?; + let family = options.family.as_ref().map(DiarizeExtension::materialize); + let mut params: sys::transcribe_diarize_params = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_params_init(&mut params) }; + params.family = family.as_ref().map_or(std::ptr::null(), |f| f.ext_ptr()); + + let status = self.model.with_compute( + Some("a stream is active on this model; finish or drop it before diarize run()"), + |_| unsafe { sys::transcribe_diarize_run(self.ptr, pcm.as_ptr(), n, ¶ms) }, + )?; + check(status, "diarize run")?; + + let count = unsafe { sys::transcribe_diarize_n_segments(self.ptr) }; + Ok((0..count) + .map(|i| { + let mut raw: sys::transcribe_speaker_segment = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_speaker_segment_init(&mut raw) }; + let _ = unsafe { sys::transcribe_diarize_get_segment(self.ptr, i, &mut raw) }; + SpeakerSegment::from_raw(&raw) + }) + .collect()) + } + + /// Model load time plus the last run's stage timings (`decode_ms` is 0). + pub fn timings(&self) -> Timings { + let mut raw: sys::transcribe_timings = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_timings_init(&mut raw) }; + let _ = unsafe { sys::transcribe_diarize_get_timings(self.ptr, &mut raw) }; + Timings::from_raw(&raw) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::Error; + use std::sync::Mutex; + + #[test] + fn run_is_busy_while_a_stream_holds_the_lease() { + // Null native handles (both frees are NULL no-ops): the Busy check runs + // under the lock before any native call. + let mut session = DiarizeSession { + ptr: std::ptr::null_mut(), + model: Arc::new(ModelInner { + ptr: std::ptr::null_mut(), + compute_lock: Mutex::new(true), + }), + cancel: None, + }; + let err = session + .run(&[0.0; 160], &DiarizeOptions::default()) + .unwrap_err(); + let Error::Busy(msg) = err else { + panic!("expected Busy, got {err:?}"); + }; + assert_eq!( + msg, + "a stream is active on this model; finish or drop it before diarize run()" + ); + } +} diff --git a/bindings/rust/transcribe-cpp/src/error.rs b/bindings/rust/transcribe-cpp/src/error.rs index f874dbb1a..eaa274add 100644 --- a/bindings/rust/transcribe-cpp/src/error.rs +++ b/bindings/rust/transcribe-cpp/src/error.rs @@ -75,6 +75,9 @@ pub enum Error { /// The (incomplete) transcript produced before the loop, one copy kept. partial: Option>, }, + /// `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` — see [`Model::roles`](crate::Model::roles). + #[error("unsupported role: {0}")] + UnsupportedRole(String), /// The loaded library's base version disagrees with the headers this crate /// was generated against (the pre-1.0 version lock). Raised on first use. #[error("native library version mismatch: {0}")] @@ -113,6 +116,7 @@ impl Error { Error::Aborted { .. } => S::TRANSCRIBE_ERR_ABORTED, Error::OutputTruncated { .. } => S::TRANSCRIBE_ERR_OUTPUT_TRUNCATED, Error::OutputRepetition { .. } => S::TRANSCRIBE_ERR_OUTPUT_REPETITION, + Error::UnsupportedRole(_) => S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, _ => S::TRANSCRIBE_OK, }; s.0 as i32 @@ -186,6 +190,7 @@ pub(crate) fn error_for_status(status: sys::transcribe_status, context: &str) -> message: msg, partial: None, }, + S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE => Error::UnsupportedRole(msg), _ => Error::Other(msg), } } @@ -198,3 +203,48 @@ pub(crate) fn check(status: sys::transcribe_status, context: &str) -> Result<()> Err(error_for_status(status, context)) } } + +#[cfg(test)] +mod tests { + use super::*; + use std::mem::discriminant; + use sys::transcribe_status as S; + + #[test] + fn every_status_maps_to_a_typed_variant() { + // Every non-OK status the linked library knows, found by walking codes + // upward until `transcribe_status_string` falls back to its "unknown" + // text, so a status appended to the header without a mapping fails here. + let unknown = status_string(-1); + let codes: Vec = (1..256u32) + .take_while(|&c| status_string(c as i32) != unknown) + .map(S) + .collect(); + assert!( + codes.len() >= S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE.0 as usize, + "status walk stopped early at {} codes", + codes.len() + ); + for status in codes { + let code = status.0; + let err = error_for_status(status, "ctx"); + assert!( + !matches!(err, Error::Other(_)), + "status {code} falls through to Error::Other: {err:?}" + ); + // raw_status must name a status that maps back to the same + // variant (grouped codes like SAMPLE_RATE report their group's + // primary status, never 0). + let raw = err.raw_status(); + assert_ne!(raw, 0, "status {code} reports raw_status 0: {err:?}"); + let back = error_for_status(S(raw as u32), "ctx"); + assert_eq!( + discriminant(&back), + discriminant(&err), + "status {code} -> {err:?} -> raw {raw} -> {back:?}" + ); + } + let role = error_for_status(S::TRANSCRIBE_ERR_UNSUPPORTED_ROLE, "ctx"); + assert!(matches!(role, Error::UnsupportedRole(_)), "{role:?}"); + } +} diff --git a/bindings/rust/transcribe-cpp/src/family.rs b/bindings/rust/transcribe-cpp/src/family.rs index 33907ac05..4de0b0506 100644 --- a/bindings/rust/transcribe-cpp/src/family.rs +++ b/bindings/rust/transcribe-cpp/src/family.rs @@ -5,7 +5,8 @@ //! every field is an `Option`, and only the fields you set override the //! defaults the C `*_init()` stamps. A [`RunExtension`] attaches to //! [`RunOptions`](crate::RunOptions); a [`StreamExtension`] attaches to -//! [`StreamOptions`](crate::StreamOptions). +//! [`StreamOptions`](crate::StreamOptions); a [`DiarizeExtension`] attaches to +//! [`DiarizeOptions`](crate::DiarizeOptions). //! //! Probe [`Model::accepts_ext`](crate::Model::accepts_ext) to learn whether a //! loaded model accepts a given kind on a slot; an unaccepted extension is @@ -122,26 +123,62 @@ impl SortformerPreset { } } -/// Sortformer diarizer run-extension knobs (run slot). Sortformer produces -/// speaker segments, no text; read results via the speaker-segment -/// accessors. `None` keeps the family default (the GGUF-shipped cfg). +/// Sortformer diarize-extension knobs (diarize-run slot). `None` keeps the +/// family default (the GGUF-shipped cfg). #[derive(Debug, Clone, Default, PartialEq, Eq)] #[cfg_attr( feature = "serde", derive(serde::Serialize, serde::Deserialize), serde(default) )] -pub struct SortformerStreamOptions { +pub struct SortformerDiarizeOptions { pub preset: Option, } +/// A family extension for the diarize-run slot +/// ([`DiarizeSession::run`](crate::DiarizeSession::run)). +#[derive(Debug, Clone, PartialEq, Eq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum DiarizeExtension { + Sortformer(SortformerDiarizeOptions), +} + +/// Owns a materialized diarize-slot C extension struct. +pub(crate) enum DiarizeExtRaw { + Sortformer(Box), +} + +impl DiarizeExtRaw { + pub(crate) fn ext_ptr(&self) -> *const sys::transcribe_ext { + match self { + DiarizeExtRaw::Sortformer(e) => { + (&**e) as *const sys::transcribe_sortformer_diarize_ext + as *const sys::transcribe_ext + } + } + } +} + +impl DiarizeExtension { + pub(crate) fn materialize(&self) -> DiarizeExtRaw { + match self { + DiarizeExtension::Sortformer(o) => { + let mut e: sys::transcribe_sortformer_diarize_ext = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_sortformer_diarize_ext_init(&mut e) }; + set(&mut e.preset, o.preset.map(SortformerPreset::to_sys)); + DiarizeExtRaw::Sortformer(Box::new(e)) + } + } + } +} + /// A family extension for the run slot (offline `run`/`run_batch`). #[derive(Debug, Clone, PartialEq)] #[non_exhaustive] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum RunExtension { Whisper(WhisperRunOptions), - Sortformer(SortformerStreamOptions), } /// A family extension for the stream slot. @@ -164,7 +201,6 @@ pub(crate) enum RunExtRaw { ext: Box, _prompt: Option, }, - Sortformer(Box), } impl RunExtRaw { @@ -174,9 +210,6 @@ impl RunExtRaw { RunExtRaw::Whisper { ext, .. } => { (&**ext) as *const sys::transcribe_whisper_run_ext as *const sys::transcribe_ext } - RunExtRaw::Sortformer(e) => { - (&**e) as *const sys::transcribe_sortformer_stream_ext as *const sys::transcribe_ext - } } } } @@ -210,12 +243,6 @@ impl RunExtension { _prompt: prompt, }) } - RunExtension::Sortformer(o) => { - let mut ext: sys::transcribe_sortformer_stream_ext = unsafe { std::mem::zeroed() }; - unsafe { sys::transcribe_sortformer_stream_ext_init(&mut ext) }; - set(&mut ext.preset, o.preset.map(SortformerPreset::to_sys)); - Ok(RunExtRaw::Sortformer(Box::new(ext))) - } } } } diff --git a/bindings/rust/transcribe-cpp/src/lib.rs b/bindings/rust/transcribe-cpp/src/lib.rs index eb34e0040..4c4c5d707 100644 --- a/bindings/rust/transcribe-cpp/src/lib.rs +++ b/bindings/rust/transcribe-cpp/src/lib.rs @@ -27,7 +27,8 @@ //! - [`Session`] is `Send` but not `Sync`; its mutating calls take `&mut self`, //! so the type system enforces one-call-at-a-time. //! - **0.x concurrency:** the C library permits at most one in-flight -//! `run`/stream across all sessions of a model. This crate enforces that with +//! `run`/stream across all sessions of a model, [`DiarizeSession`]s +//! included. This crate enforces that with //! a per-model mutex (held only for the native compute call), so concurrent //! calls from many sessions queue rather than race. For real parallelism, use //! one [`Model`] per worker today. @@ -47,6 +48,7 @@ pub use transcribe_cpp_sys as sys; mod backend; mod cancel; +mod diarize; mod error; mod family; mod logging; @@ -62,11 +64,12 @@ pub use backend::{ init_backends_default, init_backends_with, BackendMask, Device, DeviceType, }; pub use cancel::CancelToken; +pub use diarize::{DiarizeInfo, DiarizeOptions, DiarizeSession, DiarizeSessionOptions}; pub use error::{Error, Result}; pub use family::{ - MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, RunExtension, - SortformerPreset, SortformerStreamOptions, StreamExtension, VoxtralRealtimeStreamOptions, - WhisperRunOptions, + DiarizeExtension, MoonshineStreamingOptions, ParakeetBufferedStreamOptions, + ParakeetStreamOptions, RunExtension, SortformerDiarizeOptions, SortformerPreset, + StreamExtension, VoxtralRealtimeStreamOptions, WhisperRunOptions, }; pub use logging::{disable_logging, init_logging}; pub use model::{Capabilities, Model, ModelOptions, SessionLimits, SessionOptions}; @@ -74,8 +77,8 @@ pub use result::{Segment, SpeakerSegment, Timings, Token, Transcript, Word}; pub use session::{RunOptions, Session, Stream}; pub use streaming::{StreamOptions, StreamText, StreamUpdate}; pub use types::{ - AbiStruct, Backend, CommitPolicy, Diarize, ExtSlot, Feature, Itn, KvType, Pnc, StreamState, - Task, TimestampKind, + AbiStruct, Backend, CommitPolicy, Diarize, ExtSlot, Feature, Itn, KvType, Pnc, Role, Roles, + StreamState, Task, TimestampKind, }; pub use version::{ abi_struct_align, abi_struct_size, compiled_version, header_hash, version, version_commit, diff --git a/bindings/rust/transcribe-cpp/src/model.rs b/bindings/rust/transcribe-cpp/src/model.rs index 1e32daabc..615f28506 100644 --- a/bindings/rust/transcribe-cpp/src/model.rs +++ b/bindings/rust/transcribe-cpp/src/model.rs @@ -2,17 +2,19 @@ //! //! A `Model` is `Arc`-backed and `Send + Sync`: cloning it is cheap and hands //! out another handle to the same native model. The native model is freed when -//! the last handle AND every [`Session`](crate::Session) derived from it have -//! been dropped — Rust ownership gives the C "model must outlive its sessions" -//! contract (and close-ordering safety) for free, in any drop order. +//! the last handle AND every [`Session`](crate::Session) / +//! [`DiarizeSession`](crate::DiarizeSession) derived from it have been dropped +//! — Rust ownership gives the C "model must outlive its sessions" contract (and +//! close-ordering safety) for free, in any drop order. //! //! The `Arc` also carries the per-model compute lock that enforces the C //! library's 0.x concurrency limitation: at most one `run` / `run_batch` / -//! active stream may be in flight across ALL sessions of a model. One-shot -//! runs/batches queue on the lock (serialized); an *active stream* spans many -//! native calls (`begin`→`feed`*→`finalize`), so the lock also carries a flag -//! marking a stream in flight — any run/batch/stream that would overlap it is -//! refused with [`Error::Busy`](crate::Error) rather than allowed to race. +//! diarize run / active stream may be in flight across ALL sessions of a model. +//! One-shot runs/batches queue on the lock (serialized); an *active stream* +//! spans many native calls (`begin`→`feed`*→`finalize`), so the lock also +//! carries a flag marking a stream in flight — any run/batch/stream that would +//! overlap it is refused with [`Error::Busy`](crate::Error) rather than allowed +//! to race. use std::ffi::CString; use std::path::Path; @@ -21,10 +23,11 @@ use std::sync::{Arc, Mutex}; use transcribe_cpp_sys as sys; use crate::backend::Device; -use crate::error::{check, Result}; +use crate::diarize::{DiarizeInfo, DiarizeSession, DiarizeSessionOptions}; +use crate::error::{check, Error, Result}; use crate::result::owned_str; use crate::session::Session; -use crate::types::{Backend, ExtSlot, Feature, TimestampKind}; +use crate::types::{Backend, ExtSlot, Feature, Roles, TimestampKind}; use crate::version; /// Options for loading a model. @@ -73,10 +76,30 @@ pub(crate) struct ModelInner { /// is `true` while a stream is ACTIVE (from `begin` until the earliest of /// `finalize` / `reset` / the `Stream` being dropped); a held lock plus a /// `true` flag is how `run`/`run_batch`/`stream` detect and refuse an - /// overlapping compute. See the module docs. + /// overlapping compute. Only taken through [`ModelInner::with_compute`]. + /// See the module docs. pub(crate) compute_lock: Mutex, } +impl ModelInner { + /// Run `f` under the model's compute lock; every native compute call goes + /// through here. With `Some(msg)`, returns `Error::Busy(msg)` instead if a + /// stream holds the lease. `f` gets the lease flag to take or release it. + pub(crate) fn with_compute( + &self, + refuse_if_streaming: Option<&str>, + f: impl FnOnce(&mut bool) -> R, + ) -> Result { + let mut lease = self.compute_lock.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(msg) = refuse_if_streaming { + if *lease { + return Err(Error::Busy(msg.into())); + } + } + Ok(f(&mut lease)) + } +} + // SAFETY: the native model is documented thread-safe for query + session // creation; the compute path is serialized by `compute_lock`. The raw pointer // is only ever used behind `&ModelInner`, never mutated through aliasing. @@ -151,13 +174,46 @@ impl Model { Session::new(self, options) } - /// The model's immutable capabilities. - pub fn capabilities(&self) -> Capabilities { + /// Open a diarization session with default options. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Diarize`](crate::Role). + pub fn diarize_session(&self) -> Result { + self.diarize_session_with(&DiarizeSessionOptions::default()) + } + + /// Open a diarization session with explicit options. + pub fn diarize_session_with(&self, options: &DiarizeSessionOptions) -> Result { + DiarizeSession::new(self, options) + } + + /// Static facts about a diarization model. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Diarize`](crate::Role). + pub fn diarize_info(&self) -> Result { + let mut raw: sys::transcribe_diarize_info = unsafe { std::mem::zeroed() }; + unsafe { sys::transcribe_diarize_info_init(&mut raw) }; + check( + unsafe { sys::transcribe_diarize_get_info(self.inner.ptr, &mut raw) }, + "diarize info", + )?; + Ok(DiarizeInfo { + sample_rate: raw.sample_rate, + max_speakers: raw.max_speakers, + }) + } + + /// The roles (kinds of work) this model serves. + pub fn roles(&self) -> Roles { + Roles(unsafe { sys::transcribe_model_roles(self.inner.ptr) }) + } + + /// The model's immutable ASR capabilities. Errors with + /// [`Error::UnsupportedRole`] unless the model serves [`Role::Asr`](crate::Role). + pub fn capabilities(&self) -> Result { let mut caps: sys::transcribe_capabilities = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_capabilities_init(&mut caps) }; - // A NULL/struct-size fault cannot happen here (we own a valid model - // and an init'd struct), so a non-OK status leaves zeroed defaults. - let _ = unsafe { sys::transcribe_model_get_capabilities(self.inner.ptr, &mut caps) }; + check( + unsafe { sys::transcribe_model_get_capabilities(self.inner.ptr, &mut caps) }, + "capabilities", + )?; let mut languages = Vec::new(); if !caps.languages.is_null() && caps.n_languages > 0 { @@ -180,7 +236,7 @@ impl Model { } } - Capabilities { + Ok(Capabilities { native_sample_rate: caps.native_sample_rate, languages, translate_target_languages, @@ -190,7 +246,7 @@ impl Model { supports_streaming: caps.supports_streaming, supports_spec_decode: caps.supports_spec_decode, max_audio_ms: caps.max_audio_ms, - } + }) } /// Probe a yes/no feature. diff --git a/bindings/rust/transcribe-cpp/src/session.rs b/bindings/rust/transcribe-cpp/src/session.rs index d81647d8b..017c38d9c 100644 --- a/bindings/rust/transcribe-cpp/src/session.rs +++ b/bindings/rust/transcribe-cpp/src/session.rs @@ -172,22 +172,10 @@ impl Session { let (params, _lang, _target, _family, _prompting) = build_run_params(options)?; let n = clamp_len(pcm.len())?; - // The compute path is serialized per model; hold the lock for the native - // call, then materialize this session's own result storage. Refuse a run - // that would overlap an active stream on another session (the C contract). - let status = { - let guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is active on this model; finish or drop it before run()".into(), - )); - } - unsafe { sys::transcribe_run(self.ptr, pcm.as_ptr(), n, ¶ms) } - }; + let status = self.model.with_compute( + Some("a stream is active on this model; finish or drop it before run()"), + |_| unsafe { sys::transcribe_run(self.ptr, pcm.as_ptr(), n, ¶ms) }, + )?; match status { s if s == sys::transcribe_status::TRANSCRIBE_OK => Ok(self.materialize_run()), @@ -221,19 +209,12 @@ impl Session { .collect::>()?; let n = clamp_len(pcms.len())?; - let status = { - let guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is active on this model; finish or drop it before run_batch()".into(), - )); - } - unsafe { sys::transcribe_run_batch(self.ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) } - }; + let status = self.model.with_compute( + Some("a stream is active on this model; finish or drop it before run_batch()"), + |_| unsafe { + sys::transcribe_run_batch(self.ptr, ptrs.as_ptr(), lens.as_ptr(), n, ¶ms) + }, + )?; // OK and ABORTED both populate one per-utterance slot per input; fold // the batch-level abort into the per-utterance view (the C contract). @@ -293,26 +274,20 @@ impl Session { pub fn stream(&mut self, run: &RunOptions, stream: &StreamOptions) -> Result> { let (run_params, _lang, _target, _family, _prompting) = build_run_params(run)?; let (stream_params, _stream_family) = build_stream_params(stream); - { - // Claim the model's compute lease for the whole stream lifetime: a - // stream spans begin..drop, so per-call locking alone would let a - // second stream (or a run) on another session race it. - let mut guard = self - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - if *guard { - return Err(Error::Busy( - "a stream is already active on this model".into(), - )); - } - check( - unsafe { sys::transcribe_stream_begin(self.ptr, &run_params, &stream_params) }, - "stream begin", - )?; - *guard = true; // released at finalize/reset, or by Stream::drop - } + // Claim the model's compute lease for the whole stream lifetime: a + // stream spans begin..drop, so per-call locking alone would let a + // second stream (or a run) on another session race it. + self.model.with_compute( + Some("a stream is already active on this model"), + |lease| -> Result<()> { + check( + unsafe { sys::transcribe_stream_begin(self.ptr, &run_params, &stream_params) }, + "stream begin", + )?; + *lease = true; // released at finalize/reset, or by Stream::drop + Ok(()) + }, + )??; Ok(Stream { session: self, holds_lease: true, @@ -512,7 +487,7 @@ fn build_run_params(o: &RunOptions) -> Result { } /// PCM/utterance lengths cross the ABI as `int`; reject anything that overflows. -fn clamp_len(len: usize) -> Result { +pub(crate) fn clamp_len(len: usize) -> Result { i32::try_from(len).map_err(|_| Error::InvalidArgument(format!("length {len} exceeds i32::MAX"))) } @@ -542,6 +517,20 @@ fn attach_partial(err: Error, partial: Box) -> Error { /// [`Stream::text`], and end input with [`Stream::finalize`]. Dropping the /// `Stream` (without finalizing) abandons it and returns the session to idle. /// `Send` but not `Sync`, like the session it borrows. +/// +/// The borrow means a session can't be freed while its stream (or any other +/// call on it) is still in use, so no native free ever races a compute: +/// +/// ```compile_fail,E0505 +/// use transcribe_cpp::{Model, RunOptions, StreamOptions}; +/// let model = Model::load("model.gguf").unwrap(); +/// let mut session = model.session().unwrap(); +/// let mut stream = session +/// .stream(&RunOptions::default(), &StreamOptions::default()) +/// .unwrap(); +/// drop(session); // error[E0505]: cannot move out of `session` while borrowed +/// stream.feed(&[0.0; 160]).unwrap(); +/// ``` pub struct Stream<'a> { session: &'a mut Session, // True while this stream holds the model's compute lease. Set at begin, @@ -562,20 +551,7 @@ impl std::fmt::Debug for Stream<'_> { impl Drop for Stream<'_> { fn drop(&mut self) { - // Abandon any unfinalized stream (reset is idempotent and safe from any - // state) and release the lease IF this stream still holds it — under the - // lock. After finalize/reset the lease is already gone (and may now be - // another session's), so only clear it when holds_lease is still true. - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_reset(self.session.ptr) }; - if self.holds_lease { - *guard = false; - } + self.reset() } } @@ -585,17 +561,24 @@ impl Stream<'_> { let n = clamp_len(pcm.len())?; let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; - let status = { - // This stream already holds the model's compute lease (it is in - // flight); the lock here just serializes the native call. - let _guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_feed(self.session.ptr, pcm.as_ptr(), n, &mut update) } - }; + // This stream already holds the model's compute lease (it is in + // flight); the lock here just serializes the native call. A feed that + // ends the stream (FAILED) frees the lease, as finalize does; a feed + // rejected before the native hook (e.g. NaN) leaves it ACTIVE and keeps it. + let ptr = self.session.ptr; + let held = self.holds_lease; + let (status, ended) = self.session.model.with_compute(None, |lease| { + let st = unsafe { sys::transcribe_stream_feed(ptr, pcm.as_ptr(), n, &mut update) }; + let ended = unsafe { sys::transcribe_stream_get_state(ptr) } + == sys::transcribe_stream_state::TRANSCRIBE_STREAM_FAILED; + if held && ended { + *lease = false; + } + (st, ended) + })?; + if ended { + self.holds_lease = false; + } check(status, "stream feed")?; Ok(StreamUpdate::from_raw(&update)) } @@ -605,22 +588,16 @@ impl Stream<'_> { pub fn finalize(&mut self) -> Result { let mut update: sys::transcribe_stream_update = unsafe { std::mem::zeroed() }; unsafe { sys::transcribe_stream_update_init(&mut update) }; - let status = { - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); + let status = self.session.model.with_compute(None, |lease| { let st = unsafe { sys::transcribe_stream_finalize(self.session.ptr, &mut update) }; // Finalize ends the active stream (Finished, or Failed on error); // either way it is no longer active, so free the model's compute // lease now rather than holding other sessions off until drop. if self.holds_lease { - *guard = false; + *lease = false; } st - }; + })?; self.holds_lease = false; check(status, "stream finalize")?; Ok(StreamUpdate::from_raw(&update)) @@ -631,16 +608,13 @@ impl Stream<'_> { /// active), so other sessions of the model can proceed without waiting for /// this `Stream` to drop. pub fn reset(&mut self) { - let mut guard = self - .session - .model - .compute_lock - .lock() - .unwrap_or_else(|e| e.into_inner()); - unsafe { sys::transcribe_stream_reset(self.session.ptr) }; - if self.holds_lease { - *guard = false; - } + // Releases the lease only if this stream holds it (it may be another session's). + let _ = self.session.model.with_compute(None, |lease| { + unsafe { sys::transcribe_stream_reset(self.session.ptr) }; + if self.holds_lease { + *lease = false; + } + }); self.holds_lease = false; } diff --git a/bindings/rust/transcribe-cpp/src/types.rs b/bindings/rust/transcribe-cpp/src/types.rs index 4c780c6bd..eb1554eda 100644 --- a/bindings/rust/transcribe-cpp/src/types.rs +++ b/bindings/rust/transcribe-cpp/src/types.rs @@ -303,6 +303,7 @@ impl StreamState { /// A public ABI struct, for size/alignment introspection. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] pub enum AbiStruct { ModelLoadParams, SessionParams, @@ -320,6 +321,9 @@ pub enum AbiStruct { DeviceInfo, SpeakerSegment, BackendInitParams, + DiarizeInfo, + DiarizeSessionParams, + DiarizeParams, } impl AbiStruct { @@ -342,18 +346,24 @@ impl AbiStruct { AbiStruct::DeviceInfo => A::TRANSCRIBE_ABI_DEVICE_INFO, AbiStruct::SpeakerSegment => A::TRANSCRIBE_ABI_SPEAKER_SEGMENT, AbiStruct::BackendInitParams => A::TRANSCRIBE_ABI_BACKEND_INIT_PARAMS, + AbiStruct::DiarizeInfo => A::TRANSCRIBE_ABI_DIARIZE_INFO, + AbiStruct::DiarizeSessionParams => A::TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS, + AbiStruct::DiarizeParams => A::TRANSCRIBE_ABI_DIARIZE_PARAMS, } } } /// The slot a family extension is pointed at. #[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub enum ExtSlot { /// `transcribe_run_params::family`. Run, /// `transcribe_stream_params::family`. Stream, + /// `transcribe_diarize_params::family`. + DiarizeRun, } impl ExtSlot { @@ -362,6 +372,39 @@ impl ExtSlot { match self { ExtSlot::Run => E::TRANSCRIBE_EXT_SLOT_RUN, ExtSlot::Stream => E::TRANSCRIBE_EXT_SLOT_STREAM, + ExtSlot::DiarizeRun => E::TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, } } } + +/// A kind of work a model can do (`transcribe_role`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[non_exhaustive] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub enum Role { + /// Transcription: [`Session`](crate::Session). + Asr, + /// Speaker diarization: [`DiarizeSession`](crate::DiarizeSession). + Diarize, +} + +/// The set of [`Role`]s a model serves ([`Model::roles`](crate::Model::roles)). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct Roles(pub(crate) u32); + +impl Roles { + /// Whether the model serves `role`. + pub fn contains(self, role: Role) -> bool { + let bit = match role { + Role::Asr => sys::transcribe_role::TRANSCRIBE_ROLE_ASR, + Role::Diarize => sys::transcribe_role::TRANSCRIBE_ROLE_DIARIZE, + }; + self.0 & bit.0 != 0 + } + + /// The raw `transcribe_role` bitmask (`TRANSCRIBE_ROLE_*` bits). + pub fn bits(self) -> u32 { + self.0 + } +} diff --git a/bindings/rust/transcribe-cpp/tests/common/mod.rs b/bindings/rust/transcribe-cpp/tests/common/mod.rs index 3c5ebfa09..6e66a8a12 100644 --- a/bindings/rust/transcribe-cpp/tests/common/mod.rs +++ b/bindings/rust/transcribe-cpp/tests/common/mod.rs @@ -111,6 +111,23 @@ pub fn smoke_itn_model() -> Option { ) } +/// Sortformer diarizer (DIARIZE role only) plus its two-speaker sample, or +/// `None` (with a skip note) when either is absent. +pub fn smoke_sortformer_fixtures(test: &str) -> Option<(PathBuf, Vec)> { + let model = family_model( + "TRANSCRIBE_SMOKE_SORTFORMER_MODEL", + "models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-F32.gguf", + ); + let audio = repo_root().join("samples/sortformer-2spk-mix.wav"); + match model { + Some(m) if audio.is_file() => Some((m, load_wav(&audio))), + _ => { + eprintln!("skip {test}: sortformer model/audio absent (set TRANSCRIBE_SMOKE_SORTFORMER_MODEL)"); + None + } + } +} + /// Both fixtures together; prints a skip note and returns `None` if either is /// missing (so the caller can `return` early — the Rust equivalent of skip). pub fn smoke_fixtures(test: &str) -> Option<(PathBuf, Vec)> { diff --git a/bindings/rust/transcribe-cpp/tests/diarize.rs b/bindings/rust/transcribe-cpp/tests/diarize.rs new file mode 100644 index 000000000..eff77fd46 --- /dev/null +++ b/bindings/rust/transcribe-cpp/tests/diarize.rs @@ -0,0 +1,112 @@ +//! DIARIZE role on Sortformer (TRANSCRIBE_SMOKE_SORTFORMER_MODEL) and whisper +//! (TRANSCRIBE_SMOKE_MODEL); the Busy path is covered in `src/diarize.rs`. + +mod common; + +use transcribe_cpp::sys::TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE as SFDR; +use transcribe_cpp::{ + Backend, CancelToken, DiarizeExtension, DiarizeOptions, Error, ExtSlot, Model, ModelOptions, + Role, SortformerDiarizeOptions, SortformerPreset, +}; + +#[test] +fn sortformer_is_diarize_only() { + let Some((model_path, _)) = common::smoke_sortformer_fixtures("sortformer_is_diarize_only") + else { + return; + }; + let model = Model::load(&model_path).unwrap(); + let roles = model.roles(); + assert!(!roles.contains(Role::Asr) && roles.contains(Role::Diarize)); + let info = model.diarize_info().unwrap(); + assert_eq!((info.sample_rate, info.max_speakers), (16000, 4)); + assert!(model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); + assert!(!model.accepts_ext(ExtSlot::Run, SFDR)); + assert!(matches!( + model.capabilities(), + Err(Error::UnsupportedRole(_)) + )); + assert!(matches!(model.session(), Err(Error::UnsupportedRole(_)))); +} + +#[test] +fn diarize_run_returns_turns_and_applies_preset() { + let Some((model_path, pcm)) = + common::smoke_sortformer_fixtures("diarize_run_returns_turns_and_applies_preset") + else { + return; + }; + // Exact segments are pinned in C (tests/sortformer_diarize_unit.cpp); here + // the binding only has to return well-formed turns and pass the preset on. + let cpu = ModelOptions { + backend: Backend::Cpu, + ..Default::default() + }; + let model = Model::load_with(&model_path, &cpu).unwrap(); + let max_speakers = model.diarize_info().unwrap().max_speakers; + let mut diarize = model.diarize_session().unwrap(); + let rows = |turns: Vec| -> Vec<_> { + turns + .iter() + .map(|s| (s.t0_ms, s.t1_ms, s.speaker_id)) + .collect() + }; + + let default = rows(diarize.run(&pcm, &DiarizeOptions::default()).unwrap()); + assert!(!default.is_empty()); + for &(t0, t1, speaker) in &default { + assert!(t0 < t1, "{t0} >= {t1}"); + assert!((1..=max_speakers).contains(&speaker), "speaker {speaker}"); + } + assert!(diarize.timings().encode_ms > 0.0); + + let low_latency = DiarizeOptions { + family: Some(DiarizeExtension::Sortformer(SortformerDiarizeOptions { + preset: Some(SortformerPreset::LowLatency), + })), + }; + assert_ne!(rows(diarize.run(&pcm, &low_latency).unwrap()), default); +} + +#[test] +fn diarize_session_rejects_bad_input_and_cancels() { + let Some((model_path, pcm)) = + common::smoke_sortformer_fixtures("diarize_session_rejects_bad_input_and_cancels") + else { + return; + }; + // The session keeps its model alive after the last Model handle drops. + let mut diarize = Model::load(&model_path).unwrap().diarize_session().unwrap(); + let opts = DiarizeOptions::default(); + for bad in [&[][..], &[f32::NAN; 16][..]] { + let err = diarize.run(bad, &opts).unwrap_err(); + assert!(matches!(err, Error::InvalidArgument(_)), "{err:?}"); + } + let token = CancelToken::new(); + diarize.set_cancel_token(&token); + token.cancel(); + let err = diarize.run(&pcm, &opts).unwrap_err(); + assert!(matches!(err, Error::Aborted { .. }), "{err:?}"); + diarize.clear_cancel_token(); + assert!(!diarize.run(&pcm, &opts).unwrap().is_empty()); +} + +#[test] +fn asr_only_model_refuses_diarize() { + let Some(model_path) = common::smoke_model() else { + eprintln!("skip asr_only_model_refuses_diarize: smoke model absent"); + return; + }; + let model = Model::load(&model_path).unwrap(); + let roles = model.roles(); + assert!(roles.contains(Role::Asr) && !roles.contains(Role::Diarize)); + assert!(matches!( + model.diarize_info(), + Err(Error::UnsupportedRole(_)) + )); + assert!(matches!( + model.diarize_session(), + Err(Error::UnsupportedRole(_)) + )); + assert!(!model.accepts_ext(ExtSlot::DiarizeRun, SFDR)); +} diff --git a/bindings/rust/transcribe-cpp/tests/no_model.rs b/bindings/rust/transcribe-cpp/tests/no_model.rs index 9aa2c1e92..ed7df746d 100644 --- a/bindings/rust/transcribe-cpp/tests/no_model.rs +++ b/bindings/rust/transcribe-cpp/tests/no_model.rs @@ -35,6 +35,9 @@ fn abi_struct_sizes_are_live() { AbiStruct::SpeakerSegment, AbiStruct::SessionLimits, AbiStruct::BackendInitParams, + AbiStruct::DiarizeInfo, + AbiStruct::DiarizeSessionParams, + AbiStruct::DiarizeParams, ] { assert!(abi_struct_size(which) > 0, "{which:?} reported size 0"); } @@ -138,5 +141,6 @@ fn handles_are_send_sync() { fn assert_send() {} assert_send_sync::(); assert_send::(); - // Session is intentionally NOT Sync (single-threaded use). + assert_send::(); + // Sessions are intentionally NOT Sync (single-threaded use). } diff --git a/bindings/rust/transcribe-cpp/tests/serde.rs b/bindings/rust/transcribe-cpp/tests/serde.rs index d6fc94f30..1088f78b3 100644 --- a/bindings/rust/transcribe-cpp/tests/serde.rs +++ b/bindings/rust/transcribe-cpp/tests/serde.rs @@ -4,11 +4,12 @@ #![cfg(feature = "serde")] use transcribe_cpp::{ - Backend, Capabilities, CommitPolicy, DeviceType, Diarize, ExtSlot, Feature, Itn, KvType, - MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, Pnc, - RunExtension, RunOptions, Segment, SessionLimits, SessionOptions, SortformerPreset, - SortformerStreamOptions, SpeakerSegment, StreamExtension, StreamOptions, StreamState, - StreamText, StreamUpdate, Task, TimestampKind, Timings, Token, Transcript, + Backend, Capabilities, CommitPolicy, DeviceType, Diarize, DiarizeExtension, DiarizeInfo, + DiarizeOptions, DiarizeSessionOptions, ExtSlot, Feature, Itn, KvType, + MoonshineStreamingOptions, ParakeetBufferedStreamOptions, ParakeetStreamOptions, Pnc, Role, + Roles, RunExtension, RunOptions, Segment, SessionLimits, SessionOptions, + SortformerDiarizeOptions, SortformerPreset, SpeakerSegment, StreamExtension, StreamOptions, + StreamState, StreamText, StreamUpdate, Task, TimestampKind, Timings, Token, Transcript, VoxtralRealtimeStreamOptions, WhisperRunOptions, Word, }; @@ -28,9 +29,13 @@ fn plain_data_types_are_serializable() { assert_serde::(); assert_serde::(); assert_serde::(); - assert_serde::(); assert_serde::(); + assert_serde::(); + assert_serde::(); + assert_serde::(); + assert_serde::(); // Results. + assert_serde::(); assert_serde::(); assert_serde::(); assert_serde::(); @@ -54,6 +59,8 @@ fn plain_data_types_are_serializable() { assert_serde::(); assert_serde::(); assert_serde::(); + assert_serde::(); + assert_serde::(); } fn nan_transcript() -> Transcript { diff --git a/bindings/rust/transcribe-cpp/tests/streaming.rs b/bindings/rust/transcribe-cpp/tests/streaming.rs index c5b8d12e0..86b2ee6a8 100644 --- a/bindings/rust/transcribe-cpp/tests/streaming.rs +++ b/bindings/rust/transcribe-cpp/tests/streaming.rs @@ -3,9 +3,13 @@ mod common; -use transcribe_cpp::{Model, RunOptions, StreamExtension, StreamOptions, StreamState}; +use transcribe_cpp::{Error, Model, RunOptions, StreamExtension, StreamOptions, StreamState}; use transcribe_cpp::{MoonshineStreamingOptions, Stream}; +const RUN_BUSY: &str = "a stream is active on this model; finish or drop it before run()"; +const BATCH_BUSY: &str = "a stream is active on this model; finish or drop it before run_batch()"; +const STREAM_BUSY: &str = "a stream is already active on this model"; + /// Partial-decode cadence for the mechanics tests that only assert the /// post-finalize text. Aligned to the model, not an arbitrary number: the /// streaming encoder emits one frame per 20 ms (frame_len 80 × 4 subsampling), @@ -24,6 +28,20 @@ fn feed_in_chunks(stream: &mut Stream<'_>, pcm: &[f32], chunk: usize) { } } +fn expect_busy(r: transcribe_cpp::Result, want: &str) { + match r { + Err(Error::Busy(msg)) => assert_eq!(msg, want), + other => panic!("expected Error::Busy({want:?}), got {other:?}"), + } +} + +fn wrong_family_stream() -> StreamOptions { + StreamOptions { + family: Some(StreamExtension::ParakeetStream(Default::default())), + ..Default::default() + } +} + #[test] fn streams_jfk_committed_text() { let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) @@ -33,7 +51,7 @@ fn streams_jfk_committed_text() { }; let model = Model::load(&model_path).unwrap(); assert!( - model.capabilities().supports_streaming, + model.capabilities().unwrap().supports_streaming, "model does not advertise streaming" ); @@ -161,22 +179,32 @@ fn concurrent_compute_on_one_model_is_refused() { let model = Model::load(&model_path).unwrap(); let mut s1 = model.session().unwrap(); let mut s2 = model.session().unwrap(); + let chunk = &pcm[..pcm.len().min(1600)]; + // A failed begin does not take the lease. + assert!(s1 + .stream(&RunOptions::default(), &wrong_family_stream()) + .is_err()); let mut stream1 = s1 .stream(&RunOptions::default(), &StreamOptions::default()) .unwrap(); - stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + stream1.feed(chunk).unwrap(); - // Second stream on the same model while the first is live -> Busy. - match s2.stream(&RunOptions::default(), &StreamOptions::default()) { - Err(transcribe_cpp::Error::Busy(_)) => {} - other => panic!("expected Error::Busy for a second stream, got {other:?}"), - } - // An offline run on the same model while a stream is live -> Busy too. - match s2.run(&pcm, &RunOptions::default()) { - Err(transcribe_cpp::Error::Busy(_)) => {} - other => panic!("expected Error::Busy for a run mid-stream, got {other:?}"), - } + // Each refusal names the refused call; the busy check precedes native + // validation of a (wrong-family) stream begin. + expect_busy(s2.run(&pcm, &RunOptions::default()), RUN_BUSY); + expect_busy(s2.run_batch(&[&pcm], &RunOptions::default()), BATCH_BUSY); + expect_busy( + s2.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + expect_busy( + s2.stream(&RunOptions::default(), &wrong_family_stream()), + STREAM_BUSY, + ); + // The refusals did not disturb the live stream. + stream1.feed(chunk).unwrap(); + assert_eq!(stream1.state(), StreamState::Active); // Dropping the first stream releases the model's compute lease; now s2 can // begin its own stream. @@ -252,10 +280,46 @@ fn stream_family_extension_accepted_or_rejected() { drop(ok); // Wrong-family extension is rejected. - let wrong = StreamOptions { - family: Some(StreamExtension::ParakeetStream(Default::default())), - ..Default::default() + let err = session.stream(&RunOptions::default(), &wrong_family_stream()); + assert!(matches!(err, Err(Error::InvalidArgument(_))), "{err:?}"); +} + +#[test] +fn ended_stream_never_releases_another_sessions_lease() { + // After finalize() the lease belongs to whoever takes it next; a later + // reset() or drop of the ended stream must not free that other lease. + let (Some(model_path), Some(pcm)) = (common::smoke_streaming_model(), common::smoke_audio()) + else { + return; }; - let err = session.stream(&RunOptions::default(), &wrong); - assert!(err.is_err(), "wrong-family ext should be rejected"); + let model = Model::load(&model_path).unwrap(); + let mut s1 = model.session().unwrap(); + let mut s2 = model.session().unwrap(); + let mut s3 = model.session().unwrap(); + + let mut stream1 = s1 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.feed(&pcm[..pcm.len().min(1600)]).unwrap(); + stream1.finalize().unwrap(); + let stream2 = s2 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + stream1.reset(); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + drop(stream1); + expect_busy( + s3.stream(&RunOptions::default(), &StreamOptions::default()), + STREAM_BUSY, + ); + + // Releasing the real holder frees the model. + drop(stream2); + let stream3 = s3 + .stream(&RunOptions::default(), &StreamOptions::default()) + .unwrap(); + assert_eq!(stream3.state(), StreamState::Active); } diff --git a/bindings/rust/transcribe-cpp/tests/transcribe.rs b/bindings/rust/transcribe-cpp/tests/transcribe.rs index 2a10642aa..f81f4fa5d 100644 --- a/bindings/rust/transcribe-cpp/tests/transcribe.rs +++ b/bindings/rust/transcribe-cpp/tests/transcribe.rs @@ -68,7 +68,7 @@ fn capabilities_and_identity() { let model = Model::load(&model_path).unwrap(); assert_eq!(model.arch(), "whisper", "arch: {}", model.arch()); assert!(model.backend().to_lowercase().contains("cpu") || !model.backend().is_empty()); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); assert!(caps.native_sample_rate > 0, "{caps:?}"); } @@ -196,7 +196,7 @@ fn requested_timestamps_populate_rows() { return; }; let model = Model::load(&model_path).unwrap(); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); // Request the finest granularity the model actually supports (a request // finer than max_timestamp_kind correctly returns Unsupported, status 12). if caps.max_timestamp_kind == TimestampKind::None { @@ -228,7 +228,7 @@ fn timestamps_finer_than_supported_is_unsupported() { return; }; let model = Model::load(&model_path).unwrap(); - let caps = model.capabilities(); + let caps = model.capabilities().unwrap(); if caps.max_timestamp_kind == TimestampKind::Token { return; // already at the finest; nothing finer to ask for } @@ -266,8 +266,8 @@ fn close_ordering_drop_model_before_session() { #[test] fn shared_model_across_threads_serializes() { // Model is Send+Sync; the per-model mutex serializes the compute path. Two - // threads each run on their own session of one shared model — they queue - // rather than race, and both succeed. + // threads each run (one as a batch) on their own session of one shared + // model — they queue rather than race (never Busy), and both succeed. let Some((model_path, pcm)) = common::smoke_fixtures("shared_model_across_threads_serializes") else { return; @@ -275,12 +275,17 @@ fn shared_model_across_threads_serializes() { let model = Arc::new(Model::load(&model_path).unwrap()); let pcm = Arc::new(pcm); let handles: Vec<_> = (0..2) - .map(|_| { + .map(|i| { let model = Arc::clone(&model); let pcm = Arc::clone(&pcm); thread::spawn(move || { let mut s = model.session().unwrap(); - s.run(&pcm, &RunOptions::default()).unwrap().text + let opts = RunOptions::default(); + if i == 0 { + s.run(&pcm, &opts).unwrap().text + } else { + s.run_batch(&[&pcm], &opts).unwrap().remove(0).unwrap().text + } }) }) .collect(); diff --git a/bindings/swift/README.md b/bindings/swift/README.md index e647712a4..8a5bed710 100644 --- a/bindings/swift/README.md +++ b/bindings/swift/README.md @@ -5,12 +5,15 @@ a C/C++ speech-to-text library built on ggml. Native code ships as a prebuilt `.xcframework` SwiftPM `binaryTarget`, with Metal embedded on supported Apple slices. -> Status: in development (0.2.0). Core model, session, run, stream, +> Status: in development (0.3.0). Core model, session, run, stream, > cancellation, backend, and family-extension APIs are implemented and tested. Upgrading from 0.1? See the [0.2 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.2.md), -including the replacement of `gpuDevice` with exact `Device` values. +including the replacement of `gpuDevice` with exact `Device` values. From 0.3, +see the +[0.4 migration guide](https://github.com/handy-computer/transcribe.cpp/blob/main/docs/migrating-to-0.4.md) +(roles, `DiarizeSession`, throwing `capabilities`). ## Install @@ -23,7 +26,7 @@ custom artifact path through `TRANSCRIBE_XCFRAMEWORK_PATH`. The standalone SwiftPM mirror is planned but not published yet: ```swift -.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.2.0") +.package(url: "https://github.com/handy-computer/transcribe-cpp-swift.git", from: "0.3.0") ``` Until that mirror repo and tag exist, use the release xcframework directly when @@ -32,7 +35,7 @@ you only need the raw C module: ```swift .binaryTarget( name: "CTranscribe", - url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.2.0/TranscribeCpp.xcframework.zip", + url: "https://github.com/handy-computer/transcribe.cpp/releases/download/v0.3.0/TranscribeCpp.xcframework.zip", checksum: "" ) ``` @@ -98,6 +101,20 @@ let transcript = stream.snapshot // language, segments, words, tokens, timings Runnable examples live in `Sources/{transcribe-file,streaming,batch,backend-select,error-handling}`. +## Diarization + +A model's `roles` say what it serves (`.asr`, `.diarize`). Diarization models +such as Sortformer (DIARIZE only) return speaker turns from a `DiarizeSession`: + +```swift +let model = try Model(path: "diar_streaming_sortformer_4spk-v2.1-F32.gguf") +let turns = try model.diarizeSession().run( + pcm, options: DiarizeOptions(family: .sortformer(.init(preset: .veryHighLatency)))) +for t in turns { print(t.speakerId, t.t0Ms, t.t1Ms) } +``` + +`capabilities` and `session()` throw `.unsupportedRole` on a model without `.asr`. + ## Backends Backends are compiled into the xcframework per Apple slice: @@ -120,7 +137,8 @@ back to another primary device. runs queue; load one `Model` per worker for true parallelism. - `Session` is single-threaded. Use one session from one thread at a time. - An active `Stream` holds the model's compute lease until `finalize`, `reset`, - or drop. Other runs/streams on that model fail with `TranscribeError.busy`. + drop, or a `feed` that fails the stream. Other runs/streams on that model + fail with `TranscribeError.busy`. - `Transcribe.setLogHandler` is best installed at startup. Repeated calls are safe; they swap the Swift handler behind one native trampoline. - On Metal, do not keep models in globals in short-lived programs. Scope models diff --git a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift index d2c8808fe..5cae2c4dd 100644 --- a/bindings/swift/Sources/TranscribeCpp/ABIHash.swift +++ b/bindings/swift/Sources/TranscribeCpp/ABIHash.swift @@ -13,7 +13,7 @@ import CTranscribe extension Transcribe { /// sha256/16 of the normalized public FFI surface, pinned to the value in /// include/transcribe.abihash at the time this binding was last reviewed. - public static let pinnedHeaderHash = "57b1af43650f195d" + public static let pinnedHeaderHash = "bd3273dabb25a1fe" /// The public-ABI digest this binding was reviewed against (16 hex chars). public static func headerHash() -> String { pinnedHeaderHash } diff --git a/bindings/swift/Sources/TranscribeCpp/Backend.swift b/bindings/swift/Sources/TranscribeCpp/Backend.swift index 9aeef03f1..9eab54f63 100644 --- a/bindings/swift/Sources/TranscribeCpp/Backend.swift +++ b/bindings/swift/Sources/TranscribeCpp/Backend.swift @@ -110,6 +110,9 @@ public enum AbiStruct: Sendable { case sessionLimits case ext case deviceInfo + case diarizeInfo + case diarizeSessionParams + case diarizeParams var cValue: transcribe_abi_struct { switch self { @@ -127,6 +130,9 @@ public enum AbiStruct: Sendable { case .sessionLimits: return TRANSCRIBE_ABI_SESSION_LIMITS case .ext: return TRANSCRIBE_ABI_EXT case .deviceInfo: return TRANSCRIBE_ABI_DEVICE_INFO + case .diarizeInfo: return TRANSCRIBE_ABI_DIARIZE_INFO + case .diarizeSessionParams: return TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS + case .diarizeParams: return TRANSCRIBE_ABI_DIARIZE_PARAMS } } } diff --git a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift index de1fae26b..fe5221932 100644 --- a/bindings/swift/Sources/TranscribeCpp/Cancellation.swift +++ b/bindings/swift/Sources/TranscribeCpp/Cancellation.swift @@ -1,11 +1,11 @@ import CTranscribe import os -/// A thread-safe cancellation flag. Install it on a `Session` with -/// `setCancellationToken`; the native abort callback (polled between decode -/// steps / chunks) reads it, so `cancel()` may be called from any thread to -/// abort an in-flight run/stream. The run then throws `.aborted` with the -/// partial transcript preserved. +/// A thread-safe cancellation flag. Install it on a `Session` or a +/// `DiarizeSession` with `setCancellationToken`; the native abort callback +/// (polled between decode steps / chunks) reads it, so `cancel()` may be called +/// from any thread to abort an in-flight run/stream. The run then throws +/// `.aborted` (a `Session` run with the partial transcript preserved). public final class CancellationToken: @unchecked Sendable { private let cancelled = OSAllocatedUnfairLock(initialState: false) @@ -38,3 +38,18 @@ extension Session { cancelToken = nil } } + +extension DiarizeSession { + /// Install a cancellation token; a cancelled run throws `.aborted`. + public func setCancellationToken(_ token: CancellationToken) { + cancelToken = token + let context = Unmanaged.passUnretained(token).toOpaque() + transcribe_diarize_set_abort_callback(ptr, abortTrampoline, context) + } + + /// Remove any installed cancellation token. + public func clearCancellationToken() { + transcribe_diarize_set_abort_callback(ptr, nil, nil) + cancelToken = nil + } +} diff --git a/bindings/swift/Sources/TranscribeCpp/Diarize.swift b/bindings/swift/Sources/TranscribeCpp/Diarize.swift new file mode 100644 index 000000000..2bb6be4ce --- /dev/null +++ b/bindings/swift/Sources/TranscribeCpp/Diarize.swift @@ -0,0 +1,110 @@ +import CTranscribe +import Foundation + +/// Static facts about a diarization model. +public struct DiarizeInfo: Sendable, Equatable { + /// Input PCM rate (16000). + public let sampleRate: Int32 + /// `SpeakerSegment.speakerId` is in `1...maxSpeakers`. + public let maxSpeakers: Int32 +} + +public struct DiarizeOptions: Sendable { + public var family: DiarizeExtension? + public init(family: DiarizeExtension? = nil) { self.family = family } +} + +extension Model { + /// Throws `.unsupportedRole` when `roles` lacks `.diarize`. + public var diarizeInfo: DiarizeInfo { + get throws { + var info = transcribe_diarize_info() + transcribe_diarize_info_init(&info) + try TranscribeError.check(transcribe_diarize_get_info(ptr, &info), context: "diarize_get_info") + return DiarizeInfo(sampleRate: info.sample_rate, maxSpeakers: info.max_speakers) + } + } + + /// Create a DIARIZE session (`threads` 0 = library default). Throws + /// `.unsupportedRole` when `roles` lacks `.diarize`. + public func diarizeSession(threads: Int32 = 0) throws -> DiarizeSession { + var params = transcribe_diarize_session_params() + transcribe_diarize_session_params_init(¶ms) + params.n_threads = threads + var out: OpaquePointer? + try TranscribeError.check( + transcribe_diarize_session_init(ptr, ¶ms, &out), context: "creating diarize session") + guard let out else { + throw TranscribeError.other(status: 0, message: "null diarize session handle") + } + return DiarizeSession(model: self, ptr: out) + } +} + +/// A DIARIZE-role session: who spoke when. Same threading and lifetime +/// contract as `Session` (single-threaded; holds its `Model` alive), and its +/// runs share the model's compute lock and stream lease with every `Session`. +public final class DiarizeSession { + let model: Model + let ptr: OpaquePointer + /// Strong ref to the installed cancellation token (see `Session`). + var cancelToken: CancellationToken? + + init(model: Model, ptr: OpaquePointer) { + self.model = model + self.ptr = ptr + } + + deinit { transcribe_diarize_session_free(ptr) } + + /// Diarize one recording (16 kHz mono float32). Rows are grouped by + /// speaker, time-ordered within a speaker; speakers may overlap. + public func run(_ pcm: [Float], options: DiarizeOptions = .init()) throws -> [SpeakerSegment] { + try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before diarize run()" + ) { + let status = withDiarizeExtension(options.family) { ext in + var params = transcribe_diarize_params() + transcribe_diarize_params_init(¶ms) + params.family = ext + return pcm.withUnsafeBufferPointer { + transcribe_diarize_run(ptr, $0.baseAddress, Int32($0.count), ¶ms) + } + } + try TranscribeError.check(status, context: "diarize_run") + return (0.. [SpeakerSegment] { + nonisolated(unsafe) let this = self + let bridged = (cancelToken == nil) ? CancellationToken() : nil + if let bridged { setCancellationToken(bridged) } + defer { if bridged != nil { clearCancellationToken() } } + return try await withTaskCancellationHandler { + try await withCheckedThrowingContinuation { + (cont: CheckedContinuation<[SpeakerSegment], Error>) in + DispatchQueue.global().async { + cont.resume(with: Result { try this.run(pcm, options: options) }) + } + } + } onCancel: { + bridged?.cancel() + } + } + + /// Timings from the most recent run (`decodeMs` is 0). + public var timings: Timings { + var t = transcribe_timings(); transcribe_timings_init(&t) + _ = transcribe_diarize_get_timings(ptr, &t) + return Timings(t) + } +} diff --git a/bindings/swift/Sources/TranscribeCpp/Family.swift b/bindings/swift/Sources/TranscribeCpp/Family.swift index 59233a88f..7f4165c91 100644 --- a/bindings/swift/Sources/TranscribeCpp/Family.swift +++ b/bindings/swift/Sources/TranscribeCpp/Family.swift @@ -7,7 +7,7 @@ import CTranscribe // `transcribe_ext` pointer is handed to the run/begin call; the library copies // what it needs before returning. -// MARK: - Run-slot extensions (whisper, sortformer) +// MARK: - Run-slot extensions (whisper) public struct WhisperRunOptions: Sendable { public var initialPrompt: String? @@ -46,42 +46,12 @@ public struct WhisperRunOptions: Sendable { } } -/// Sortformer streaming operating point (latency / accuracy trade-off). -/// The menu is discrete (jointly-tuned bundles), not a latency dial; -/// `.default` keeps the GGUF-shipped checkpoint configuration. -/// `.veryHighLatency` (~30 s lookahead) is the offline-file operating -/// point; `.lowLatency` (~1 s) is the real-time point. -public enum SortformerPreset: Sendable { - case `default` - case veryHighLatency - case highLatency - case lowLatency - - var cValue: transcribe_sortformer_preset { - switch self { - case .default: return TRANSCRIBE_SORTFORMER_PRESET_DEFAULT - case .veryHighLatency: return TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY - case .highLatency: return TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY - case .lowLatency: return TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY - } - } -} - -/// Sortformer diarizer options (run slot). Sortformer produces speaker -/// segments, no text; read results via the speaker-segment accessors. -public struct SortformerStreamOptions: Sendable { - public var preset: SortformerPreset? - public init(preset: SortformerPreset? = nil) { self.preset = preset } -} - public enum RunExtension: Sendable { case whisper(WhisperRunOptions) - case sortformer(SortformerStreamOptions) var kind: UInt32 { switch self { case .whisper: return TRANSCRIBE_EXT_KIND_WHISPER_RUN - case .sortformer: return TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM } } } @@ -109,11 +79,6 @@ func withRunExtension( c.initial_prompt = prompt return try withUnsafePointer(to: &c.ext) { try body($0) } } - case .sortformer(let o): - var c = transcribe_sortformer_stream_ext() - transcribe_sortformer_stream_ext_init(&c) - if let v = o.preset { c.preset = v.cValue } - return try withUnsafePointer(to: &c.ext) { try body($0) } } } @@ -193,6 +158,58 @@ func withStreamExtension( } } +// MARK: - DIARIZE_RUN-slot extensions + +/// Sortformer streaming operating point (latency / accuracy trade-off). +/// The menu is discrete (jointly-tuned bundles), not a latency dial; +/// `.default` keeps the GGUF-shipped checkpoint configuration. +/// `.veryHighLatency` (~30 s lookahead) is the offline-file operating +/// point; `.lowLatency` (~1 s) is the real-time point. +public enum SortformerPreset: Sendable { + case `default` + case veryHighLatency + case highLatency + case lowLatency + + var cValue: transcribe_sortformer_preset { + switch self { + case .default: return TRANSCRIBE_SORTFORMER_PRESET_DEFAULT + case .veryHighLatency: return TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY + case .highLatency: return TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY + case .lowLatency: return TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY + } + } +} + +/// Sortformer options for `DiarizeSession.run`. +public struct SortformerDiarizeOptions: Sendable { + public var preset: SortformerPreset? + public init(preset: SortformerPreset? = nil) { self.preset = preset } +} + +public enum DiarizeExtension: Sendable { + case sortformer(SortformerDiarizeOptions) + + var kind: UInt32 { + switch self { + case .sortformer: return TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE + } + } +} + +func withDiarizeExtension( + _ ext: DiarizeExtension?, _ body: (UnsafePointer?) throws -> R +) rethrows -> R { + guard let ext else { return try body(nil) } + switch ext { + case .sortformer(let o): + var c = transcribe_sortformer_diarize_ext() + transcribe_sortformer_diarize_ext_init(&c) + if let v = o.preset { c.preset = v.cValue } + return try withUnsafePointer(to: &c.ext) { try body($0) } + } +} + // MARK: - Acceptance probe extension Model { @@ -204,4 +221,8 @@ extension Model { public func accepts(_ family: StreamExtension) -> Bool { transcribe_model_accepts_ext_kind(ptr, TRANSCRIBE_EXT_SLOT_STREAM, family.kind) } + /// Whether this model accepts the given extension on the DIARIZE_RUN slot. + public func accepts(_ family: DiarizeExtension) -> Bool { + transcribe_model_accepts_ext_kind(ptr, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, family.kind) + } } diff --git a/bindings/swift/Sources/TranscribeCpp/Model.swift b/bindings/swift/Sources/TranscribeCpp/Model.swift index 6045b17b0..34ef8dc7e 100644 --- a/bindings/swift/Sources/TranscribeCpp/Model.swift +++ b/bindings/swift/Sources/TranscribeCpp/Model.swift @@ -1,6 +1,15 @@ import CTranscribe import Foundation +/// The kinds of work a model serves (`transcribe_model_roles`). ASR is +/// `Session`; DIARIZE is `DiarizeSession`. +public struct Roles: OptionSet, Sendable { + public let rawValue: UInt32 + public init(rawValue: UInt32) { self.rawValue = rawValue } + public static let asr = Roles(rawValue: TRANSCRIBE_ROLE_ASR.rawValue) + public static let diarize = Roles(rawValue: TRANSCRIBE_ROLE_DIARIZE.rawValue) +} + /// A loaded model. Safe to share across threads (`@unchecked Sendable`): the C /// API allows concurrent queries and session creation, and the compute path is /// serialized by an internal lock (the C "one in-flight run per model" @@ -10,17 +19,32 @@ import Foundation public final class Model: @unchecked Sendable { let ptr: OpaquePointer /// Serializes the run/feed/finalize compute path across all sessions, and - /// guards `streamActive`. + /// guards `streamActive`. Taken only through `withCompute`. let runLock = NSLock() /// The compute lease: `true` while some session holds an ACTIVE stream. /// The C contract allows at most one in-flight run/stream across ALL /// sessions of a model, and an active stream spans begin..finalize/reset/ - /// drop — so `run`/`runBatch`/another `stream` are refused with `.busy` - /// while it is held, rather than racing into the documented UB (corrupted - /// decodes on CPU, command-buffer failures on Metal). Always accessed under - /// `runLock`. + /// drop — so `run`/`runBatch`/another `stream`/a diarize `run` are refused + /// with `.busy` while it is held, rather than racing into the documented UB + /// (corrupted decodes on CPU, command-buffer failures on Metal). Always + /// accessed under `runLock`. var streamActive = false + /// Run `body` (a native compute call and its copy-out) under the model-wide + /// compute lock; `busyIfStreaming` then refuses an active stream with `.busy`. + func withCompute(_ body: () throws -> R) rethrows -> R { + runLock.lock() + defer { runLock.unlock() } + return try body() + } + + func withCompute(busyIfStreaming message: String, _ body: () throws -> R) throws -> R { + try withCompute { + if streamActive { throw TranscribeError.busy(message) } + return try body() + } + } + /// Load a model from a GGUF file. Runs the pre-1.0 version gate first. public init(path: String, options: ModelOptions = .init()) throws { try Transcribe.ensureCompatible() @@ -55,11 +79,18 @@ public final class Model: @unchecked Sendable { return Session(model: self, ptr: out) } + /// The roles this model serves, fixed at load. + public var roles: Roles { Roles(rawValue: transcribe_model_roles(ptr)) } + + /// ASR capabilities. Throws `.unsupportedRole` when `roles` lacks `.asr`. public var capabilities: Capabilities { - var caps = transcribe_capabilities() - transcribe_capabilities_init(&caps) - _ = transcribe_model_get_capabilities(ptr, &caps) - return Capabilities(caps) + get throws { + var caps = transcribe_capabilities() + transcribe_capabilities_init(&caps) + try TranscribeError.check( + transcribe_model_get_capabilities(ptr, &caps), context: "capabilities") + return Capabilities(caps) + } } public func supports(_ feature: Feature) -> Bool { diff --git a/bindings/swift/Sources/TranscribeCpp/Session.swift b/bindings/swift/Sources/TranscribeCpp/Session.swift index 9357899d3..e8393b102 100644 --- a/bindings/swift/Sources/TranscribeCpp/Session.swift +++ b/bindings/swift/Sources/TranscribeCpp/Session.swift @@ -51,17 +51,15 @@ public final class Session { /// Transcribe one utterance. `pcm` is mono float32 at 16 kHz in [-1, 1]. public func run(_ pcm: [Float], options: RunOptions = .init()) throws -> Transcript { try options.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before run()") - } - return try options.withCParams { params in - let status = pcm.withUnsafeBufferPointer { - transcribe_run(ptr, $0.baseAddress, Int32($0.count), params) + return try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before run()" + ) { + try options.withCParams { params in + let status = pcm.withUnsafeBufferPointer { + transcribe_run(ptr, $0.baseAddress, Int32($0.count), params) + } + return try makeTranscript(status, context: "run") } - return try makeTranscript(status, context: "run") } } @@ -72,46 +70,47 @@ public final class Session { _ inputs: [[Float]], options: RunOptions = .init() ) throws -> [Result] { try options.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy( - "a stream is active on this model; finish or drop it before runBatch()") - } - return try options.withCParams { params in - let counts = inputs.map { Int32($0.count) } - let status = withPCMPointers(inputs[...], []) { pointers in - pointers.withUnsafeBufferPointer { pp in - counts.withUnsafeBufferPointer { cc in - transcribe_run_batch( - ptr, pp.baseAddress, cc.baseAddress, Int32(inputs.count), params) + return try model.withCompute( + busyIfStreaming: "a stream is active on this model; finish or drop it before runBatch()" + ) { + try options.withCParams { params in + let counts = inputs.map { Int32($0.count) } + let status = withPCMPointers(inputs[...], []) { pointers in + pointers.withUnsafeBufferPointer { pp in + counts.withUnsafeBufferPointer { cc in + transcribe_run_batch( + ptr, pp.baseAddress, cc.baseAddress, Int32(inputs.count), params) + } } } - } - if status != TRANSCRIBE_OK && status != TRANSCRIBE_ERR_ABORTED { - throw TranscribeError.make(status, context: "run_batch") - } - let n = Int(transcribe_batch_n_results(ptr)) - return (0.. StreamUpdate { - session.model.runLock.lock() - defer { session.model.runLock.unlock() } - var update = transcribe_stream_update() - transcribe_stream_update_init(&update) - let status = frame.withUnsafeBufferPointer { - transcribe_stream_feed(session.ptr, $0.baseAddress, Int32($0.count), &update) + try session.model.withCompute { + var update = transcribe_stream_update() + transcribe_stream_update_init(&update) + let status = frame.withUnsafeBufferPointer { + transcribe_stream_feed(session.ptr, $0.baseAddress, Int32($0.count), &update) + } + if holdsLease && transcribe_stream_get_state(session.ptr) == TRANSCRIBE_STREAM_FAILED { + session.model.streamActive = false; holdsLease = false + } + try TranscribeError.check(status, context: "stream_feed") + return StreamUpdate(update) } - try TranscribeError.check(status, context: "stream_feed") - return StreamUpdate(update) } /// Signal end of input; flushes buffered audio and emits remaining text. @@ -138,24 +145,24 @@ public final class Stream { /// proceed without waiting for this `Stream` to drop. @discardableResult public func finalize() throws -> StreamUpdate { - session.model.runLock.lock() - defer { session.model.runLock.unlock() } - var update = transcribe_stream_update() - transcribe_stream_update_init(&update) - let status = transcribe_stream_finalize(session.ptr, &update) - if holdsLease { session.model.streamActive = false; holdsLease = false } - try TranscribeError.check(status, context: "stream_finalize") - return StreamUpdate(update) + try session.model.withCompute { + var update = transcribe_stream_update() + transcribe_stream_update_init(&update) + let status = transcribe_stream_finalize(session.ptr, &update) + if holdsLease { session.model.streamActive = false; holdsLease = false } + try TranscribeError.check(status, context: "stream_finalize") + return StreamUpdate(update) + } } /// Abandon the stream and return the session to idle. Releases the model's /// compute lease (the stream is no longer active). @discardableResult public func reset() -> StreamState { - session.model.runLock.lock() - transcribe_stream_reset(session.ptr) - if holdsLease { session.model.streamActive = false; holdsLease = false } - session.model.runLock.unlock() + session.model.withCompute { + transcribe_stream_reset(session.ptr) + if holdsLease { session.model.streamActive = false; holdsLease = false } + } return state } @@ -187,23 +194,20 @@ extension Session { /// `.notImplemented`) and be idle/finished/failed (not already streaming). /// Claims the model's compute lease for the whole stream lifetime: a second /// stream — or an offline run — on ANY session of the same model is refused - /// with `.busy` until this stream finalizes, resets, or is dropped. + /// with `.busy` until this stream finalizes, fails, resets, or is dropped. public func stream( _ runOptions: RunOptions = .init(), _ streamOptions: StreamOptions = .init() ) throws -> Stream { try runOptions.checkCStrings() - model.runLock.lock() - defer { model.runLock.unlock() } - if model.streamActive { - throw TranscribeError.busy("a stream is already active on this model") - } - let status = runOptions.withCParams { runParams in - streamOptions.withCParams { streamParams in - transcribe_stream_begin(ptr, runParams, streamParams) + return try model.withCompute(busyIfStreaming: "a stream is already active on this model") { + let status = runOptions.withCParams { runParams in + streamOptions.withCParams { streamParams in + transcribe_stream_begin(ptr, runParams, streamParams) + } } + try TranscribeError.check(status, context: "stream_begin") + model.streamActive = true + return Stream(self) } - try TranscribeError.check(status, context: "stream_begin") - model.streamActive = true - return Stream(self) } } diff --git a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift index ce2c8b032..4e0db09c9 100644 --- a/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift +++ b/bindings/swift/Sources/TranscribeCpp/TranscribeError.swift @@ -28,6 +28,8 @@ public enum TranscribeError: Error { /// The decode was stopped because the output began repeating itself; the /// repeats are dropped from `partial`, which is incomplete. case outputRepetition(message: String, partial: Transcript?) + /// The model's `roles` lack the one the call needs (`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`). + case unsupportedRole(String) case versionMismatch(String) case busy(String) case other(status: Int32, message: String) @@ -69,6 +71,8 @@ public enum TranscribeError: Error { return .outputTruncated(message: message, partial: nil) case TRANSCRIBE_ERR_OUTPUT_REPETITION: return .outputRepetition(message: message, partial: nil) + case TRANSCRIBE_ERR_UNSUPPORTED_ROLE: + return .unsupportedRole(message) default: return .other(status: raw, message: message) } diff --git a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift index e133008d2..c571419b5 100644 --- a/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/CancelTests.swift @@ -77,6 +77,7 @@ final class CancelTests: XCTestCase { } } XCTAssertTrue(session.wasAborted) + XCTAssertNil(session.cancelToken, "the bridged token is removed after the call") } /// The bridge must never clobber a caller-installed token: there is a single diff --git a/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift new file mode 100644 index 000000000..1315e240b --- /dev/null +++ b/bindings/swift/Tests/TranscribeCppTests/DiarizeTests.swift @@ -0,0 +1,91 @@ +import XCTest + +@testable import TranscribeCpp + +/// DIARIZE role (`DiarizeSession`). +final class DiarizeTests: XCTestCase { + /// `p` is NaN on diarize rows, so compare the turns, not `SpeakerSegment`s. + private func turns(_ rows: [SpeakerSegment]) -> [[Int64]] { + rows.map { [$0.t0Ms, $0.t1Ms, Int64($0.speakerId)] } + } + + private func assertUnsupportedRole(_ calls: [(String, () throws -> Void)]) { + for (what, call) in calls { + XCTAssertThrowsError(try call(), what) { error in + guard case TranscribeError.unsupportedRole = error else { + return XCTFail("\(what): expected .unsupportedRole, got \(error)") + } + } + } + } + + func testAsrOnlyModelRefusesDiarize() throws { + guard let path = Fixtures.modelPath() else { throw XCTSkip("no canary model") } + let model = try Model(path: path) + XCTAssertEqual(model.roles, .asr) + XCTAssertFalse(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) + assertUnsupportedRole([ + ("diarizeInfo", { _ = try model.diarizeInfo }), + ("diarizeSession", { _ = try model.diarizeSession() }), + ]) + } + + func testSortformerDiarizes() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + // The golden turns are CPU numerics. + let model = try Model(path: path, options: ModelOptions(backend: .cpu)) + XCTAssertEqual(model.roles, [.diarize]) + XCTAssertEqual(try model.diarizeInfo, DiarizeInfo(sampleRate: 16000, maxSpeakers: 4)) + XCTAssertTrue(model.accepts(DiarizeExtension.sortformer(SortformerDiarizeOptions()))) + assertUnsupportedRole([ + ("capabilities", { _ = try model.capabilities }), + ("session", { _ = try model.session() }), + ]) + + // Segment values are pinned by the C test; here, check the rows are sane + // and that the preset reaches the native run (.lowLatency moves turns). + let session = try model.diarizeSession() + let base = try session.run(pcm) + XCTAssertFalse(base.isEmpty) + for row in base { + XCTAssertTrue((1...4).contains(row.speakerId), "\(row)") + XCTAssertLessThan(row.t0Ms, row.t1Ms, "\(row)") + } + XCTAssertGreaterThan(session.timings.encodeMs, 0) + let low = try session.run( + pcm, options: DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(preset: .lowLatency)))) + XCTAssertNotEqual(turns(low), turns(base), ".lowLatency must reach the native run") + } + + func testCancelledRunAbortsAndRecovers() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + let session = try Model(path: path).diarizeSession() + let token = CancellationToken() + token.cancel() + session.setCancellationToken(token) + XCTAssertThrowsError(try session.run(pcm)) { error in + guard case TranscribeError.aborted = error else { + return XCTFail("expected .aborted, got \(error)") + } + } + session.clearCancellationToken() + XCTAssertFalse(try session.run(pcm).isEmpty) + } + + /// A diarize run shares the model's stream lease with ASR sessions. + func testDiarizeRunIsBusyWhileAStreamIsActive() throws { + let (path, pcm) = try Fixtures.sortformerModelAndAudio() + let model = try Model(path: path) + let session = try model.diarizeSession() + // Sortformer cannot stream, so set the lease a stream would hold directly. + model.withCompute { model.streamActive = true } + defer { model.withCompute { model.streamActive = false } } + XCTAssertThrowsError(try session.run(pcm)) { error in + guard case TranscribeError.busy(let message) = error else { + return XCTFail("expected .busy, got \(error)") + } + XCTAssertEqual( + message, "a stream is active on this model; finish or drop it before diarize run()") + } + } +} diff --git a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift index 61fc38e15..77d5d0f5a 100644 --- a/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/NoModelTests.swift @@ -1,5 +1,6 @@ import Foundation import XCTest +import CTranscribe @testable import TranscribeCpp @@ -48,6 +49,29 @@ final class NoModelTests: XCTestCase { XCTAssertNil(TranscribeError.inputTooLong("").partial) } + // Every native status maps to its case (compared by case name); the + // "unknown status" check below flags a newly appended C status. + func testEveryStatusMapsToItsCase() { + let expected: [Int32: String] = [ + 1: "invalidArgument", 2: "notImplemented", 3: "modelFileNotFound", + 4: "modelLoad", 5: "modelLoad", 6: "modelLoad", 7: "outOfMemory", + 8: "backend", + 9: "other", // SAMPLE_RATE: reserved, never returned; not mapped + 10: "unsupported", 11: "unsupported", 12: "unsupported", + 13: "aborted", 14: "badStructSize", 15: "unsupported", + 16: "unsupported", 17: "inputTooLong", 18: "outputTruncated", + 19: "outputRepetition", 20: "unsupportedRole", + ] + for raw in 1...20 { + let status = transcribe_status(rawValue: UInt32(raw)) + XCTAssertNotEqual(Transcribe.statusString(Int32(raw)), "unknown status", "status \(raw)") + let name = String(describing: TranscribeError.make(status)).prefix { $0 != "(" } + XCTAssertEqual(String(name), expected[Int32(raw)], "status \(raw)") + } + XCTAssertEqual(Transcribe.statusString(21), "unknown status", + "a new status was appended; map it in TranscribeError.make") + } + func testAtLeastOneDevice() { XCTAssertGreaterThanOrEqual(Transcribe.devices().count, 1) } diff --git a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift index d18743972..d0476924a 100644 --- a/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/StreamingTests.swift @@ -79,48 +79,6 @@ final class StreamingTests: XCTestCase { // MARK: - Compute lease (C contract: one in-flight run/stream per model) - /// Regression for the streaming compute lease. The C library allows at most - /// one in-flight run/stream across ALL sessions of a model, and an active - /// stream spans begin..finalize/reset/drop. Before the lease, a second - /// stream — or an offline run — on another session of the same model began - /// concurrently and raced into the documented UB (corrupted decodes / Metal - /// command-buffer failures); now it is refused with `.busy`. Mirrors Rust's - /// `concurrent_compute_on_one_model_is_refused`. - func testConcurrentComputeOnOneModelIsRefused() throws { - let (path, pcm) = try Fixtures.streamingModelAndAudio() - let model = try Model(path: path) - let s1 = try model.session() - let s2 = try model.session() - - let stream1 = try s1.stream() - _ = try stream1.feed(Array(pcm.prefix(1600))) // s1's stream is ACTIVE - - // A second stream on the same model while the first is live -> .busy. - XCTAssertThrowsError(try s2.stream()) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a second stream, got \(error)") - } - } - // An offline run on the same model while a stream is live -> .busy too. - XCTAssertThrowsError(try s2.run(pcm)) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a run mid-stream, got \(error)") - } - } - // runBatch is gated on the same lease. - XCTAssertThrowsError(try s2.runBatch([pcm])) { error in - guard case TranscribeError.busy = error else { - return XCTFail("expected .busy for a runBatch mid-stream, got \(error)") - } - } - - // Releasing the first stream frees the lease; s2 can now stream. - stream1.reset() - XCTAssertFalse(model.streamActive) - let stream2 = try s2.stream() - stream2.reset() - } - /// 2b: a `Stream` dropped without `finalize()`/`reset()` must reset the /// session and release the model's compute lease in `deinit` — otherwise /// the session is wedged ACTIVE forever and the model stays `.busy`. @@ -173,4 +131,31 @@ final class StreamingTests: XCTestCase { _ = stream1 } } + + /// While a stream is active, run / runBatch / stream on any session of the + /// model throw `.busy`. + func testActiveStreamRefusesSiblingCompute() throws { + let (path, pcm) = try Fixtures.streamingModelAndAudio() + let model = try Model(path: path) + let s1 = try model.session() + let s2 = try model.session() + let active = try s1.stream() + _ = try active.feed(Array(pcm.prefix(1600))) + + let cases: [(String, () throws -> Void)] = [ + ("a stream is active on this model; finish or drop it before run()", { _ = try s2.run(pcm) }), + ("a stream is active on this model; finish or drop it before runBatch()", + { _ = try s2.runBatch([pcm]) }), + ("a stream is already active on this model", { _ = try s2.stream() }), + ] + for (message, call) in cases { + XCTAssertThrowsError(try call()) { error in + guard case TranscribeError.busy(let got) = error else { + return XCTFail("expected .busy, got \(error)") + } + XCTAssertEqual(got, message) + } + } + active.reset() + } } diff --git a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift index 43eff078c..5d495ecaa 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TestSupport.swift @@ -86,6 +86,16 @@ enum Fixtures { "models/SenseVoiceSmall/SenseVoiceSmall-Q8_0.gguf") } + /// Sortformer (DIARIZE only) + its in-repo two-speaker sample. + static func sortformerModelAndAudio() throws -> (model: String, pcm: [Float]) { + guard let model = familyModel( + "TRANSCRIBE_SMOKE_SORTFORMER_MODEL", + "models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-F32.gguf") + else { throw XCTSkip("no Sortformer model (set TRANSCRIBE_SMOKE_SORTFORMER_MODEL)") } + let audio = repoRoot().appendingPathComponent("samples/sortformer-2spk-mix.wav").path + return (model, try loadWav(audio)) + } + /// The model path + decoded PCM, or `XCTSkip` when either is absent. static func modelAndAudio() throws -> (model: String, pcm: [Float]) { guard let model = modelPath() else { diff --git a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift index 2d8cbed54..74cb4c1ce 100644 --- a/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift +++ b/bindings/swift/Tests/TranscribeCppTests/TranscribeTests.swift @@ -24,7 +24,7 @@ final class TranscribeTests: XCTestCase { func testFinerThanSupportedTimestampsIsUnsupported() throws { let (path, pcm) = try Fixtures.modelAndAudio() let model = try Model(path: path) - guard let finer = finerThanSupported(model.capabilities.maxTimestampKind) else { + guard let finer = finerThanSupported(try model.capabilities.maxTimestampKind) else { throw XCTSkip("model already supports the finest timestamps") } let session = try model.session() @@ -61,7 +61,7 @@ final class TranscribeTests: XCTestCase { let model = try Model(path: path) XCTAssertFalse(model.arch.isEmpty) XCTAssertFalse(model.backend.isEmpty) - XCTAssertGreaterThan(model.capabilities.nativeSampleRate, 0) + XCTAssertGreaterThan(try model.capabilities.nativeSampleRate, 0) } func testPncChangesCanaryPrompt() throws { @@ -125,12 +125,16 @@ final class TranscribeTests: XCTestCase { let (path, pcm) = try Fixtures.modelAndAudio() // Drop the local Model reference; the Session's strong ref must keep the // native model alive (close-ordering safety under ARC). - let session: Session = try { + weak var weakModel: Model? + var session: Session? = try { let model = try Model(path: path) + weakModel = model return try model.session() }() - let transcript = try session.run(pcm) - XCTAssertTrue(transcript.text.lowercased().contains("country")) + XCTAssertNotNil(weakModel, "a live Session must keep its Model alive") + XCTAssertTrue(try session!.run(pcm).text.lowercased().contains("country")) + session = nil + XCTAssertNil(weakModel, "the Model is freed once its last Session is") } func testSharedModelAcrossThreadsSerializes() throws { diff --git a/bindings/typescript/README.md b/bindings/typescript/README.md index 5121e7999..0a33edcf8 100644 --- a/bindings/typescript/README.md +++ b/bindings/typescript/README.md @@ -109,10 +109,28 @@ const stream = await session.stream({ family: { kind: "moonshine" } }); model.accepts({ kind: "whisper" }); // does this model take that extension? ``` +### Diarization (DIARIZE role) + +`model.roles` lists what a model serves (`"asr"`, `"diarize"`; Sortformer is +diarize-only). Calls for a role the model lacks throw `UnsupportedRole`. + +```ts +const { sampleRate, maxSpeakers } = model.diarizeInfo; +using diarizer = model.createDiarizeSession(); +const turns = await diarizer.run(pcm, { + family: { kind: "sortformer_diarize", preset: "very_high_latency" }, +}); +for (const t of turns) console.log(t.speakerId, t.t0Ms, t.t1Ms); +``` + +`run` accepts a `signal`; cancelling throws `Aborted` (with no partial result). +`diarizer.timings` reports the last run. Diarize runs wait on the same +model-wide lock as other compute calls (see below). + ### Resource management -`TranscribeModel`, `Session`, and `Stream` all implement `Symbol.dispose`, so -`using` works (TypeScript 5.2+ / Node 22+): +`TranscribeModel`, `Session`, `DiarizeSession`, and `Stream` all implement +`Symbol.dispose`, so `using` works (TypeScript 5.2+ / Node 22+): ```ts using model = await TranscribeModel.load("model.gguf"); @@ -150,14 +168,14 @@ native compute to a **libuv worker thread** (via koffi's async calls), so the event loop stays responsive while inference runs. The C library allows **one compute in flight per model** — a `run`, a `runBatch`, -or an *active stream* — across all of its sessions. The binding enforces this: -every compute call serializes through an internal model-wide mutex, and an active -stream holds a model-wide lease for its whole lifetime. While a stream is active -(after `stream()`, before `finalize()`/`reset()`), a `run`/`runBatch`/`stream` on -any session of that model is refused with a `Busy` error rather than allowed to -race. So to parallelize, load one model per worker; to share a model, finalize or -reset the stream first. A single `Session` is single-use-at-a-time — don't call -`run`/`feed` on the same session concurrently. +a diarize `run`, or an *active stream* — across all of its sessions. The binding +enforces this: every compute call serializes through an internal model-wide +mutex, and an active stream holds a model-wide lease for its whole lifetime. +While a stream is active (after `stream()`, before `finalize()`/`reset()`), a +`run`/`runBatch`/`stream` on any session of that model is refused with a `Busy` +error rather than allowed to race. So to parallelize, load one model per worker; +to share a model, finalize or reset the stream first. A single `Session` is +single-use-at-a-time — don't call `run`/`feed` on the same session concurrently. Hand-offs are ordered, not racy: `finalize()`/`reset()`/`dispose()` release the lease only after the native teardown runs on the shared queue, so the slot is @@ -166,20 +184,24 @@ is correctly serialized — `stream.reset(); const next = await session.stream() works without awaiting the (void) `reset()`. The reverse order — beginning before the teardown — is refused with `Busy`, by design. +A `feed()` rejected up front (e.g. NaN samples) leaves the stream `"active"` and +the lease held; a failure inside the model makes it `"failed"` and frees the lease. + Because the compute is genuinely on another thread, **do not touch a session while a call against it is in flight** — it is single-threaded in the C library: -- Reading a stream's `text`/`snapshot`/`state`/`revision`/`lastStatus`, or a - session's `limits`/`wasAborted`, during an un-awaited - `feed`/`finalize`/`run`/`runBatch` **throws**. +- Reading a stream's `text`/`snapshot`/`state`/`revision`/`lastStatus`, a + session's `limits`/`wasAborted`, or a `DiarizeSession`'s `timings`, during an + un-awaited `feed`/`finalize`/`run`/`runBatch` **throws**. - `reset()` and `dispose()` are safe to call any time: the native teardown is deferred behind any in-flight call, so it never frees a session mid-compute. - Disposing a `Session` or `TranscribeModel` while a stream is still active releases the lease and invalidates the stream — its later calls throw rather than touch the freed handle. -- The **input PCM is borrowed, not copied**: `run`/`runBatch`/`feed` hand the - buffer to native code that reads it on the worker thread, so do not mutate it - (e.g. reuse a scratch/capture buffer) until the returned promise resolves. +- The **input PCM is borrowed, not copied**: `run`/`runBatch`/`feed` (and + `DiarizeSession.run`) hand the buffer to native code that reads it on the + worker thread, so do not mutate it (e.g. reuse a scratch/capture buffer) + until the returned promise resolves. Pass a fresh buffer per call, or `await` before overwriting. The normal pattern is safe — `await` first, then read: diff --git a/bindings/typescript/src/_generated.ts b/bindings/typescript/src/_generated.ts index e5e5be8aa..5e7ea6f31 100644 --- a/bindings/typescript/src/_generated.ts +++ b/bindings/typescript/src/_generated.ts @@ -11,7 +11,7 @@ // Stable digest of the ABI surface (structs, enums, macros, layout, // prototypes), computed by the Python oracle and pinned here so a header // ABI change turns this binding's drift check red for conscious review. -export const PUBLIC_HEADER_HASH = "57b1af43650f195d"; +export const PUBLIC_HEADER_HASH = "bd3273dabb25a1fe"; // === enum constants === export const TRANSCRIBE_OK = 0; @@ -34,6 +34,7 @@ export const TRANSCRIBE_ERR_UNSUPPORTED_ITN = 16; export const TRANSCRIBE_ERR_INPUT_TOO_LONG = 17; export const TRANSCRIBE_ERR_OUTPUT_TRUNCATED = 18; export const TRANSCRIBE_ERR_OUTPUT_REPETITION = 19; +export const TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20; export const TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0; export const TRANSCRIBE_ABI_SESSION_PARAMS = 1; export const TRANSCRIBE_ABI_RUN_PARAMS = 2; @@ -50,6 +51,9 @@ export const TRANSCRIBE_ABI_EXT = 12; export const TRANSCRIBE_ABI_DEVICE_INFO = 13; export const TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14; export const TRANSCRIBE_ABI_BACKEND_INIT_PARAMS = 15; +export const TRANSCRIBE_ABI_DIARIZE_INFO = 16; +export const TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 17; +export const TRANSCRIBE_ABI_DIARIZE_PARAMS = 18; export const TRANSCRIBE_LOG_LEVEL_NONE = 0; export const TRANSCRIBE_LOG_LEVEL_INFO = 1; export const TRANSCRIBE_LOG_LEVEL_WARN = 2; @@ -78,6 +82,7 @@ export const TRANSCRIBE_DIARIZE_MODE_OFF = 1; export const TRANSCRIBE_DIARIZE_MODE_ON = 2; export const TRANSCRIBE_EXT_SLOT_RUN = 0; export const TRANSCRIBE_EXT_SLOT_STREAM = 1; +export const TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2; export const TRANSCRIBE_BACKEND_AUTO = 0; export const TRANSCRIBE_BACKEND_CPU = 1; export const TRANSCRIBE_BACKEND_METAL = 2; @@ -89,6 +94,8 @@ export const TRANSCRIBE_DEVICE_TYPE_CPU = 0; export const TRANSCRIBE_DEVICE_TYPE_GPU = 1; export const TRANSCRIBE_DEVICE_TYPE_IGPU = 2; export const TRANSCRIBE_DEVICE_TYPE_ACCEL = 3; +export const TRANSCRIBE_ROLE_ASR = 1; +export const TRANSCRIBE_ROLE_DIARIZE = 2; export const TRANSCRIBE_FEATURE_INITIAL_PROMPT = 0; export const TRANSCRIBE_FEATURE_TEMPERATURE_FALLBACK = 1; export const TRANSCRIBE_FEATURE_LONG_FORM = 2; @@ -125,7 +132,7 @@ export const TRANSCRIBE_BACKEND_MASK_VULKAN = 4; export const TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM = 1414746957; export const TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM = 1396853584; export const TRANSCRIBE_EXT_KIND_PARAKEET_STREAM = 1414744912; -export const TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM = 1414743635; +export const TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE = 1380206163; export const TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM = 1414746710; export const TRANSCRIBE_EXT_KIND_WHISPER_RUN = 1314015319; @@ -147,10 +154,13 @@ export const STRUCT_LAYOUT: Record = { 'transcribe_word': { size: 48, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'seg_index': 24, 'first_token': 28, 'n_tokens': 32, 'text': 40} }, 'transcribe_token': { size: 48, align: 8, offsets: {'struct_size': 0, 'id': 8, 'p': 12, 't0_ms': 16, 't1_ms': 24, 'seg_index': 32, 'word_index': 36, 'text': 40} }, 'transcribe_speaker_segment': { size: 32, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'speaker_id': 24, 'p': 28} }, + 'transcribe_diarize_info': { size: 16, align: 8, offsets: {'struct_size': 0, 'sample_rate': 8, 'max_speakers': 12} }, + 'transcribe_diarize_session_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'n_threads': 8} }, + 'transcribe_diarize_params': { size: 16, align: 8, offsets: {'struct_size': 0, 'family': 8} }, 'transcribe_moonshine_streaming_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'min_decode_interval_ms': 16} }, 'transcribe_parakeet_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'att_context_right': 16} }, 'transcribe_parakeet_buffered_stream_ext': { size: 32, align: 8, offsets: {'ext': 0, 'left_ms': 16, 'chunk_ms': 20, 'right_ms': 24} }, - 'transcribe_sortformer_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, + 'transcribe_sortformer_diarize_ext': { size: 24, align: 8, offsets: {'ext': 0, 'preset': 16} }, 'transcribe_voxtral_realtime_stream_ext': { size: 24, align: 8, offsets: {'ext': 0, 'num_delay_tokens': 16, 'min_decode_interval_ms': 20} }, 'transcribe_whisper_run_ext': { size: 80, align: 8, offsets: {'ext': 0, 'initial_prompt': 16, 'prompt_tokens': 24, 'n_prompt_tokens': 32, 'prompt_condition': 40, 'condition_on_prev_tokens': 44, 'max_prev_context_tokens': 48, 'temperature': 52, 'temperature_inc': 56, 'compression_ratio_thold': 60, 'logprob_thold': 64, 'no_speech_thold': 68, 'seed': 72, 'max_initial_timestamp': 76} }, 'transcribe_whisper_chunk_trace': { size: 48, align: 8, offsets: {'struct_size': 0, 't0_ms': 8, 't1_ms': 16, 'temperature_used': 24, 'compression_ratio': 28, 'avg_logprob': 32, 'no_speech_prob': 36, 'no_speech_triggered': 40, 'n_fallbacks': 44} }, @@ -173,6 +183,9 @@ export const ABI_STRUCT_IDS: Record = { 'transcribe_word': 7, 'transcribe_token': 8, 'transcribe_speaker_segment': 14, + 'transcribe_diarize_info': 16, + 'transcribe_diarize_session_params': 17, + 'transcribe_diarize_params': 18, }; // Build koffi struct types; returns a name -> koffi.IKoffiCType map. @@ -194,10 +207,13 @@ export function defineTypes(koffi: any): Record { T['transcribe_word'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', seg_index: 'int', first_token: 'int', n_tokens: 'int', text: 'char *' }); T['transcribe_token'] = koffi.struct({ struct_size: 'uint64_t', id: 'int', p: 'float', t0_ms: 'int64_t', t1_ms: 'int64_t', seg_index: 'int', word_index: 'int', text: 'char *' }); T['transcribe_speaker_segment'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', speaker_id: 'int32_t', p: 'float' }); + T['transcribe_diarize_info'] = koffi.struct({ struct_size: 'uint64_t', sample_rate: 'int32_t', max_speakers: 'int32_t' }); + T['transcribe_diarize_session_params'] = koffi.struct({ struct_size: 'uint64_t', n_threads: 'int32_t' }); + T['transcribe_diarize_params'] = koffi.struct({ struct_size: 'uint64_t', family: 'void *' }); T['transcribe_moonshine_streaming_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], min_decode_interval_ms: 'int32_t' }); T['transcribe_parakeet_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], att_context_right: 'int32_t' }); T['transcribe_parakeet_buffered_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], left_ms: 'int32_t', chunk_ms: 'int32_t', right_ms: 'int32_t' }); - T['transcribe_sortformer_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); + T['transcribe_sortformer_diarize_ext'] = koffi.struct({ ext: T['transcribe_ext'], preset: 'int' }); T['transcribe_voxtral_realtime_stream_ext'] = koffi.struct({ ext: T['transcribe_ext'], num_delay_tokens: 'int32_t', min_decode_interval_ms: 'int32_t' }); T['transcribe_whisper_run_ext'] = koffi.struct({ ext: T['transcribe_ext'], initial_prompt: 'char *', prompt_tokens: 'void *', n_prompt_tokens: 'size_t', prompt_condition: 'int', condition_on_prev_tokens: 'bool', max_prev_context_tokens: 'int32_t', temperature: 'float', temperature_inc: 'float', compression_ratio_thold: 'float', logprob_thold: 'float', no_speech_thold: 'float', seed: 'uint32_t', max_initial_timestamp: 'float' }); T['transcribe_whisper_chunk_trace'] = koffi.struct({ struct_size: 'uint64_t', t0_ms: 'int64_t', t1_ms: 'int64_t', temperature_used: 'float', compression_ratio: 'float', avg_logprob: 'float', no_speech_prob: 'float', no_speech_triggered: 'bool', n_fallbacks: 'int32_t' }); @@ -233,6 +249,17 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_device_get': { ret: 'transcribe_device_t', args: ['int'] }, 'transcribe_device_get_info': { ret: 'transcribe_status', args: ['transcribe_device_t', 'struct transcribe_device_info *'] }, 'transcribe_device_info_init': { ret: 'void', args: ['struct transcribe_device_info *'] }, + 'transcribe_diarize_get_info': { ret: 'transcribe_status', args: ['const struct transcribe_model *', 'struct transcribe_diarize_info *'] }, + 'transcribe_diarize_get_segment': { ret: 'transcribe_status', args: ['const struct transcribe_diarize_session *', 'int', 'struct transcribe_speaker_segment *'] }, + 'transcribe_diarize_get_timings': { ret: 'transcribe_status', args: ['const struct transcribe_diarize_session *', 'struct transcribe_timings *'] }, + 'transcribe_diarize_info_init': { ret: 'void', args: ['struct transcribe_diarize_info *'] }, + 'transcribe_diarize_n_segments': { ret: 'int', args: ['const struct transcribe_diarize_session *'] }, + 'transcribe_diarize_params_init': { ret: 'void', args: ['struct transcribe_diarize_params *'] }, + 'transcribe_diarize_run': { ret: 'transcribe_status', args: ['struct transcribe_diarize_session *', 'const float *', 'int', 'const struct transcribe_diarize_params *'] }, + 'transcribe_diarize_session_free': { ret: 'void', args: ['struct transcribe_diarize_session *'] }, + 'transcribe_diarize_session_init': { ret: 'transcribe_status', args: ['struct transcribe_model *', 'const struct transcribe_diarize_session_params *', 'struct transcribe_diarize_session **'] }, + 'transcribe_diarize_session_params_init': { ret: 'void', args: ['struct transcribe_diarize_session_params *'] }, + 'transcribe_diarize_set_abort_callback': { ret: 'void', args: ['struct transcribe_diarize_session *', 'transcribe_abort_callback', 'void *'] }, 'transcribe_ext_check': { ret: 'transcribe_status', args: ['const struct transcribe_ext *', 'uint32_t', 'uint64_t'] }, 'transcribe_full_text': { ret: 'const char *', args: ['const struct transcribe_session *'] }, 'transcribe_get_model': { ret: 'const struct transcribe_model *', args: ['const struct transcribe_session *'] }, @@ -256,6 +283,7 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_model_load_file': { ret: 'transcribe_status', args: ['const char *', 'const struct transcribe_model_load_params *', 'struct transcribe_model **'] }, 'transcribe_model_load_params_init': { ret: 'void', args: ['struct transcribe_model_load_params *'] }, 'transcribe_model_meta_val_str': { ret: 'const char *', args: ['const struct transcribe_model *', 'const char *'] }, + 'transcribe_model_roles': { ret: 'uint32_t', args: ['const struct transcribe_model *'] }, 'transcribe_model_supports': { ret: '_Bool', args: ['const struct transcribe_model *', 'transcribe_feature'] }, 'transcribe_model_variant_string': { ret: 'const char *', args: ['const struct transcribe_model *'] }, 'transcribe_moonshine_streaming_stream_ext_init': { ret: 'void', args: ['struct transcribe_moonshine_streaming_stream_ext *'] }, @@ -280,7 +308,7 @@ export const FUNCTION_SIGNATURES: Record = { 'transcribe_session_limits_init': { ret: 'void', args: ['struct transcribe_session_limits *'] }, 'transcribe_session_params_init': { ret: 'void', args: ['struct transcribe_session_params *'] }, 'transcribe_set_abort_callback': { ret: 'void', args: ['struct transcribe_session *', 'transcribe_abort_callback', 'void *'] }, - 'transcribe_sortformer_stream_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_stream_ext *'] }, + 'transcribe_sortformer_diarize_ext_init': { ret: 'void', args: ['struct transcribe_sortformer_diarize_ext *'] }, 'transcribe_speaker_segment_init': { ret: 'void', args: ['struct transcribe_speaker_segment *'] }, 'transcribe_status_string': { ret: 'const char *', args: ['int'] }, 'transcribe_stream_begin': { ret: 'transcribe_status', args: ['struct transcribe_session *', 'const struct transcribe_run_params *', 'const struct transcribe_stream_params *'] }, diff --git a/bindings/typescript/src/errors.ts b/bindings/typescript/src/errors.ts index be517cf8a..f0495ea83 100644 --- a/bindings/typescript/src/errors.ts +++ b/bindings/typescript/src/errors.ts @@ -37,6 +37,9 @@ export class AbiError extends TranscribeError {} export class InputTooLong extends TranscribeError {} export class VersionMismatch extends TranscribeError {} +/** Raised when the model does not serve the role (ASR, diarize) the call needs. */ +export class UnsupportedRole extends TranscribeError {} + /** Raised when a run is cancelled; carries any partial transcript in `partialResult`. */ export class Aborted extends TranscribeError {} @@ -70,6 +73,7 @@ const STATUS_TO_EXC: Record TranscribeErr [g.TRANSCRIBE_ERR_INPUT_TOO_LONG]: InputTooLong, [g.TRANSCRIBE_ERR_OUTPUT_TRUNCATED]: OutputTruncated, [g.TRANSCRIBE_ERR_OUTPUT_REPETITION]: OutputRepetition, + [g.TRANSCRIBE_ERR_UNSUPPORTED_ROLE]: UnsupportedRole, }; /** Build (do not throw) the mapped exception for a status. */ diff --git a/bindings/typescript/src/ffi.ts b/bindings/typescript/src/ffi.ts index a02226db9..c6d3c4bf6 100644 --- a/bindings/typescript/src/ffi.ts +++ b/bindings/typescript/src/ffi.ts @@ -77,6 +77,7 @@ export function bindLibrary(libraryPath: string): Bound { modelBackend: lib.func("transcribe_model_backend", "str", ["void *"]), modelDevice: lib.func("transcribe_model_device", "void *", ["void *"]), modelSupports: lib.func("transcribe_model_supports", "bool", ["void *", "int"]), + modelRoles: lib.func("transcribe_model_roles", "uint32", ["void *"]), tokenize: lib.func("transcribe_tokenize", "int", ["void *", "str", "int32_t *", "size_t"]), capabilitiesInit: lib.func("transcribe_capabilities_init", "void", [ outp(T.transcribe_capabilities), @@ -173,8 +174,50 @@ export function bindLibrary(libraryPath: string): Bound { voxtralRealtimeStreamExtInit: lib.func("transcribe_voxtral_realtime_stream_ext_init", "void", [ outp(T.transcribe_voxtral_realtime_stream_ext), ]), - sortformerStreamExtInit: lib.func("transcribe_sortformer_stream_ext_init", "void", [ - outp(T.transcribe_sortformer_stream_ext), + sortformerDiarizeExtInit: lib.func("transcribe_sortformer_diarize_ext_init", "void", [ + outp(T.transcribe_sortformer_diarize_ext), + ]), + + // diarize role + diarizeInfoInit: lib.func("transcribe_diarize_info_init", "void", [ + outp(T.transcribe_diarize_info), + ]), + diarizeGetInfo: lib.func("transcribe_diarize_get_info", "int", [ + "void *", + iop(T.transcribe_diarize_info), + ]), + diarizeSessionParamsInit: lib.func("transcribe_diarize_session_params_init", "void", [ + outp(T.transcribe_diarize_session_params), + ]), + diarizeSessionInit: lib.func("transcribe_diarize_session_init", "int", [ + "void *", + inp(T.transcribe_diarize_session_params), + handleOut, + ]), + diarizeSessionFree: lib.func("transcribe_diarize_session_free", "void", ["void *"]), + diarizeSetAbortCallback: lib.func("transcribe_diarize_set_abort_callback", "void", [ + "void *", + "void *", + "void *", + ]), + diarizeParamsInit: lib.func("transcribe_diarize_params_init", "void", [ + outp(T.transcribe_diarize_params), + ]), + diarizeRun: lib.func("transcribe_diarize_run", "int", [ + "void *", + inp("float"), + "int", + inp(T.transcribe_diarize_params), + ]), + diarizeNSegments: lib.func("transcribe_diarize_n_segments", "int", ["void *"]), + diarizeGetSegment: lib.func("transcribe_diarize_get_segment", "int", [ + "void *", + "int", + iop(T.transcribe_speaker_segment), + ]), + diarizeGetTimings: lib.func("transcribe_diarize_get_timings", "int", [ + "void *", + iop(T.transcribe_timings), ]), // batch (offline) diff --git a/bindings/typescript/src/index.ts b/bindings/typescript/src/index.ts index c2e07394a..32adc1eaf 100644 --- a/bindings/typescript/src/index.ts +++ b/bindings/typescript/src/index.ts @@ -34,6 +34,9 @@ import type { CommitPolicy, DeviceType, Diarize, + DiarizeInfo, + DiarizeOptions, + DiarizeSessionOptions, ExtSlot, FamilyExtension, Feature, @@ -42,6 +45,7 @@ import type { ModelOptions, PcmLike, Pnc, + Role, Segment, SpeakerSegment, SessionLimits, @@ -122,6 +126,10 @@ const FEATURES: Record = { instruct: g.TRANSCRIBE_FEATURE_INSTRUCT, transcript_prefix: g.TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX, }; +const ROLES: Record = { + asr: g.TRANSCRIBE_ROLE_ASR, + diarize: g.TRANSCRIBE_ROLE_DIARIZE, +}; // ---- helpers --------------------------------------------------------------- @@ -499,40 +507,50 @@ function materialize(n: Native, acc: Accessors): Transcript { }); } - const speakerSegments: SpeakerSegment[] = []; - for (let i = 0, c = acc.nSpeakerSegments(); i < c; i++) { + return { + text: acc.fullText() ?? "", + rawText: acc.rawText() ?? "", + language: acc.detectedLanguage() ?? "", + timestampKind: TIMESTAMP_NAMES[acc.returnedTimestampKind()] ?? "none", + segments, + speakerSegments: readSpeakerSegments(n, acc.nSpeakerSegments(), acc.getSpeakerSegment), + words, + tokens, + timings: readTimings(n, acc.getTimings), + }; +} + +// Shared by transcripts and the DIARIZE role's results. +function readSpeakerSegments( + n: Native, + count: number, + get: (i: number, out: any) => number, +): SpeakerSegment[] { + const out: SpeakerSegment[] = []; + for (let i = 0; i < count; i++) { const s: any = {}; - F.speakerSegmentInit(s); - check(n, acc.getSpeakerSegment(i, s), `reading speaker segment ${i}`); - speakerSegments.push({ + n.F.speakerSegmentInit(s); + check(n, get(i, s), `reading speaker segment ${i}`); + out.push({ t0Ms: num(s.t0_ms), t1Ms: num(s.t1_ms), speakerId: s.speaker_id, p: s.p, }); } + return out; +} +function readTimings(n: Native, get: (out: any) => number): Timings { const tm: any = {}; - F.timingsInit(tm); - check(n, acc.getTimings(tm), "reading timings"); - const timings: Timings = { + n.F.timingsInit(tm); + check(n, get(tm), "reading timings"); + return { loadMs: tm.load_ms, melMs: tm.mel_ms, encodeMs: tm.encode_ms, decodeMs: tm.decode_ms, }; - - return { - text: acc.fullText() ?? "", - rawText: acc.rawText() ?? "", - language: acc.detectedLanguage() ?? "", - timestampKind: TIMESTAMP_NAMES[acc.returnedTimestampKind()] ?? "none", - segments, - speakerSegments, - words, - tokens, - timings, - }; } // ---- family extensions ----------------------------------------------------- @@ -551,6 +569,7 @@ const STREAM_STATES: Record = { const SLOT: Record = { run: g.TRANSCRIBE_EXT_SLOT_RUN, stream: g.TRANSCRIBE_EXT_SLOT_STREAM, + diarize_run: g.TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, }; interface FamilyReg { @@ -616,12 +635,15 @@ const FAMILY: Record = { min_decode_interval_ms: o.minDecodeIntervalMs, }), }, - sortformer: { - slot: "run", - kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - type: "transcribe_sortformer_stream_ext", - init: "sortformerStreamExtInit", - map: (o) => ({ preset: SORTFORMER_PRESET[o.preset as string] }), + sortformer_diarize: { + slot: "diarize_run", + kind: g.TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, + type: "transcribe_sortformer_diarize_ext", + init: "sortformerDiarizeExtInit", + map: (o) => ({ + preset: + o.preset === undefined ? undefined : lookup(SORTFORMER_PRESET, o.preset, "sortformer preset"), + }), }, }; @@ -636,7 +658,7 @@ const SORTFORMER_PRESET: Record = { * Build a native ext-struct buffer for a family extension and return the koffi * pointer to assign to `params.family`. Validates the slot and that the model * accepts the kind. The returned buffer must be kept alive (held via the params - * object) until the native call returns. + * object) until the native call returns, then freed by freeRunParams. */ function buildFamily( n: Native, @@ -665,7 +687,12 @@ function buildFamily( if (v !== undefined) ext[k] = v; } const buf = n.koffi.alloc(n.T[reg.type], 1); - n.koffi.encode(buf, n.T[reg.type], ext); + try { + n.koffi.encode(buf, n.T[reg.type], ext); + } catch (e) { + n.koffi.free(buf); + throw e; + } return buf; } @@ -676,13 +703,20 @@ function cstr(value: string, name: string): string { return value; } -/** Free what #buildRunParams allocated; call once the native call returns. */ +/** + * Free what #buildRunParams / buildFamily allocated into a params struct; call + * only once the native call has returned (never while a worker reads it). + */ function freeRunParams(n: Native, p: any): void { if (p.vocabulary) { n.koffi.free(p.vocabulary); p.vocabulary = null; p.n_vocabulary = 0; } + if (p.family) { + n.koffi.free(p.family); + p.family = null; + } } function toStreamUpdate(u: any): StreamUpdate { @@ -713,10 +747,20 @@ const STREAM_TEARDOWN = new WeakMap< { deactivate(): void; invalidate(): void; releaseLease(): void } >(); +/** + * Makes one worker call `fn(...args)` with `signal`'s abort callback installed + * and the session marked in flight as `kind` (result reads fail fast) until it + * settles. Only valid inside an exclusive() body. + */ +type ComputeCall = ( + kind: string, + signal: AbortSignal | undefined, + fn: any, + ...args: any[] +) => Promise; + interface SessionControl { - enterCompute(kind: string): void; - leaveCompute(kind: string): void; - currentCompute(): string | null; + core: SessionCore; isCurrentStream(stream: Stream): boolean; replaceCurrentStream(stream: Stream): void; clearCurrentStream(stream: Stream): void; @@ -724,17 +768,124 @@ interface SessionControl { const SESSION_CONTROL = new WeakMap(); +/** + * One native session handle under the model's compute rules, shared by Session + * and DiarizeSession. Each holds it in a private field, so none of this is + * reachable from user code. `setAbort` is the role's set_abort_callback. + */ +class SessionCore { + #n: Native; + #h: any; + #lock: Mutex; // shared with the model; serializes compute model-wide + #setAbort: any; + #inFlight: string | null = null; // set while a native call runs on a worker + #disposed = false; + + constructor(n: Native, handle: any, lock: Mutex, setAbort: any) { + this.#n = n; + this.#h = handle; + this.#lock = lock; + this.#setAbort = setAbort; + } + + get handle(): any { + if (this.#disposed) throw new TranscribeError("session has been disposed"); + return this.#h; + } + + get disposed(): boolean { + return this.#disposed; + } + + /** Reads touch the session; forbidden while a worker call is in flight. */ + assertNotComputing(what: string): void { + if (this.#inFlight) { + throw new TranscribeError( + `cannot read ${what} while ${this.#inFlight} is in flight; await it first`, + ); + } + } + + /** + * Run `body` as this session's one native compute on the model-wide FIFO + * lock (it copies results out before release). Refuses a disposed session, + * then an active stream (Busy naming `busyOp`); null `busyOp` = that stream. + * Not `async`: refusals and `body`'s promise are returned as-is (no extra ticks). + */ + exclusive( + busyOp: string | null, + body: (call: ComputeCall) => Promise, + ): Promise { + return this.#lock.run(() => { + if (busyOp !== null && this.#disposed) + return Promise.reject(new TranscribeError("session has been disposed")); + if (busyOp !== null && this.#lock.streamActive) + return Promise.reject(busyError(busyOp)); + return body(this.#call); + }); + } + + #call: ComputeCall = (kind, signal, fn, ...args) => { + const cancel = this.#installAbort(signal); + this.#inFlight = kind; + return callAsync(fn, ...args).finally(() => { + if (this.#inFlight === kind) this.#inFlight = null; + cancel?.(); + }); + }; + + /** + * Wire an AbortSignal to a native abort callback for one run. The callback is + * installed on *this session's* handle, but install/run/uninstall is only safe + * because every caller holds the model-wide #lock for the whole run — the lock, + * not the per-session handle, is what guarantees no run overlaps the window + * between setAbort(cb) and setAbort(null). A future change that relaxes the + * lock must keep this install/uninstall paired within one run. + */ + #installAbort(signal?: AbortSignal): (() => void) | null { + if (!signal) return null; + const n = this.#n; + const flag = { aborted: signal.aborted }; + const onAbort = () => { + flag.aborted = true; + }; + signal.addEventListener("abort", onAbort, { once: true }); + const cbPtr = n.koffi.register( + () => flag.aborted, + n.koffi.pointer(n.abortProto), + ); + const h = this.#h; // still valid at cleanup: a mid-run dispose frees behind the lock + this.#setAbort(h, cbPtr, null); + return () => { + this.#setAbort(h, null, null); + n.koffi.unregister(cbPtr); + signal.removeEventListener("abort", onAbort); + }; + } + + /** + * Mark disposed now (use-after-dispose throws immediately) and free the + * native handle behind the model lock: a worker may still hold it, and a + * free mid-call is a use-after-free. Queuing on the FIFO lock runs the free + * (then `after`) once any in-flight and queued compute drains. + */ + dispose(free: (h: any) => void, after?: () => void): void { + this.#disposed = true; + const h = this.#h; + this.#h = null; + deferFree(this.#lock, () => free(h), after); + } +} + // ---- Session --------------------------------------------------------------- export class Session { #n: Native; - #h: any; + #core: SessionCore; #model: TranscribeModel; // keep the model alive while this session lives #lock: Mutex; // shared with the model; serializes compute model-wide #untrack: (self: Session) => void; // drop self from the model's session set - #inFlight: string | null = null; // set while a native call runs on a worker #activeStream: Stream | null = null; // current wrapper for the session's native stream slot - #disposed = false; /** @internal */ constructor( @@ -746,17 +897,11 @@ export class Session { ) { this.#n = n; this.#model = model; - this.#h = handle; + this.#core = new SessionCore(n, handle, lock, n.F.setAbortCallback); this.#lock = lock; this.#untrack = untrack; SESSION_CONTROL.set(this, { - enterCompute: (kind) => { - this.#inFlight = kind; - }, - leaveCompute: (kind) => { - if (this.#inFlight === kind) this.#inFlight = null; - }, - currentCompute: () => this.#inFlight, + core: this.#core, isCurrentStream: (stream) => this.#activeStream === stream, replaceCurrentStream: (stream) => { if (this.#activeStream && this.#activeStream !== stream) { @@ -772,21 +917,11 @@ export class Session { /** @internal */ get handle(): any { - if (this.#disposed) throw new TranscribeError("session has been disposed"); - return this.#h; - } - - /** Reads touch the session; forbidden while a worker call is in flight. */ - #assertNotComputing(what: string): void { - if (this.#inFlight) { - throw new TranscribeError( - `cannot read session ${what} while ${this.#inFlight} is in flight; await it first`, - ); - } + return this.#core.handle; } get limits(): SessionLimits { - this.#assertNotComputing("limits"); + this.#core.assertNotComputing("session limits"); const n = this.#n; const l: any = {}; n.F.sessionLimitsInit(l); @@ -814,19 +949,8 @@ export class Session { const p = this.#buildRunParams(opts); - return this.#lock.run(async () => { - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("run"); - const cancel = this.#installAbort(opts.signal); - let status: number; - this.#inFlight = "run()"; - try { - status = await callAsync(F.run, h, samples, samples.length, p); - } finally { - this.#inFlight = null; - cancel?.(); - } + return this.#core.exclusive("run", async (call) => { + const status = await call("run()", opts.signal, F.run, h, samples, samples.length, p); if ( status === g.TRANSCRIBE_ERR_ABORTED || @@ -861,6 +985,16 @@ export class Session { const n = this.#n; const p: any = {}; n.F.runParamsInit(p); + try { + this.#fillRunParams(n, p, opts); + } catch (e) { + freeRunParams(n, p); // the caller never gets p, so free what was allocated + throw e; + } + return p; + } + + #fillRunParams(n: Native, p: any, opts: TranscribeOptions): void { p.task = lookup(TASKS, opts.task ?? "transcribe", "task"); // Default "auto" mirrors the C transcribe_run_params_init default: // whisper resolves it to "segment" (its robust path), no-timestamp @@ -875,8 +1009,6 @@ export class Session { if (opts.keepSpecialTags !== undefined) p.keep_special_tags = opts.keepSpecialTags; if (opts.specKDrafts !== undefined) p.spec_k_drafts = opts.specKDrafts; - if (opts.family) - p.family = buildFamily(n, this.#model.handle, opts.family, "run"); if (opts.vocabulary !== undefined) { const terms = opts.vocabulary; if (!Array.isArray(terms) || !terms.every((t) => typeof t === "string")) @@ -893,7 +1025,8 @@ export class Session { } if (opts.prompt !== undefined) p.prompt = cstr(opts.prompt, "prompt"); if (opts.prefix !== undefined) p.prefix = cstr(opts.prefix, "prefix"); - return p; + if (opts.family) + p.family = buildFamily(n, this.#model.handle, opts.family, "run"); } /** @@ -914,26 +1047,10 @@ export class Session { const counts = Int32Array.from(arrays, (a) => a.length); const p = this.#buildRunParams(opts); - return this.#lock.run(async () => { - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("runBatch"); - const cancel = this.#installAbort(opts.signal); - let status: number; - this.#inFlight = "runBatch()"; - try { - status = await callAsync( - F.runBatch, - h, - arrays, - counts, - arrays.length, - p, - ); - } finally { - this.#inFlight = null; - cancel?.(); - } + return this.#core.exclusive("runBatch", async (call) => { + const status = await call( + "runBatch()", opts.signal, F.runBatch, h, arrays, counts, arrays.length, p, + ); // A batch returns OK even with per-utterance failures; only a top-level // error (or a whole-batch abort) is fatal here. if (status !== g.TRANSCRIBE_OK && status !== g.TRANSCRIBE_ERR_ABORTED) { @@ -998,23 +1115,24 @@ export class Session { }); const sp: any = {}; F.streamParamsInit(sp); - sp.commit_policy = lookup( - COMMIT_POLICIES, - opts.commitPolicy ?? "auto", - "commitPolicy", - ); - if (opts.stablePrefixAgreementN !== undefined) { - sp.stable_prefix_agreement_n = opts.stablePrefixAgreementN; + try { + sp.commit_policy = lookup( + COMMIT_POLICIES, + opts.commitPolicy ?? "auto", + "commitPolicy", + ); + if (opts.stablePrefixAgreementN !== undefined) { + sp.stable_prefix_agreement_n = opts.stablePrefixAgreementN; + } + if (opts.family) + sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); + } catch (e) { + freeRunParams(n, rp); // sp holds nothing yet: buildFamily frees on throw + throw e; } - if (opts.family) - sp.family = buildFamily(n, this.#model.handle, opts.family, "stream"); - - return this.#lock.run(async () => { - // Recheck inside the lock: dispose() may have run after we captured `h` - // but before this queued body — don't begin a stream on a dead session. - if (this.#disposed) - throw new TranscribeError("session has been disposed"); - if (this.#lock.streamActive) throw busyError("begin a stream"); + + // Begin is a synchronous native call, so no in-flight window (no call()). + return this.#core.exclusive("begin a stream", async () => { check(n, F.streamBegin(h, rp, sp), "transcribe_stream_begin"); this.#lock.streamActive = true; // claim the lease for the whole stream lifetime // The Stream holds the Session (not a raw handle) so its calls fail fast @@ -1024,45 +1142,20 @@ export class Session { if (!control) throw new TranscribeError("session control is missing"); control.replaceCurrentStream(stream); return stream; - }).finally(() => freeRunParams(n, rp)); // begin copied the prompting strings - } - - /** - * Wire an AbortSignal to a native abort callback for one run. The callback is - * installed on *this session's* handle, but install/run/uninstall is only safe - * because every caller holds the model-wide #lock for the whole run — the lock, - * not the per-session handle, is what guarantees no run overlaps the window - * between setAbortCallback(cb) and setAbortCallback(null). A future change that - * relaxes the lock must keep this install/uninstall paired within one run. - */ - #installAbort(signal?: AbortSignal): (() => void) | null { - if (!signal) return null; - const n = this.#n; - const flag = { aborted: signal.aborted }; - const onAbort = () => { - flag.aborted = true; - }; - signal.addEventListener("abort", onAbort, { once: true }); - const cbPtr = n.koffi.register( - () => flag.aborted, - n.koffi.pointer(n.abortProto), - ); - n.F.setAbortCallback(this.handle, cbPtr, null); - return () => { - n.F.setAbortCallback(this.handle, null, null); - n.koffi.unregister(cbPtr); - signal.removeEventListener("abort", onAbort); - }; + }).finally(() => { + // begin copied the prompting strings and the family extension + freeRunParams(n, rp); + freeRunParams(n, sp); + }); } get wasAborted(): boolean { - this.#assertNotComputing("wasAborted"); + this.#core.assertNotComputing("session wasAborted"); return this.#n.F.wasAborted(this.handle); } dispose(): void { - if (this.#disposed) return; - this.#disposed = true; + if (this.#core.disposed) return; this.#untrack(this); // stop the model from holding a dead session // Deactivate any live stream NOW (its reset() no-ops, reads throw via the // disposed handle), but release its model lease only inside the deferred @@ -1072,19 +1165,7 @@ export class Session { this.#activeStream = null; const teardown = stream ? STREAM_TEARDOWN.get(stream) : undefined; teardown?.deactivate(); - // Free behind the model lock: a run/feed worker may still hold this handle, - // and sessionFree mid-call is a use-after-free. Queuing on the FIFO lock - // runs the free after any in-flight (and queued) compute drains. The JS-side - // guard is already synchronous (#disposed/handle), so use-after-dispose - // still throws immediately; only the native free + lease release are deferred. - const n = this.#n; - const h = this.#h; - this.#h = null; - deferFree( - this.#lock, - () => n.F.sessionFree(h), - () => teardown?.releaseLease(), - ); + this.#core.dispose(this.#n.F.sessionFree, () => teardown?.releaseLease()); } [Symbol.dispose](): void { @@ -1164,33 +1245,19 @@ export class Stream { this.#assertCurrent("feed"); if (!this.#active) throw new TranscribeError("stream has been reset"); const samples = toFloat32(pcm); - return this.#lock.run(async () => { + return this.#sessionControl.core.exclusive(null, async (call) => { const u: any = {}; n.F.streamUpdateInit(u); - // The native feed runs on a libuv worker. While it is in flight the - // session must not be touched from the main thread — the C session API - // is single-threaded (transcribe.h), and stream_get_text hands back - // pointers the feed may free/realloc. Flag the owning session so every - // result getter fails fast instead of racing into a use-after-free. - this.#sessionControl.enterCompute("feed()/finalize()"); - try { - const status = await callAsync( - n.F.streamFeed, - h, - samples, - samples.length, - u, - ); - if (status !== g.TRANSCRIBE_OK) { - // Native feed failures leave the stream in FAILED, which is no longer - // an active stream in the C API. Keep the wrapper readable for - // state/lastStatus, but free the model-wide compute slot. - this.#releaseLease(); - } - check(n, status, "transcribe_stream_feed"); - } finally { - this.#sessionControl.leaveCompute("feed()/finalize()"); - } + const status = await call( + "feed()/finalize()", undefined, n.F.streamFeed, h, samples, samples.length, u, + ); + // The lease follows the native stream: a feed refused before the family + // hook (e.g. non-finite samples) leaves it ACTIVE, so keep the lease + // (siblings stay Busy); a hook failure moves it to FAILED, no longer + // active, so free the model-wide slot (state/lastStatus stay readable). + if (status !== g.TRANSCRIBE_OK && n.F.streamGetState(h) !== g.TRANSCRIBE_STREAM_ACTIVE) + this.#releaseLease(); + check(n, status, "transcribe_stream_feed"); return toStreamUpdate(u); }); } @@ -1201,18 +1268,13 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("finalize stream"); if (!this.#active) throw new TranscribeError("stream has been reset"); - return this.#lock.run(async () => { + return this.#sessionControl.core.exclusive(null, async (call) => { const u: any = {}; n.F.streamUpdateInit(u); - this.#sessionControl.enterCompute("feed()/finalize()"); try { - check( - n, - await callAsync(n.F.streamFinalize, h, u), - "transcribe_stream_finalize", - ); + const status = await call("feed()/finalize()", undefined, n.F.streamFinalize, h, u); + check(n, status, "transcribe_stream_finalize"); } finally { - this.#sessionControl.leaveCompute("feed()/finalize()"); // Finalize ends the active stream (FINISHED on success, FAILED on // error), so the model is free again — release the lease either way. this.#releaseLease(); @@ -1221,25 +1283,11 @@ export class Stream { }); } - /** - * Reads borrow session-owned snapshot memory, so they are forbidden while - * any worker call is computing on this session (concurrent use is undefined - * per transcribe.h). The natural await-then-read pattern is unaffected. - */ - #assertNotComputing(what: string): void { - const compute = this.#sessionControl.currentCompute(); - if (compute) { - throw new TranscribeError( - `cannot read stream ${what} while ${compute} is in flight; await it first`, - ); - } - } - /** Current text snapshot (copied at the boundary). */ get text(): StreamText { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream text"); - this.#assertNotComputing("text"); + this.#sessionControl.core.assertNotComputing("stream text"); const n = this.#n; const t: any = {}; n.F.streamTextInit(t); @@ -1256,7 +1304,7 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream snapshot"); if (!this.#active) throw new TranscribeError("stream has been reset"); - this.#assertNotComputing("snapshot"); + this.#sessionControl.core.assertNotComputing("stream snapshot"); return materialize(this.#n, singleAccessors(this.#n, h)); } @@ -1264,14 +1312,14 @@ export class Stream { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream state"); if (!this.#active) return "idle"; // reset() returns to idle; native reset may still be queued - this.#assertNotComputing("state"); + this.#sessionControl.core.assertNotComputing("stream state"); return STREAM_STATES[this.#n.F.streamGetState(h)] ?? "idle"; } get revision(): number { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream revision"); - this.#assertNotComputing("revision"); + this.#sessionControl.core.assertNotComputing("stream revision"); return this.#n.F.streamRevision(h); } @@ -1283,7 +1331,7 @@ export class Stream { get lastStatus(): TranscribeError | null { const h = this.#session.handle; // throws if the session was disposed this.#assertCurrent("read stream lastStatus"); - this.#assertNotComputing("lastStatus"); + this.#sessionControl.core.assertNotComputing("stream lastStatus"); const n = this.#n; const status = n.F.streamLastStatus(h); if (status === g.TRANSCRIBE_OK) return null; @@ -1321,13 +1369,78 @@ export class Stream { } } +// ---- DiarizeSession -------------------------------------------------------- + +/** A DIARIZE-role session: who spoke when. Same compute rules as Session. */ +export class DiarizeSession { + #n: Native; + #core: SessionCore; + #model: TranscribeModel; // keep the model alive while this session lives + #untrack: (self: DiarizeSession) => void; + + /** @internal */ + constructor( + n: Native, + model: TranscribeModel, + handle: any, + lock: Mutex, + untrack: (self: DiarizeSession) => void, + ) { + this.#n = n; + this.#model = model; + this.#core = new SessionCore(n, handle, lock, n.F.diarizeSetAbortCallback); + this.#untrack = untrack; + } + + /** + * Diarize one recording; returns its speaker turns, grouped by speaker and + * time-ordered within a speaker (turns of different speakers may overlap). + * The input PCM is borrowed, not copied (see Session.run). + */ + async run(pcm: PcmLike, opts: DiarizeOptions = {}): Promise { + const n = this.#n; + const F = n.F; + const h = this.#core.handle; + const samples = toFloat32(pcm); + const p: any = {}; + F.diarizeParamsInit(p); + if (opts.family) + p.family = buildFamily(n, this.#model.handle, opts.family, "diarize_run"); + + return this.#core.exclusive("diarize", async (call) => { + const status = await call("run()", opts.signal, F.diarizeRun, h, samples, samples.length, p); + check(n, status, "transcribe_diarize_run"); + return readSpeakerSegments(n, F.diarizeNSegments(h), (i, o) => + F.diarizeGetSegment(h, i, o), + ); + }).finally(() => freeRunParams(n, p)); + } + + /** load_ms plus the last run's mel / encode time. */ + get timings(): Timings { + this.#core.assertNotComputing("session timings"); + const h = this.#core.handle; + return readTimings(this.#n, (o) => this.#n.F.diarizeGetTimings(h, o)); + } + + dispose(): void { + if (this.#core.disposed) return; + this.#untrack(this); + this.#core.dispose(this.#n.F.diarizeSessionFree); + } + + [Symbol.dispose](): void { + this.dispose(); + } +} + // ---- Model ----------------------------------------------------------------- export class TranscribeModel { #n: Native; #h: any; #disposed = false; - #sessions = new Set(); + #sessions = new Set(); #lock = new Mutex(); // serializes compute across all sessions of this model private constructor(n: Native, handle: any) { @@ -1409,6 +1522,38 @@ export class TranscribeModel { } } + /** The roles this model serves; ASR calls on a model without "asr" throw UnsupportedRole. */ + get roles(): readonly Role[] { + const mask = this.#n.F.modelRoles(this.handle); + return (Object.keys(ROLES) as Role[]).filter((r) => mask & ROLES[r]); + } + + /** Static facts of a "diarize" model; UnsupportedRole otherwise. */ + get diarizeInfo(): DiarizeInfo { + const n = this.#n; + const info: any = {}; + n.F.diarizeInfoInit(info); + check(n, n.F.diarizeGetInfo(this.handle, info), "reading diarize info"); + return { sampleRate: info.sample_rate, maxSpeakers: info.max_speakers }; + } + + /** Open a DIARIZE-role session; UnsupportedRole on a model without "diarize". */ + createDiarizeSession(opts: DiarizeSessionOptions = {}): DiarizeSession { + const n = this.#n; + const p: any = {}; + n.F.diarizeSessionParamsInit(p); + if (opts.nThreads !== undefined) p.n_threads = opts.nThreads; + const out: any[] = [null]; + check(n, n.F.diarizeSessionInit(this.handle, p, out), "opening diarize session"); + if (!out[0]) + throw new TranscribeError("diarize session init returned a null handle"); + const session = new DiarizeSession(n, this, out[0], this.#lock, (s) => + this.#sessions.delete(s), + ); + this.#sessions.add(session); + return session; + } + get capabilities(): Capabilities { const n = this.#n; const c: any = {}; diff --git a/bindings/typescript/src/types.ts b/bindings/typescript/src/types.ts index d986a5c8f..1ddd6bfb1 100644 --- a/bindings/typescript/src/types.ts +++ b/bindings/typescript/src/types.ts @@ -197,7 +197,7 @@ export type BatchItem = export type CommitPolicy = "auto" | "on_finalize" | "stable_prefix"; export type StreamState = "idle" | "active" | "finished" | "failed"; -export type ExtSlot = "run" | "stream"; +export type ExtSlot = "run" | "stream" | "diarize_run"; export interface StreamUpdate { resultChanged: boolean; @@ -276,9 +276,8 @@ export type SortformerPreset = | "very_high_latency" | "high_latency" | "low_latency"; -/** Sortformer diarizer options (run slot). A run produces speaker - * segments, no text. */ -export interface SortformerStreamOptions { +/** Sortformer options for {@link DiarizeOptions.family} (diarize_run slot). */ +export interface SortformerDiarizeOptions { preset?: SortformerPreset; } @@ -288,4 +287,28 @@ export type FamilyExtension = | ({ kind: "parakeet" } & ParakeetStreamOptions) | ({ kind: "parakeet_buffered" } & ParakeetBufferedStreamOptions) | ({ kind: "voxtral" } & VoxtralRealtimeStreamOptions) - | ({ kind: "sortformer" } & SortformerStreamOptions); + | ({ kind: "sortformer_diarize" } & SortformerDiarizeOptions); + +// ---- roles ----------------------------------------------------------------- + +/** What a model serves: "asr" (transcription) and/or "diarize" (speaker turns). */ +export type Role = "asr" | "diarize"; + +export interface DiarizeInfo { + /** Input PCM rate. */ + sampleRate: number; + /** Speaker ids are in [1, maxSpeakers]. */ + maxSpeakers: number; +} + +export interface DiarizeSessionOptions { + /** CPU threads for CPU-side ops; 0 = library default. */ + nThreads?: number; +} + +export interface DiarizeOptions { + /** Cancel the run cooperatively. */ + signal?: AbortSignal; + /** A diarize_run-slot family extension (e.g. sortformer_diarize). */ + family?: FamilyExtension; +} diff --git a/bindings/typescript/test/cancel.test.mjs b/bindings/typescript/test/cancel.test.mjs index 7f9e3beb6..d8b22fcaa 100644 --- a/bindings/typescript/test/cancel.test.mjs +++ b/bindings/typescript/test/cancel.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { getEventListeners } from "node:events"; import { modelTest, MODEL, jfk } from "./common.mjs"; import { TranscribeModel, Aborted } from "../dist/index.js"; @@ -27,3 +28,18 @@ modelTest("a pre-aborted signal raises Aborted with a partial result", MODEL, as m.dispose(); } }); + +modelTest("dispose during a run with a signal returns the result and removes the listener", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + const s = m.createSession(); + const ac = new AbortController(); + const p = s.run(jfk(), { signal: ac.signal }); + await Promise.resolve(); + s.dispose(); + assert.match((await p).text, /ask not what your country/i); + assert.equal(getEventListeners(ac.signal, "abort").length, 0); + } finally { + m.dispose(); + } +}); diff --git a/bindings/typescript/test/common.mjs b/bindings/typescript/test/common.mjs index e26d09cae..de5e68e0e 100644 --- a/bindings/typescript/test/common.mjs +++ b/bindings/typescript/test/common.mjs @@ -22,6 +22,9 @@ export const ITN_MODEL = // Voxtral realtime is local-only (~2.5 GB+, too heavy for the CI canary set): // its env var is NOT exported by fetch-canary, so this skips cleanly in CI. export const VOXTRAL_MODEL = process.env.TRANSCRIBE_SMOKE_VOXTRAL_MODEL || ""; +// Sortformer (DIARIZE role only): fetch-canary exports the Q8_0 canary. +export const SORTFORMER_MODEL = process.env.TRANSCRIBE_SMOKE_SORTFORMER_MODEL || ""; +export const SORTFORMER_AUDIO = path.resolve(HERE, "../../../samples/sortformer-2spk-mix.wav"); // jfk.wav ships in-repo; fetch-canary exports only the model paths. export const AUDIO = diff --git a/bindings/typescript/test/diarize.test.mjs b/bindings/typescript/test/diarize.test.mjs new file mode 100644 index 000000000..6e6981216 --- /dev/null +++ b/bindings/typescript/test/diarize.test.mjs @@ -0,0 +1,80 @@ +// DIARIZE role: model.roles, diarizeInfo and DiarizeSession wiring. Segment +// values are pinned by the C test (sortformer_diarize_unit); here we check the +// binding returns well-formed rows and passes the preset through. + +import assert from "node:assert/strict"; +import { modelTest, MODEL, SORTFORMER_MODEL, SORTFORMER_AUDIO, readWav } from "./common.mjs"; +import { TranscribeModel, InvalidArgument, TranscribeError, UnsupportedRole } from "../dist/index.js"; + +const mix = () => readWav(SORTFORMER_AUDIO); + +modelTest("an ASR-only model refuses the diarize role with UnsupportedRole", MODEL, async () => { + const m = await TranscribeModel.load(MODEL); + try { + assert.deepEqual(m.roles, ["asr"]); + assert.throws(() => m.diarizeInfo, UnsupportedRole); + assert.throws(() => m.createDiarizeSession(), UnsupportedRole); + assert.equal(m.accepts({ kind: "sortformer_diarize" }), false); + } finally { + m.dispose(); + } +}); + +modelTest("sortformer serves only diarize; diarizeInfo; ASR calls are refused", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + assert.deepEqual(m.roles, ["diarize"]); + assert.deepEqual(m.diarizeInfo, { sampleRate: 16000, maxSpeakers: 4 }); + assert.equal(m.accepts({ kind: "sortformer_diarize" }), true); + assert.throws(() => m.capabilities, UnsupportedRole); + assert.throws(() => m.createSession(), UnsupportedRole); + } finally { + m.dispose(); + } +}); + +modelTest("diarize run returns speaker turns and honors the preset", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL, { backend: "cpu" }); + try { + const pcm = mix(); + const d = m.createDiarizeSession({ nThreads: 4 }); + const turns = async (preset) => + (await d.run(pcm, { family: { kind: "sortformer_diarize", preset } })).map((r) => [ + r.t0Ms, + r.t1Ms, + r.speakerId, + ]); + const def = await turns("default"); + assert.ok(def.length > 0); + for (const [t0, t1, spk] of def) { + assert.ok(t0 < t1, `${t0} < ${t1}`); + assert.ok(spk >= 1 && spk <= m.diarizeInfo.maxSpeakers, `speaker ${spk}`); + } + assert.notDeepEqual(await turns("low_latency"), def, "preset reaches the native run"); + assert.ok(d.timings.encodeMs > 0); + d.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a bad preset or wrong-slot extension is rejected", SORTFORMER_MODEL, async () => { + const m = await TranscribeModel.load(SORTFORMER_MODEL); + try { + const d = m.createDiarizeSession(); + const pcm = mix(); + await assert.rejects( + () => d.run(pcm, { family: { kind: "sortformer_diarize", preset: "ultra_low_latency" } }), + (e) => e instanceof TranscribeError && /invalid sortformer preset/.test(e.message), + ); + await assert.rejects(() => d.run(pcm, { family: { kind: "whisper" } }), InvalidArgument); + await assert.rejects( + () => d.run(pcm, { family: { kind: "sortformer" } }), + (e) => e instanceof InvalidArgument && /unknown family extension kind/.test(e.message), + ); + assert.ok((await d.run(pcm)).length > 0); + d.dispose(); + } finally { + m.dispose(); + } +}); diff --git a/bindings/typescript/test/errors.test.mjs b/bindings/typescript/test/errors.test.mjs new file mode 100644 index 000000000..0c0d13a2c --- /dev/null +++ b/bindings/typescript/test/errors.test.mjs @@ -0,0 +1,77 @@ +// No-model tier: status -> error-class mapping, pure (no native call, no GGUF). +// Mirrors bindings/python/tests/test_errors.py. +import { test } from "node:test"; +import assert from "node:assert/strict"; +import * as g from "../dist/_generated.js"; +import { + exceptionForStatus, + raiseForStatus, + TranscribeError, + InvalidArgument, + NotImplementedByModel, + ModelFileNotFound, + ModelLoadError, + OutOfMemory, + BackendError, + UnsupportedRequest, + UnsupportedRole, + Aborted, + AbiError, + InputTooLong, + OutputTruncated, + OutputRepetition, +} from "../dist/index.js"; + +const EXPECTED = { + TRANSCRIBE_ERR_INVALID_ARG: InvalidArgument, + TRANSCRIBE_ERR_NOT_IMPLEMENTED: NotImplementedByModel, + TRANSCRIBE_ERR_FILE_NOT_FOUND: ModelFileNotFound, + TRANSCRIBE_ERR_GGUF: ModelLoadError, + TRANSCRIBE_ERR_UNSUPPORTED_ARCH: ModelLoadError, + TRANSCRIBE_ERR_UNSUPPORTED_VARIANT: ModelLoadError, + TRANSCRIBE_ERR_OOM: OutOfMemory, + TRANSCRIBE_ERR_BACKEND: BackendError, + TRANSCRIBE_ERR_SAMPLE_RATE: InvalidArgument, + TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_TASK: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS: UnsupportedRequest, + TRANSCRIBE_ERR_ABORTED: Aborted, + TRANSCRIBE_ERR_BAD_STRUCT_SIZE: AbiError, + TRANSCRIBE_ERR_UNSUPPORTED_PNC: UnsupportedRequest, + TRANSCRIBE_ERR_UNSUPPORTED_ITN: UnsupportedRequest, + TRANSCRIBE_ERR_INPUT_TOO_LONG: InputTooLong, + TRANSCRIBE_ERR_OUTPUT_TRUNCATED: OutputTruncated, + TRANSCRIBE_ERR_OUTPUT_REPETITION: OutputRepetition, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE: UnsupportedRole, +}; + +test("every transcribe_status maps to its documented error class", () => { + // The expectation covers every non-OK status the header defines, and nothing + // else: a new C status must be mapped (and listed here) deliberately. + const generated = Object.keys(g) + .filter((k) => k.startsWith("TRANSCRIBE_ERR_")) + .sort(); + assert.deepEqual(generated, Object.keys(EXPECTED).sort()); + + for (const [name, Cls] of Object.entries(EXPECTED)) { + const status = g[name]; + assert.throws( + () => raiseForStatus(status, "synthetic", "ctx"), + (e) => { + assert.equal(e.constructor, Cls, name); + assert.ok(e instanceof TranscribeError); + assert.equal(e.status, status); + assert.equal(e.name, Cls.name); + assert.match(e.message, /ctx: synthetic/); + return true; + }, + ); + } + assert.equal(Object.getPrototypeOf(UnsupportedRole), TranscribeError); +}); + +test("unknown status degrades to the base class", () => { + const e = exceptionForStatus(999, "mystery"); + assert.equal(e.constructor, TranscribeError); + assert.equal(e.status, 999); +}); diff --git a/bindings/typescript/test/prompting.test.mjs b/bindings/typescript/test/prompting.test.mjs index fdac27bca..fe60aeb18 100644 --- a/bindings/typescript/test/prompting.test.mjs +++ b/bindings/typescript/test/prompting.test.mjs @@ -2,6 +2,7 @@ // streaming canaries. Mirrors bindings/python/tests/test_prompting.py. import assert from "node:assert/strict"; +import koffi from "koffi"; import { modelTest, MODEL, STREAMING_MODEL, jfk, feedChunks } from "./common.mjs"; import { TranscribeModel, InvalidArgument, UnsupportedRequest } from "../dist/index.js"; @@ -33,6 +34,23 @@ modelTest("vocabulary and prompt reach run and runBatch", MODEL, async () => { }); }); +modelTest("a rejected family option frees the vocabulary it followed", MODEL, async () => { + await withSession(MODEL, async (_m, s) => { + const { alloc, free } = koffi; + let live = 0; + koffi.alloc = (...a) => (live++, alloc(...a)); + koffi.free = (...a) => (live--, free(...a)); + try { + await assert.rejects(s.run(jfk(), { vocabulary: TERMS, family: { kind: "nope" } })); + await assert.rejects(s.stream({ vocabulary: TERMS, commitPolicy: "nope" })); + } finally { + koffi.alloc = alloc; + koffi.free = free; + } + assert.equal(live, 0, "koffi allocations leaked"); + }); +}); + modelTest("prefix: text is the continuation, rawText leads with it", MODEL, async () => { await withSession(MODEL, async (m, s) => { if (!m.supports("transcript_prefix")) return; diff --git a/bindings/typescript/test/streaming.test.mjs b/bindings/typescript/test/streaming.test.mjs index 7c38b1716..9ad775001 100644 --- a/bindings/typescript/test/streaming.test.mjs +++ b/bindings/typescript/test/streaming.test.mjs @@ -1,6 +1,9 @@ import assert from "node:assert/strict"; +import koffi from "koffi"; import { modelTest, MODEL, STREAMING_MODEL, jfk, feedChunks } from "./common.mjs"; -import { TranscribeModel, Busy } from "../dist/index.js"; +import { TranscribeModel, Aborted, Busy, InvalidArgument, libraryPath } from "../dist/index.js"; + +const oneSecond = () => jfk().subarray(0, 16000); modelTest("streaming commits text and finalizes", STREAMING_MODEL, async () => { const m = await TranscribeModel.load(STREAMING_MODEL); @@ -249,6 +252,76 @@ modelTest("a stale stream wrapper cannot reset a newer same-session stream", STR } }); +modelTest("a feed rejected before the native hook keeps the stream lease", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(oneSecond()); + const bad = new Float32Array(1600); + bad[5] = Number.NaN; + // Native rejects non-finite samples before the family hook and leaves the + // stream ACTIVE, so the binding keeps the model-wide lease. + await assert.rejects(() => stream.feed(bad), InvalidArgument); + assert.equal(stream.state, "active"); + await assert.rejects(() => b.run(oneSecond()), Busy); + + // The stream is still usable, and finalize releases the lease as usual. + const u = await stream.feed(jfk().subarray(16000, 32000)); + assert.equal(typeof u.revision, "number"); + await stream.finalize(); + const r = await b.run(oneSecond()); + assert.equal(typeof r.text, "string"); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + +modelTest("a feed that fails inside the native hook releases the stream lease", STREAMING_MODEL, async () => { + const m = await TranscribeModel.load(STREAMING_MODEL); + try { + const a = m.createSession(); + const b = m.createSession(); + const stream = await a.stream({ commitPolicy: "stable_prefix" }); + await stream.feed(oneSecond()); + + // Provoke a hook failure: an abort callback that always fires makes the + // streaming hook return ABORTED, which moves the native stream to FAILED. + // feed() has no signal option, so install it on the native session + // directly (test-only; the same library instance the binding loaded). + const lib = koffi.load(libraryPath()); + const proto = koffi.proto("bool StreamingTestAbortCb(void *udata)"); + const setAbort = lib.func("transcribe_set_abort_callback", "void", [ + "void *", + koffi.pointer(proto), + "void *", + ]); + const cb = koffi.register(() => true, koffi.pointer(proto)); + setAbort(a.handle, cb, null); + try { + await assert.rejects(() => stream.feed(oneSecond()), Aborted); + } finally { + setAbort(a.handle, null, null); + koffi.unregister(cb); + } + assert.equal(stream.state, "failed"); + assert.ok(stream.lastStatus instanceof Aborted); + + // FAILED is no longer an active stream: the lease is free for a sibling. + const r = await b.run(oneSecond()); + assert.equal(typeof r.text, "string"); + stream.reset(); + a.dispose(); + b.dispose(); + } finally { + m.dispose(); + } +}); + modelTest("a non-streaming model rejects stream begin", MODEL, async () => { const m = await TranscribeModel.load(MODEL); try { diff --git a/catalog/_schema.json b/catalog/_schema.json index 697287654..219e539b4 100644 --- a/catalog/_schema.json +++ b/catalog/_schema.json @@ -24,6 +24,10 @@ "type": "string", "description": "Loader family (the GGUF's general.architecture)" }, + "role": { + "enum": ["asr", "diarize"], + "description": "The role the model is published for (transcribe_model_roles). Absent means asr." + }, "display_name": { "type": "string" }, diff --git a/catalog/diar_streaming_sortformer_4spk-v2.1.json b/catalog/diar_streaming_sortformer_4spk-v2.1.json index 3e243a663..845a037e0 100644 --- a/catalog/diar_streaming_sortformer_4spk-v2.1.json +++ b/catalog/diar_streaming_sortformer_4spk-v2.1.json @@ -2,6 +2,7 @@ "schema": "transcribe-catalog-v1", "variant": "diar_streaming_sortformer_4spk-v2.1", "family": "sortformer", + "role": "diarize", "display_name": "diar_streaming_sortformer_4spk-v2.1", "params": 117711368, "license": { diff --git a/docs/bindings.md b/docs/bindings.md index 51bd6d5c8..9440f3621 100644 --- a/docs/bindings.md +++ b/docs/bindings.md @@ -96,6 +96,44 @@ explicit unsupported combinations return `UNSUPPORTED_TIMESTAMPS`; `AUTO` chooses the richest granularity compatible with the selected task. Result objects own copies of every row and remain valid after the next run. +That contract is for ASR models that attribute speakers inside a transcript. + +## Compute lock and stream lease + +The C library allows one compute in flight per model and leaves enforcing it +to the caller. Every first-class binding enforces it with a model-wide lock +shared by every session of every role: a call waits behind any other compute +on the same model. + +The bindings are also stricter than C about streams. `transcribe.h` lets +several sessions of one model each hold an active stream and interleave +their feeds; a binding allows one active stream per model. Starting a stream +takes the model's stream lease, and ending it (finalize, reset, a feed that +leaves the stream FAILED, or dropping / closing the stream) releases it. +While the lease is held, `run`, `run_batch`, a new stream and a diarize run +on any session of that model raise `Busy` instead of waiting. A feed +rejected before the native call (e.g. NaN input) keeps the lease. + +## Roles and the DIARIZE session + +Each binding exposes the role mask as `Model.roles` (Python `frozenset[Role]`, +TypeScript `readonly ('asr' | 'diarize')[]`, Rust `Roles`, Swift `Roles` +option set). Status 20 surfaces as `UnsupportedRole` (`.unsupportedRole` in +Swift), including from capabilities on a model without ASR. + +A DIARIZE model (`docs/roles.md`) gets its own session type: `DiarizeSession` +from `model.diarize_session()` (Python, Rust), `model.diarizeSession()` +(Swift) or `model.createDiarizeSession()` (TypeScript), plus `diarize_info` / +`diarizeInfo` (sample rate, max speakers). `run(pcm, …)` returns copied-out +speaker-turn rows (the same row type as above) and takes the family's +diarize extension (Sortformer: the preset on the DIARIZE_RUN slot). + +A diarize run follows the same execution rules as an ASR run: it holds the +model-wide compute lock, waits behind other compute on the model, raises +`Busy` while a stream on any session of the model holds the stream lease, +keeps its model alive, honours cancellation, and defers native frees that +race an in-flight call. + ## Raw text Every first-class binding exposes `raw_text` / `rawText` on the materialized diff --git a/docs/environment-variables.md b/docs/environment-variables.md index f8147112b..053b0f7e6 100644 --- a/docs/environment-variables.md +++ b/docs/environment-variables.md @@ -86,7 +86,7 @@ its var is unset. Convention: `TRANSCRIBE__GGUF`. | `TRANSCRIBE_PARAKEET_UNIFIED_GGUF` | `parakeet_buffered_stream_eos_smoke`, `stream_offline_interleave_smoke` | | `TRANSCRIBE_GIGAAM_GGUF` | `gigaam_workspace_release_smoke` | | `TRANSCRIBE_MULTITALKER_BUNDLE_GGUF` | `parakeet_multitalker_e2e_smoke` | -| `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_stream_ext_unit` | +| `TRANSCRIBE_SORTFORMER_GGUF` | `sortformer_diarize_unit`, `cli_diarize_smoke` | | `TRANSCRIBE_COHERE_GGUF` | `cohere_real_smoke`, `cohere_e2e_smoke` | | `TRANSCRIBE_GRANITE5_CTC_GGUF` | `granite5_ctc_real_smoke`, `granite5_ctc_e2e_smoke` | | `TRANSCRIBE_WHISPER_GGUF` | `whisper_e2e_smoke`, `whisper_tokenize_parity` | diff --git a/docs/extension-kinds.md b/docs/extension-kinds.md index cb8dd94da..c2df730cf 100644 --- a/docs/extension-kinds.md +++ b/docs/extension-kinds.md @@ -32,6 +32,8 @@ entry-point family whose `family` field the typed extension is pointed at: by `transcribe_run`. - `TRANSCRIBE_EXT_SLOT_STREAM` — `transcribe_stream_params::family`, validated by `transcribe_stream_begin`. +- `TRANSCRIBE_EXT_SLOT_DIARIZE_RUN` — `transcribe_diarize_params::family`, + validated by `transcribe_diarize_run`. Mismatched-slot extensions are rejected with `TRANSCRIBE_ERR_INVALID_ARG` at the entry point. A future slot (e.g. `_SESSION`, `_MODEL_LOAD`, @@ -47,7 +49,8 @@ parameter; existing kinds keep their slot for life. | `0x5453534D` | `MSST` | `STREAM` | `TRANSCRIBE_EXT_KIND_MOONSHINE_STREAMING_STREAM` | moonshine_streaming | `include/transcribe/moonshine_streaming.h` | | `0x4E524857` | `WHRN` | `RUN` | `TRANSCRIBE_EXT_KIND_WHISPER_RUN` | whisper | `include/transcribe/whisper.h` | | `0x54535256` | `VRST` | `STREAM` | `TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM` | voxtral_realtime | `include/transcribe/voxtral_realtime.h` | -| `0x54534653` | `SFST` | `RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM` | sortformer | `include/transcribe/sortformer.h` | +| `0x54534653` | `SFST` | `RUN` | retired in 0.4 (was `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM`); never reuse | sortformer | - | +| `0x52444653` | `SFDR` | `DIARIZE_RUN` | `TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE` | sortformer | `include/transcribe/sortformer.h` | Empty rows reserved for future allocations: diff --git a/docs/migrating-to-0.4.md b/docs/migrating-to-0.4.md new file mode 100644 index 000000000..42c8ed57f --- /dev/null +++ b/docs/migrating-to-0.4.md @@ -0,0 +1,79 @@ +# Migrating to transcribe.cpp 0.4 + +Version 0.4 introduces **roles** (`docs/roles.md`): a loaded model serves +one or more kinds of work, each with its own API and session type. Rebuild +native consumers against the 0.4 headers and upgrade each language package +and native provider together. + +## Diarizers moved to the DIARIZE role + +Models whose product is speaker turns (Sortformer) no longer run through the +ASR API. `transcribe_model_roles()` reports `TRANSCRIBE_ROLE_DIARIZE` for +them, and `transcribe_session_init` / `transcribe_open` return +`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`. + +| 0.3 | 0.4 | +| --- | --- | +| `transcribe_session_init(model, ...)` | `transcribe_diarize_session_init(model, ...)` | +| `transcribe_run(session, pcm, n, &run_params)` | `transcribe_diarize_run(session, pcm, n, &diarize_params)` | +| `transcribe_n_speaker_segments` / `transcribe_get_speaker_segment` | `transcribe_diarize_n_segments` / `transcribe_diarize_get_segment` (same `transcribe_speaker_segment` rows) | +| `transcribe_sortformer_stream_ext` (`SFST`, RUN slot) | `transcribe_sortformer_diarize_ext` (`SFDR`, DIARIZE_RUN slot); same preset enum | +| `transcribe_model_supports(sortformer, TRANSCRIBE_FEATURE_DIARIZATION)` was true | false; check `transcribe_model_roles()` instead | +| `transcribe-cli --batch ... --batch-jsonl` "speakers" for Sortformer | `transcribe-cli -m sortformer.gguf audio.wav` prints the segments (`-o PATH` writes them); `--batch` is ASR-only | + +Segments are byte-identical to 0.3 for the same audio, preset and backend. +Speaker-attributing ASR models (granite, moss, multitalker parakeet) are +unchanged; see `docs/roles.md`. + +## Capabilities are ASR-only + +`transcribe_model_get_capabilities` returns `TRANSCRIBE_ERR_UNSUPPORTED_ROLE` +for a model without the ASR role, instead of a zeroed struct. Role facts come +from the role's own query (`transcribe_diarize_get_info`). + +## Behavior changes on every model + +- **Non-finite audio is rejected.** `transcribe_run`, `transcribe_run_batch` + and `transcribe_stream_feed` return `TRANSCRIBE_ERR_INVALID_ARG` when any + sample is NaN or +-Inf, before the previous result is touched. In a batch, + one such utterance rejects the whole batch. A rejected feed leaves the + stream ACTIVE. Silence is still valid. +- **A stream whose family hook throws now ends.** begin / feed / finalize + mark it FAILED with `last_status` equal to the returned status (OOM or + BACKEND); `transcribe_stream_reset` always ends IDLE. +- **Host thread-pool failures return an error** instead of terminating the + process. + +## Language bindings + +- New `Model.roles`, `DiarizeSession` and an `UnsupportedRole` error for + status 20 (`.unsupportedRole` in Swift); see `docs/bindings.md`. The + Sortformer ASR-path extension (`SortformerStreamOptions` and equivalents) + is removed. Sortformer runs through a diarize session instead: + + | Binding | 0.3 | 0.4 | + | --- | --- | --- | + | Python | `model.session()` + `SortformerStreamOptions` | `model.diarize_session()` + `SortformerDiarizeOptions` | + | Rust | `RunExtension::Sortformer(SortformerStreamOptions)` | `model.diarize_session()` + `DiarizeOptions` with `DiarizeExtension::Sortformer(SortformerDiarizeOptions)` | + | Swift | `RunExtension.sortformer(SortformerStreamOptions)` | `model.diarizeSession()` + `DiarizeOptions(family: .sortformer(SortformerDiarizeOptions(...)))` | + | TypeScript | `{ kind: "sortformer" }` on the `run` slot | `model.createDiarizeSession()` + `{ kind: "sortformer_diarize" }` on the new `"diarize_run"` slot | + + In every binding, `capabilities` on a model without ASR (Sortformer) + raises / throws / returns `UnsupportedRole` instead of reporting a zeroed + struct, and `supports(diarization)` is false for Sortformer; use + `Model.roles`. +- A stream that ends FAILED inside a feed (abort, or a native error) + releases the model's stream lease in every binding, so other sessions on + the model can run without `Busy`. A feed rejected before the native call + (e.g. NaN) keeps the lease and the stream stays usable. +- **Rust:** `Model::capabilities()` returns `Result` + (`Err(Error::UnsupportedRole)` on a model without ASR). `ExtSlot` gains + `DiarizeRun` and `AbiStruct` gains the three diarize structs; both are now + `#[non_exhaustive]`, so later roles add variants without another break. +- **Swift:** `Model.capabilities` is `get throws`. +- **Python:** calls on one model now serialize (they used to race), and a + run / run_batch / second stream on any session of a model with an active + stream raises `transcribe_cpp.Busy`, matching the other bindings. On the + stream's own session these raised `InvalidArgument` in 0.3. +- **TypeScript:** an unknown Sortformer preset string is rejected instead of + silently using the default. diff --git a/docs/models/diar_streaming_sortformer_4spk-v2.1.md b/docs/models/diar_streaming_sortformer_4spk-v2.1.md index 93952f7e0..95a2a02e2 100644 --- a/docs/models/diar_streaming_sortformer_4spk-v2.1.md +++ b/docs/models/diar_streaming_sortformer_4spk-v2.1.md @@ -63,12 +63,12 @@ Q5_K_M on one AMI meeting; details in cmake -B build cmake --build build -# Speaker segments as JSON (one line per file): -echo audio.wav > files.txt build/bin/transcribe-cli \ -m models/diar_streaming_sortformer_4spk-v2.1/diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf \ - --batch files.txt --batch-jsonl -# {"file":"audio.wav","text":"","speakers":[{"t0_ms":320,"t1_ms":2400,"speaker_id":1},...]} + audio.wav +# speaker segments: 5 +# [ 0.32 -> 2.40] S1 +# ... ``` If your audio is not already 16 kHz mono WAV, convert it first: @@ -77,16 +77,17 @@ If your audio is not already 16 kHz mono WAV, convert it first: ffmpeg -i input.mp3 -ar 16000 -ac 1 output.wav ``` -From the C API, read results via `transcribe_n_speaker_segments` / -`transcribe_get_speaker_segment` (the transcript accessors return empty -text). The streaming operating point (latency / accuracy trade-off) is -selected with the run extension in `include/transcribe/sortformer.h`: +From the C API, use the DIARIZE role (`include/transcribe/diarize.h`): +`transcribe_diarize_session_init`, `transcribe_diarize_run`, then +`transcribe_diarize_n_segments` / `transcribe_diarize_get_segment`. The +streaming operating point (latency / accuracy trade-off) is selected with +the extension in `include/transcribe/sortformer.h`: ```c -transcribe_sortformer_stream_ext ext; -transcribe_sortformer_stream_ext_init(&ext); /* DEFAULT = model cfg */ +transcribe_sortformer_diarize_ext ext; +transcribe_sortformer_diarize_ext_init(&ext); /* DEFAULT = model cfg */ ext.preset = TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY; -run_params.family = &ext.ext; +diarize_params.family = &ext.ext; ``` `VERY_HIGH_LATENCY` (~30 s lookahead) is the offline-file operating @@ -150,8 +151,8 @@ full 39-minute AMI meeting (87 compression calls). - **`LOW_LATENCY` is compute-heavy on CPU** (~1.2x realtime on Apple M4; the 0.5 s chunks rebuild the compute graph often). Use Metal or a higher-latency preset for offline files. -- **No batch fast path** (`run_batch`); multi-file CLI batches run - serially. +- **No batch mode**: run one file per call (`transcribe-cli --batch` is + ASR-only). ## Reproduction diff --git a/docs/porting/families/sortformer.md b/docs/porting/families/sortformer.md index 118c20146..09032cab1 100644 --- a/docs/porting/families/sortformer.md +++ b/docs/porting/families/sortformer.md @@ -13,11 +13,11 @@ Shipped matrix: **F32 + F16 + Q8_0** (k tiers withdrawn; see "Quant policy `handy-computer/diar_streaming_sortformer_4spk-v2.1-gguf`. Batch posture: **ACCEPTED GAP — no `run_batch()`** (user-approved Stage 4 -deferral; single-session `transcribe_run` is the shipped path; revisit with +deferral; single-session `transcribe_diarize_run` is the shipped path; revisit with multitalker interop). Streaming posture: **natively streaming — PASS**; the -chunk/lookahead contract is the preset menu in "Public API (run extension)" +chunk/lookahead contract is the preset menu in "Public API (diarize extension)" (default = GGUF-shipped checkpoint cfg; very_high_latency ~30.4 s lookahead -... low_latency ~1.04 s), exposed via `transcribe_sortformer_stream_ext`. +... low_latency ~1.04 s), exposed via `transcribe_sortformer_diarize_ext`. Push-audio `transcribe_stream_*` entry point is future work (STREAM-slot kind reserved by design). @@ -57,21 +57,20 @@ verbatim). A diarizer meeting neither clause (e.g. a pyannote segmentation+clustering port: new architecture, new dependency surface, no in-repo ASR consumer) is out of scope. Standalone diarization output is exposed because it falls out of the multitalker dependency for free, via -the pre-existing transcript-independent `transcribe_speaker_segment` ABI — -no diarizer-specific output surface was added. +the DIARIZE role (`include/transcribe/diarize.h`). -## Public API (run extension) +## Public API (diarize extension) -`include/transcribe/sortformer.h` — `TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM` -(`SFST`, RUN slot; registered in `docs/extension-kinds.md`). A run produces -no text; results are read via `transcribe_n_speaker_segments` / -`transcribe_get_speaker_segment` (`TRANSCRIBE_FEATURE_DIARIZATION`). +Sortformer serves the DIARIZE role only (`include/transcribe/diarize.h`). +`include/transcribe/sortformer.h` — `TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE` +(`SFDR`, DIARIZE_RUN slot; registered in `docs/extension-kinds.md`). The +0.3 RUN-slot kind `SFST` is retired. ```c -transcribe_sortformer_stream_ext ext; -transcribe_sortformer_stream_ext_init(&ext); /* preset = DEFAULT (GGUF cfg) */ +transcribe_sortformer_diarize_ext ext; +transcribe_sortformer_diarize_ext_init(&ext); /* preset = DEFAULT (GGUF cfg) */ ext.preset = TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY; -run_params.family = &ext.ext; /* probe accepts_ext_kind first */ +diarize_params.family = &ext.ext; /* probe accepts_ext_kind first */ ``` Preset menu (frames are 80 ms; chunk/rc/fifo/update/spkcache geometry per @@ -90,8 +89,8 @@ these bundles are accuracy-validated. Precedence: GGUF cfg < ext preset < `TRANSCRIBE_SORTFORMER_STREAM_PRESET` env < per-field env (env layers are validation hooks; `small` is a diagnostic-only env preset, deliberately not in the public enum). An out-of-range preset or wrong-kind ext is -rejected pre-clear (`run_validate`), preserving the previous result. -Unit test: `tests/sortformer_stream_ext_unit.cpp` +rejected pre-clear (`DiarizeOps::run_validate`), preserving the previous +result. Unit test: `tests/sortformer_diarize_unit.cpp` (`TRANSCRIBE_SORTFORMER_GGUF`-gated). ## Quant policy (Stage 7) diff --git a/docs/roles.md b/docs/roles.md new file mode 100644 index 000000000..7330de12a --- /dev/null +++ b/docs/roles.md @@ -0,0 +1,30 @@ +# Roles + +A loaded model serves one or more **roles**, the kinds of work it can do. +`transcribe_model_roles(model)` returns them as a bitmask of +`transcribe_role`; an entry point that needs a role the model lacks returns +`TRANSCRIBE_ERR_UNSUPPORTED_ROLE`. + +| Role | API | Session | Product | +|---|---|---|---| +| `TRANSCRIBE_ROLE_ASR` | `include/transcribe.h` | `transcribe_session` | text, timestamps, speaker-attributed segments | +| `TRANSCRIBE_ROLE_DIARIZE` | `include/transcribe/diarize.h` | `transcribe_diarize_session` | who spoke when | + +Every role follows the same rules: + +- **One load per model**, with its own backend (`transcribe_model_load_params`). +- **Sessions.** Each role has its own session type created from a model. + A session is used by one thread at a time; the model outlives its + sessions; at most one compute is in flight per model across all sessions + of all roles. Different models compute in parallel. +- **Input.** 16 kHz mono float32 PCM; NaN / Inf is `INVALID_ARG`. +- **Results** are copied out and replaced by the next run; malformed input + is rejected before the previous result is touched. + +ASR models that attribute speakers inside a transcript (granite, moss, +multitalker parakeet) keep `transcribe_run_params::diarize` and stay ASR. +The DIARIZE role is for models whose product is speaker turns +(Sortformer). + +Family extensions name the entry point they apply to through their slot; +see `docs/extension-kinds.md`. diff --git a/examples/cli/CMakeLists.txt b/examples/cli/CMakeLists.txt index 2b1284cf8..07a681522 100644 --- a/examples/cli/CMakeLists.txt +++ b/examples/cli/CMakeLists.txt @@ -6,6 +6,8 @@ add_executable(transcribe-cli main.cpp + asr.cpp + diarize.cpp ) target_link_libraries(transcribe-cli diff --git a/examples/cli/asr.cpp b/examples/cli/asr.cpp new file mode 100644 index 000000000..071ba338b --- /dev/null +++ b/examples/cli/asr.cpp @@ -0,0 +1,962 @@ +// asr.cpp - transcribe-cli ASR driver: batch and single-file transcription +// through the public ASR API (transcribe_run, transcribe_run_batch, +// transcribe_stream_*), plus the JSONL / text output for them. + +#include "cli.h" +#include "transcribe.h" +#include "transcribe/parakeet.h" +#include "transcribe/voxtral_realtime.h" +#include "transcribe/whisper.h" +#include "wav.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using transcribe_cli::cli_args; + +namespace { + +// Minimal JSON string escape: covers the characters MUST be escaped by +// the JSON spec (quote, backslash, control chars). Transcribed text is +// short UTF-8 in practice; we don't need unicode escaping. +std::string json_escape(const char * s) { + std::string out; + for (const char * p = s ? s : ""; *p; ++p) { + const unsigned char c = static_cast(*p); + if (c == '"') { + out += "\\\""; + } else if (c == '\\') { + out += "\\\\"; + } else if (c == '\n') { + out += "\\n"; + } else if (c == '\r') { + out += "\\r"; + } else if (c == '\t') { + out += "\\t"; + } else if (c < 0x20) { + char buf[8]; + std::snprintf(buf, sizeof(buf), "\\u%04x", c); + out += buf; + } else { + out += static_cast(c); + } + } + return out; +} + +// Shared row formatter for segments_json / batch_segments_json below. +std::string segment_row_json(const struct transcribe_segment & seg) { + std::string out; + char head[128]; + if (seg.speaker_id > 0) { + std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"text\":\"", + static_cast(seg.t0_ms), static_cast(seg.t1_ms), + static_cast(seg.speaker_id)); + } else { + std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"text\":\"", + static_cast(seg.t0_ms), static_cast(seg.t1_ms)); + } + out += head; + out += json_escape(seg.text != nullptr ? seg.text : ""); + out += "\"}"; + return out; +} + +// Build the ",\"segments\":[...]" fragment when the context has real +// segment rows: segment timestamps, or speaker-attributed turns (which +// may carry no timing — granite's speaker-attribution task). Returns an +// empty string otherwise (the library still populates a single dummy +// entry when the kind is NONE, but with zero timing — don't pollute the +// JSON with it). +std::string segments_json(const transcribe_session * ctx) { + if (transcribe_returned_timestamp_kind(ctx) == TRANSCRIBE_TIMESTAMPS_NONE && + transcribe_n_speaker_segments(ctx) <= 0) { + return {}; + } + const int n_seg = transcribe_n_segments(ctx); + if (n_seg <= 0) { + return {}; + } + std::string out = ",\"segments\":["; + for (int s = 0; s < n_seg; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_get_segment(ctx, s, &seg); + out += segment_row_json(seg); + } + out += "]"; + return out; +} + +// Batch variant: segments JSON for utterance `i` of a transcribe_run_batch +// result, using the indexed transcribe_batch_* accessors. +std::string batch_segments_json(const transcribe_session * ctx, int i) { + if (transcribe_batch_returned_timestamp_kind(ctx, i) == TRANSCRIBE_TIMESTAMPS_NONE && + transcribe_batch_n_speaker_segments(ctx, i) <= 0) { + return {}; + } + const int n_seg = transcribe_batch_n_segments(ctx, i); + if (n_seg <= 0) { + return {}; + } + std::string out = ",\"segments\":["; + for (int s = 0; s < n_seg; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_batch_get_segment(ctx, i, s, &seg); + out += segment_row_json(seg); + } + out += "]"; + return out; +} + +// ",\"raw_text\":\"...\"" fragment: the model's pre-cleanup decode +// (transcribe_raw_text). Emitted only when it differs from the clean text, +// so JSONL stays compact for families with no post-processing. +std::string raw_text_json(const char * raw, const char * clean) { + if (raw == nullptr || raw[0] == '\0' || (clean != nullptr && std::strcmp(raw, clean) == 0)) { + return {}; + } + std::string out = ",\"raw_text\":\""; + out += json_escape(raw); + out += "\""; + return out; +} + +// A decode cut short before end-of-stream (budget or repetition stop): non-OK, +// but the partial transcript is preserved (see docs/input-limits.md). +bool is_cut_short(transcribe_status st) { + return st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED || st == TRANSCRIBE_ERR_OUTPUT_REPETITION; +} + +const char * cut_short_label(transcribe_status st) { + return st == TRANSCRIBE_ERR_OUTPUT_REPETITION ? "stopped repeating" : "truncated"; +} + +// ",\"speakers\":[...]" fragment: the "who spoke when" rows. Emitted only +// when the run produced speaker segments. p is omitted unless finite +// (NaN — "model provides no confidence" — is not representable in JSON). +std::string speaker_row_json(const struct transcribe_speaker_segment & row) { + char buf[160]; + if (std::isfinite(row.p)) { + std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"p\":%.4f}", + static_cast(row.t0_ms), static_cast(row.t1_ms), + static_cast(row.speaker_id), static_cast(row.p)); + } else { + std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d}", + static_cast(row.t0_ms), static_cast(row.t1_ms), + static_cast(row.speaker_id)); + } + return buf; +} + +std::string speakers_json(const transcribe_session * ctx) { + const int n = transcribe_n_speaker_segments(ctx); + if (n <= 0) { + return {}; + } + std::string out = ",\"speakers\":["; + for (int s = 0; s < n; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + (void) transcribe_get_speaker_segment(ctx, s, &row); + out += speaker_row_json(row); + } + out += "]"; + return out; +} + +std::string batch_speakers_json(const transcribe_session * ctx, int i) { + const int n = transcribe_batch_n_speaker_segments(ctx, i); + if (n <= 0) { + return {}; + } + std::string out = ",\"speakers\":["; + for (int s = 0; s < n; ++s) { + if (s > 0) { + out += ","; + } + struct transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + (void) transcribe_batch_get_speaker_segment(ctx, i, s, &row); + out += speaker_row_json(row); + } + out += "]"; + return out; +} + +// Point rp's generic prompting fields at args' storage; vocabulary_ptrs +// backs rp.vocabulary and must outlive the run. +void apply_prompting(const cli_args & args, transcribe_run_params & rp, std::vector & vocabulary_ptrs) { + if (args.instruct) { + rp.task = TRANSCRIBE_TASK_INSTRUCT; + } + vocabulary_ptrs.clear(); + for (const std::string & term : args.vocabulary) { + vocabulary_ptrs.push_back(term.c_str()); + } + rp.vocabulary = vocabulary_ptrs.empty() ? nullptr : vocabulary_ptrs.data(); + rp.n_vocabulary = static_cast(vocabulary_ptrs.size()); + rp.prompt = args.prompt.empty() ? nullptr : args.prompt.c_str(); + rp.prefix = args.prefix.empty() ? nullptr : args.prefix.c_str(); +} + +} // namespace + +namespace transcribe_cli { + +bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { + if (output == nullptr) { + return true; + } + const char * value = text != nullptr ? text : ""; + *output << value; + if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { + *output << '\n'; + } + output->flush(); + if (!*output) { + std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); + return false; + } + return true; +} + +int run_asr_batch(const cli_args & args, std::ofstream * output) { + bool output_ok = true; + if (args.model_path.empty()) { + std::fprintf(stderr, "error: --batch requires --model\n"); + return EXIT_FAILURE; + } + + std::vector wav_paths; + { + std::ifstream fin(args.batch_file); + if (!fin) { + std::fprintf(stderr, "error: cannot open batch file %s\n", args.batch_file.c_str()); + return EXIT_FAILURE; + } + std::string line; + while (std::getline(fin, line)) { + while (!line.empty() && + (line.back() == '\n' || line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { + line.pop_back(); + } + if (!line.empty()) { + wav_paths.push_back(line); + } + } + } + if (wav_paths.empty()) { + std::fprintf(stderr, "error: batch file is empty\n"); + return EXIT_FAILURE; + } + if (!args.batch_jsonl) { + std::fprintf(stderr, "batch: %zu files\n", wav_paths.size()); + } + + struct transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; + } + struct transcribe_model * model = nullptr; + const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); + if (load_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "model load: %s\n", transcribe_status_string(load_st)); + return EXIT_FAILURE; + } + if ((transcribe_model_roles(model) & TRANSCRIBE_ROLE_ASR) == 0) { + std::fprintf(stderr, + "error: --batch is ASR-only and this model has no ASR role; " + "run it on one file instead: transcribe-cli -m MODEL audio.wav\n"); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + struct transcribe_session_params cp; + transcribe_session_params_init(&cp); + cp.n_threads = args.n_threads; + cp.n_ctx = args.n_ctx; + cp.kv_type = args.kv_type; + struct transcribe_session * ctx = nullptr; + const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); + if (init_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + struct transcribe_run_params rp; + transcribe_run_params_init(&rp); + if (args.translate) { + rp.task = TRANSCRIBE_TASK_TRANSLATE; + } + std::vector vocabulary_ptrs; + apply_prompting(args, rp, vocabulary_ptrs); + if (!args.language.empty()) { + rp.language = args.language.c_str(); + } + if (!args.target_language.empty()) { + rp.target_language = args.target_language.c_str(); + } + rp.timestamps = args.timestamps; + rp.spec_k_drafts = args.spec_k_drafts; + + if (args.itn_set) { + rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + } + if (args.canary_pnc_set) { + rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + } + if (args.diarize_set) { + rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + } + + // Whisper run extension. Allocated outside rp's scope so its + // bytes outlive the per-file loop below; the library copies + // initial_prompt/prompt_tokens before transcribe_run returns, + // but rp aliases &wx.ext for the run call itself. + struct transcribe_whisper_run_ext wx; + transcribe_whisper_run_ext_init(&wx); + if (args.whisper_set) { + if (!args.initial_prompt.empty()) { + wx.initial_prompt = args.initial_prompt.c_str(); + } + wx.condition_on_prev_tokens = args.condition_on_prev_tokens; + if (args.temperature_set) { + wx.temperature = args.temperature; + } + wx.prompt_condition = args.prompt_condition; + if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { + rp.family = &wx.ext; + } + } + + if (args.keep_special_tags) { + rp.keep_special_tags = true; + } + + // Emit a batch header line once, before any per-file output. Carries + // the one-shot load time so downstream WER tooling can record it + // without parsing stderr. Per-file lines follow on subsequent lines. + if (args.batch_jsonl) { + struct transcribe_timings load_tm; + transcribe_timings_init(&load_tm); + (void) transcribe_get_timings(ctx, &load_tm); + std::printf("{\"type\":\"batch_header\",\"load_ms\":%.1f}\n", (double) load_tm.load_ms); + std::fflush(stdout); + } + + int n_ok = 0; + int n_truncated = 0; // result-bearing: hit the generation cap, partial hyp emitted + int n_repeating = 0; // result-bearing: stopped when the output looped, partial hyp emitted + int n_fail = 0; // no usable result (wav load / backend / unsupported / whole-batch) + + // Offline batched path: group up to batch_size utterances into one + // transcribe_run_batch call. Mutually exclusive with the streaming + // path (--stream-chunk-ms), which is per-utterance by construction. + // Per-file JSONL output is byte-identical in shape to the serial + // path so the WER harness (scripts/wer/run.py) consumes either. + const bool use_batched = args.batch_size > 1 && args.stream_chunk_ms <= 0; + if (use_batched) { + const size_t total = wav_paths.size(); + const size_t group = static_cast(args.batch_size); + for (size_t base = 0; base < total; base += group) { + const size_t end = std::min(total, base + group); + + // Load each wav in the group. A wav that fails to load is + // emitted as an error line and excluded from the batch, so + // its failure cannot abort its neighbours. + std::vector> pcms; + std::vector pcm_ptrs; + std::vector n_samps; + std::vector src_index; // -> wav_paths + for (size_t i = base; i < end; ++i) { + std::vector pcm; + std::string wav_err; + if (!transcribe_cli::load_wav_mono_16k(wav_paths[i], pcm, wav_err)) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav_paths[i].c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"wav: %s\"}\n", + file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); + } else { + std::fprintf(stderr, "SKIP %s: %s\n", wav_paths[i].c_str(), wav_err.c_str()); + } + ++n_fail; + std::fflush(stdout); + continue; + } + pcms.push_back(std::move(pcm)); + src_index.push_back(i); + } + for (auto & p : pcms) { + pcm_ptrs.push_back(p.data()); + n_samps.push_back(static_cast(p.size())); + } + if (pcms.empty()) { + continue; + } + + const transcribe_status bst = + transcribe_run_batch(ctx, pcm_ptrs.data(), n_samps.data(), static_cast(pcm_ptrs.size()), &rp); + if (bst != TRANSCRIBE_OK && transcribe_batch_n_results(ctx) == 0) { + // Whole-batch failure (no per-utterance results): emit an + // error line per file in the group and continue. + for (size_t k = 0; k < src_index.size(); ++k) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav_paths[src_index[k]].c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"%s\"}\n", + file_esc.c_str(), json_escape(transcribe_status_string(bst)).c_str()); + } + ++n_fail; + } + std::fflush(stdout); + continue; + } + + for (size_t k = 0; k < src_index.size(); ++k) { + const std::string & wav = wav_paths[src_index[k]]; + const transcribe_status ust = transcribe_batch_status(ctx, static_cast(k)); + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the + // partial transcript is preserved and readable via + // transcribe_batch_full_text (see transcribe.h). Emit it as the hyp + // so downstream tooling scores the partial rather than an empty + // string; the error field below still tags it so the stop stays + // visible. + const bool result_present = ust == TRANSCRIBE_OK || is_cut_short(ust); + const char * text = ""; + if (result_present) { + const char * t = transcribe_batch_full_text(ctx, static_cast(k)); + if (t && *t) { + text = t; + } + } + if (ust == TRANSCRIBE_OK) { + ++n_ok; + } else if (ust == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + ++n_truncated; + } else if (ust == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + ++n_repeating; + } else { + ++n_fail; + } + if (args.batch_jsonl) { + const std::string escaped = json_escape(text); + std::string segments = batch_segments_json(ctx, static_cast(k)); + segments += batch_speakers_json(ctx, static_cast(k)); + segments += raw_text_json(transcribe_batch_raw_text(ctx, static_cast(k)), text); + std::string err_field; + if (ust != TRANSCRIBE_OK) { + err_field = ",\"error\":\""; + err_field += json_escape(transcribe_status_string(ust)); + err_field += "\""; + } + // Per-utterance timings: the batched encoder time is + // amortized across the batch, decode is real per-utt, + // so summing across utterances gives the true encode vs + // decode split for the whole batched run. + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_batch_get_timings(ctx, static_cast(k), &tm); + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"%s\"%s," + "\"mel_ms\":%.1f,\"encode_ms\":%.1f," + "\"decode_ms\":%.1f%s}\n", + file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, + (double) tm.decode_ms, err_field.c_str()); + } else { + std::printf("[%zu/%zu] %s", src_index[k] + 1, total, wav.c_str()); + if (ust == TRANSCRIBE_OK) { + std::printf("\n text: %s\n", text); + } else if (is_cut_short(ust)) { + std::printf(" (%s)\n text: %s\n", cut_short_label(ust), text); + } else { + std::printf(" ERROR: %s\n", transcribe_status_string(ust)); + } + } + output_ok = write_output_file(output, args.output_path, text) && output_ok; + std::fflush(stdout); + } + } + } else { + for (size_t i = 0; i < wav_paths.size(); ++i) { + const std::string & wav = wav_paths[i]; + + std::vector pcm; + std::string wav_err; + if (!transcribe_cli::load_wav_mono_16k(wav, pcm, wav_err)) { + if (args.batch_jsonl) { + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"\"," + "\"error\":\"wav: %s\"}\n", + file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); + } else { + std::fprintf(stderr, "SKIP %s: %s\n", wav.c_str(), wav_err.c_str()); + } + ++n_fail; + std::fflush(stdout); + continue; + } + + // Run. When --stream-chunk-ms > 0, drive the streaming API + // for this utterance (begin/feed/finalize) so the WER + // harness can measure cache-aware streaming output. + transcribe_status run_st = TRANSCRIBE_OK; + if (args.stream_chunk_ms > 0) { + struct transcribe_stream_params sp; + transcribe_stream_params_init(&sp); + struct transcribe_parakeet_stream_ext pkt_sp; + transcribe_parakeet_stream_ext_init(&pkt_sp); + struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; + transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); + struct transcribe_voxtral_realtime_stream_ext vx_sp; + transcribe_voxtral_realtime_stream_ext_init(&vx_sp); + const bool want_cache_aware = (args.stream_att_right >= 0); + const bool want_buffered = + args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; + if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { + pkt_sp.att_context_right = args.stream_att_right; + sp.family = &pkt_sp.ext; + } else if (want_buffered && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { + pkt_buf_sp.left_ms = args.stream_buf_left_ms; + pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; + pkt_buf_sp.right_ms = args.stream_buf_right_ms; + sp.family = &pkt_buf_sp.ext; + } else if (args.stream_voxtral_delay != -1 && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { + vx_sp.num_delay_tokens = args.stream_voxtral_delay; + sp.family = &vx_sp.ext; + } + run_st = transcribe_stream_begin(ctx, &rp, &sp); + if (run_st == TRANSCRIBE_OK) { + const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); + size_t pos = 0; + while (pos < pcm.size()) { + const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); + struct transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); + if (run_st != TRANSCRIBE_OK) { + break; + } + pos += take; + } + if (run_st == TRANSCRIBE_OK) { + struct transcribe_stream_update fin_upd; + transcribe_stream_update_init(&fin_upd); + run_st = transcribe_stream_finalize(ctx, &fin_upd); + } + } + } else { + run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + } + + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the partial + // transcript is preserved and readable via transcribe_full_text (see + // transcribe.h). Emit it as the hyp so downstream tooling scores the + // partial rather than an empty string; the error field below still + // tags it so the stop stays visible. + const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st); + const char * text = ""; + if (result_present) { + const char * t = transcribe_full_text(ctx); + if (t && *t) { + text = t; + } + } + if (run_st == TRANSCRIBE_OK) { + ++n_ok; + } else if (run_st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { + ++n_truncated; + } else if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + ++n_repeating; + } else { + ++n_fail; + } + + // Output. When run_st != OK we still emit a JSON line so + // batch consumers see one record per input wav, but we tag + // it with the error string. Without this the wav-load error + // path (above) reports errors but transcribe_run failures + // (e.g. unsupported language) produce empty hyp_text with no + // error tag, so downstream tools count them as silent + // successes. + if (args.batch_jsonl) { + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_get_timings(ctx, &tm); + const std::string escaped = json_escape(text); + std::string segments = segments_json(ctx); + segments += speakers_json(ctx); + segments += raw_text_json(transcribe_raw_text(ctx), text); + std::string err_field; + if (run_st != TRANSCRIBE_OK) { + err_field = ",\"error\":\""; + err_field += json_escape(transcribe_status_string(run_st)); + err_field += "\""; + } + const std::string file_esc = json_escape(wav.c_str()); + std::printf( + "{\"file\":\"%s\",\"text\":\"%s\"%s," + "\"mel_ms\":%.1f,\"encode_ms\":%.1f," + "\"decode_ms\":%.1f%s}\n", + file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, + (double) tm.decode_ms, err_field.c_str()); + } else { + std::printf("[%zu/%zu] %s", i + 1, wav_paths.size(), wav.c_str()); + if (run_st == TRANSCRIBE_OK) { + std::printf("\n text: %s\n", text); + } else if (is_cut_short(run_st)) { + std::printf(" (%s)\n text: %s\n", cut_short_label(run_st), text); + } else { + std::printf(" ERROR: %s\n", transcribe_status_string(run_st)); + } + } + output_ok = write_output_file(output, args.output_path, text) && output_ok; + std::fflush(stdout); + } + } + + if (!args.batch_jsonl) { + std::fprintf(stderr, "batch: %d ok, %d truncated, %d stopped repeating, %d failed out of %zu\n", n_ok, + n_truncated, n_repeating, n_fail, wav_paths.size()); + } + + transcribe_session_free(ctx); + transcribe_model_free(model); + // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing and do not fail the batch, but + // hard per-utterance failures must remain visible to automation. + return n_fail > 0 || !output_ok ? EXIT_FAILURE : EXIT_SUCCESS; +} + +int run_asr_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output) { + bool output_ok = true; + + struct transcribe_session_params cp; + transcribe_session_params_init(&cp); + cp.n_threads = args.n_threads; + cp.n_ctx = args.n_ctx; + cp.kv_type = args.kv_type; + struct transcribe_session * ctx = nullptr; + const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); + if (init_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + // Surface the effective input-length limit so it's obvious how much + // audio this session accepts (reflects --n-ctx). 0 means "no practical + // limit", which covers two different families: one that chunks long + // audio internally (FEATURE_LONG_FORM) and one that is genuinely + // unbounded and encodes the clip in a single pass. Distinguish them + // rather than asserting the first. See docs/input-limits.md. + { + struct transcribe_session_limits lim; + transcribe_session_limits_init(&lim); + if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { + if (lim.effective_max_audio_ms > 0) { + std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); + if (lim.effective_n_ctx > 0) { + std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, + (long long) (lim.max_kv_bytes >> 20)); + } + std::printf("\n"); + } else if (lim.effective_n_ctx > 0) { + // Capped family whose context is too small to fit any audio + // plus a prompt (e.g. an aggressively low --n-ctx). + std::printf( + " max audio: ~0 s (context %d tok too small for " + "audio + prompt)\n", + lim.effective_n_ctx); + } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { + std::printf(" max audio: unbounded (long audio chunked internally)\n"); + } else { + // No context cap and no chunker: the family encodes the + // whole clip in one pass (e.g. block-local attention, + // where cost is linear in audio length). + std::printf(" max audio: unbounded (whole clip in one pass)\n"); + } + } + } + + struct transcribe_run_params rp; + transcribe_run_params_init(&rp); + if (args.translate) { + rp.task = TRANSCRIBE_TASK_TRANSLATE; + } + std::vector vocabulary_ptrs; + apply_prompting(args, rp, vocabulary_ptrs); + if (!args.language.empty()) { + rp.language = args.language.c_str(); + } + if (!args.target_language.empty()) { + rp.target_language = args.target_language.c_str(); + } + rp.timestamps = args.timestamps; + rp.spec_k_drafts = args.spec_k_drafts; + + if (args.itn_set) { + rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; + } + if (args.canary_pnc_set) { + rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; + } + if (args.diarize_set) { + rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; + } + + struct transcribe_whisper_run_ext wx; + transcribe_whisper_run_ext_init(&wx); + if (args.whisper_set) { + if (!args.initial_prompt.empty()) { + wx.initial_prompt = args.initial_prompt.c_str(); + } + wx.condition_on_prev_tokens = args.condition_on_prev_tokens; + if (args.temperature_set) { + wx.temperature = args.temperature; + } + wx.prompt_condition = args.prompt_condition; + if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { + rp.family = &wx.ext; + } + } + + if (args.keep_special_tags) { + rp.keep_special_tags = true; + } + + // Streaming demo: drive transcribe_stream_begin/feed/finalize + // with fixed-size PCM chunks. Families with true per-feed + // partial decoding (moonshine_streaming) flip + // update.result_changed whenever the transcript advances; the + // CLI prints the live tentative text on each such feed. + // Families that only commit at finalize keep result_changed + // false until the finalize call. + transcribe_status run_st = TRANSCRIBE_OK; + if (args.stream_chunk_ms > 0) { + struct transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); + if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { + std::fprintf(stderr, + "stream: model does not advertise " + "supports_streaming; use a streaming-capable " + "model or drop --stream-chunk-ms\n"); + transcribe_session_free(ctx); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); + std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); + + struct transcribe_stream_params sp; + transcribe_stream_params_init(&sp); + struct transcribe_parakeet_stream_ext pkt_sp; + transcribe_parakeet_stream_ext_init(&pkt_sp); + struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; + transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); + struct transcribe_voxtral_realtime_stream_ext vx_sp; + transcribe_voxtral_realtime_stream_ext_init(&vx_sp); + const bool want_cache_aware = (args.stream_att_right >= 0); + const bool want_buffered = + args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; + if (want_cache_aware && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { + pkt_sp.att_context_right = args.stream_att_right; + sp.family = &pkt_sp.ext; + std::printf("stream: att_context_right=%d\n", args.stream_att_right); + } else if (want_buffered && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { + pkt_buf_sp.left_ms = args.stream_buf_left_ms; + pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; + pkt_buf_sp.right_ms = args.stream_buf_right_ms; + sp.family = &pkt_buf_sp.ext; + std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, args.stream_buf_chunk_ms, + args.stream_buf_right_ms); + } else if (args.stream_voxtral_delay != -1 && + transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, + TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { + vx_sp.num_delay_tokens = args.stream_voxtral_delay; + sp.family = &vx_sp.ext; + std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); + } + run_st = transcribe_stream_begin(ctx, &rp, &sp); + if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); + } else { + size_t pos = 0; + int feed_n = 0; + while (pos < pcm.size()) { + const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); + struct transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); + if (run_st != TRANSCRIBE_OK) { + std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); + break; + } + pos += take; + std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, (long long) upd.input_received_ms, + (long long) upd.buffered_ms); + if (upd.result_changed) { + const char * partial = transcribe_full_text(ctx); + std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); + } + std::printf("\n"); + ++feed_n; + } + if (run_st == TRANSCRIBE_OK) { + struct transcribe_stream_update fin_upd; + transcribe_stream_update_init(&fin_upd); + run_st = transcribe_stream_finalize(ctx, &fin_upd); + std::printf( + " finalize: status=%s " + "revision=%d input=%lld ms committed=%lld ms\n", + transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, + (long long) fin_upd.audio_committed_ms); + } + } + } else { + // --repeat N runs transcribe_run() N times for steady-state + // perf measurements. + for (int r = 0; r < args.repeat; ++r) { + run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); + if (run_st != TRANSCRIBE_OK) { + break; + } + } + } + std::printf("run: %s\n", transcribe_status_string(run_st)); + // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but + // preserve the partial transcript (see docs/input-limits.md), so show + // the result for them too — just flagged. + const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; + if (result_present) { + const char * text = transcribe_full_text(ctx); + std::printf("text: %s\n", (text && *text) ? text : "(empty)"); + output_ok = write_output_file(output, args.output_path, text) && output_ok; + + // A decode cut short before end-of-stream; the text above is + // incomplete. + if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { + std::printf( + " note: decode stopped when the output began repeating " + "itself (repeats dropped); transcript is incomplete\n"); + } else if (transcribe_was_truncated(ctx)) { + std::printf( + " note: output truncated (hit the model's " + "context/generation cap before end-of-stream); " + "transcript is incomplete\n"); + } + + const char * dl = transcribe_detected_language(ctx); + if (dl && *dl) { + std::printf("detected-language: %s\n", dl); + } + + const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); + const int n_seg = transcribe_n_segments(ctx); + const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; + if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { + std::printf("segments: %d\n", n_seg); + for (int i = 0; i < n_seg; ++i) { + struct transcribe_segment seg; + transcribe_segment_init(&seg); + (void) transcribe_get_segment(ctx, i, &seg); + char spk[16] = ""; + if (seg.speaker_id > 0) { + std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); + } + if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { + std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, + (seg.text != nullptr) ? seg.text : ""); + } else { + // Speaker-attributed turns without timing (granite SAA). + std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); + } + } + } + if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_wrd = transcribe_n_words(ctx); + std::printf("words: %d\n", n_wrd); + for (int i = 0; i < n_wrd; ++i) { + struct transcribe_word wrd; + transcribe_word_init(&wrd); + (void) transcribe_get_word(ctx, i, &wrd); + std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, + (wrd.text != nullptr) ? wrd.text : ""); + } + } + if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { + const int n_tok = transcribe_n_tokens(ctx); + std::printf("tokens: %d\n", n_tok); + for (int i = 0; i < n_tok; ++i) { + struct transcribe_token tok; + transcribe_token_init(&tok); + (void) transcribe_get_token(ctx, i, &tok); + std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, + (tok.text != nullptr) ? tok.text : ""); + } + } + } + + transcribe_print_timings(ctx); + + { + struct transcribe_timings tm; + transcribe_timings_init(&tm); + (void) transcribe_get_timings(ctx, &tm); + const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; + if (total_ms > 0.0 && duration_s > 0.0) { + std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, + duration_s); + } + } + + transcribe_session_free(ctx); + transcribe_model_free(model); + + if (run_st != TRANSCRIBE_OK || !output_ok) { + return EXIT_FAILURE; + } + + return EXIT_SUCCESS; +} + +} // namespace transcribe_cli diff --git a/examples/cli/cli.h b/examples/cli/cli.h new file mode 100644 index 000000000..fbe4b5b50 --- /dev/null +++ b/examples/cli/cli.h @@ -0,0 +1,132 @@ +// cli.h - shared declarations for the transcribe-cli example. +// +// main.cpp parses arguments, owns process setup (log sink, output file), and +// routes each model to its role's driver (asr.cpp, diarize.cpp). Shared +// helpers live in namespace transcribe_cli next to the WAV loader in +// examples/common. + +#pragma once + +#include "transcribe.h" +#include "transcribe/whisper.h" + +#include +#include +#include + +namespace transcribe_cli { + +struct cli_args { + std::string wav_path; + std::string model_path; + std::string language; + std::string target_language; // --target-language: target lang for translation + std::string batch_file; // --batch: one wav path per line + int batch_size = 0; // --batch-size: >1 groups utterances into + // transcribe_run_batch calls (offline only). + // 0/1 keeps the per-file serial loop. + bool translate = false; + bool instruct = false; // --task instruct + bool quiet = false; + bool list_devices = false; // --list-devices: print devices and exit + bool batch_jsonl = false; // --batch-jsonl: output JSONL + std::string output_path; // -o/--output: write raw text here + int repeat = 1; + int n_threads = 0; // 0 = library default (all cores) + int n_ctx = 0; // 0 = model's true max; >0 lowers the cap + transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; + transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; + int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device + transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; + + // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). + std::vector vocabulary; // --vocabulary TERMS / --vocabulary-file PATH + std::string prompt; // --prompt TEXT + std::string prefix; // --prefix TEXT + + // Whisper-family knobs. Ignored for non-Whisper models. + std::string initial_prompt; // --initial-prompt TEXT + bool whisper_set = false; + bool temperature_set = false; // --temperature F + float temperature = 0.0f; // tier-0 sampling temp + bool condition_on_prev_tokens = false; // --condition-on-prev-tokens + enum transcribe_whisper_prompt_condition prompt_condition = + TRANSCRIBE_WHISPER_PROMPT_FIRST_SEGMENT; // --prompt-condition first|all + + // SenseVoice / FunASR-Nano family knobs. The `--itn` flag is shared: + // it routes to whichever family the loaded model belongs to. Ignored + // by non-ITN-aware families. Unset leaves the library default in place, + // which differs per family (sensevoice: on; funasr-nano: off), so the + // initializer here is only read once --itn / --no-itn has been seen. + bool use_itn = false; // --itn / --no-itn + bool itn_set = false; + bool keep_special_tags = false; // --raw-tokens + + // Canary family knobs. Ignored by non-Canary families. + bool canary_pnc = true; // default: punctuation+caps on + bool canary_pnc_set = false; // --pnc / --no-pnc set this + + // Speaker diarization toggle (moss / granite-plus). Unset = library + // default (OFF). --diarize / --no-diarize set this. + bool diarize = true; + bool diarize_set = false; + + // Streaming demo: when > 0, the single-file path feeds the WAV + // through transcribe_stream_begin/feed/finalize in fixed-size + // ms-aligned chunks instead of one transcribe_run call. Requires + // the loaded model to advertise supports_streaming. Set by + // --stream-chunk-ms N. + int stream_chunk_ms = 0; + // Parakeet streaming: pick a right-context (lookahead) setting + // from the model's training menu. -1 = model default (max + // accuracy / max latency); 0/1/6/13 select the published + // nemotron-speech-streaming-en-0.6b settings. Set by + // --stream-att-right N. Ignored when stream_chunk_ms == 0. + int stream_att_right = -1; + // Parakeet buffered streaming (parakeet-unified-en-0.6b): override + // the (L, C, R) attention context tuple in milliseconds. -1 = use + // the model's default (highest-accuracy row of the training menu). + // Frame-aligned: the lib rounds each value down to the nearest + // post-subsample frame (80ms at 4x subsampling). Ignored when + // stream_chunk_ms == 0 or when the model is not buffered-streaming. + int stream_buf_left_ms = -1; + int stream_buf_chunk_ms = -1; + int stream_buf_right_ms = -1; + // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio + // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by + // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when + // the model is not voxtral_realtime. + int stream_voxtral_delay = -1; + // Speculative-decode draft length passed through to + // transcribe_run_params::spec_k_drafts on the offline path. -1 = family + // default (each family picks its tuned K). 0 = explicitly off. >0 = + // explicit K. Silently ignored by families without + // supports_spec_decode. Set by --spec-k-drafts N. + int spec_k_drafts = -1; +}; + +// -o/--output: write `text` (newline-terminated) to `output` if non-null. +// Returns false (after an error message naming `path`) on a write failure. +bool write_output_file(std::ofstream * output, const std::string & path, const char * text); + +// ASR batch driver (asr.cpp): loads the model, runs every listed file, prints, +// and returns the process exit code. +int run_asr_batch(const cli_args & args, std::ofstream * output); + +// Single-file drivers, one per role. main.cpp loads the audio and the model, +// prints them, and dispatches on transcribe_model_roles(); the driver takes +// ownership of the model and returns the process exit code. +int run_asr_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output); + +// With -o, writes the segment lines. +int run_diarize_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output); + +} // namespace transcribe_cli diff --git a/examples/cli/diarize.cpp b/examples/cli/diarize.cpp new file mode 100644 index 000000000..15a3a21ad --- /dev/null +++ b/examples/cli/diarize.cpp @@ -0,0 +1,76 @@ +// diarize.cpp - transcribe-cli DIARIZE driver: speaker segments for one file +// through include/transcribe/diarize.h. + +#include "transcribe/diarize.h" + +#include "cli.h" + +#include +#include +#include +#include +#include + +int transcribe_cli::run_diarize_file(const cli_args & args, + transcribe_model * model, + const std::vector & pcm, + double duration_s, + std::ofstream * output) { + // The DIARIZE role has no push-audio entry point yet; refuse rather than + // silently diarizing the whole file. + if (args.stream_chunk_ms > 0) { + std::fprintf(stderr, "stream: the diarize path has no streaming entry point; drop --stream-chunk-ms\n"); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + transcribe_diarize_session_params sp; + transcribe_diarize_session_params_init(&sp); + sp.n_threads = args.n_threads; + transcribe_diarize_session * session = nullptr; + transcribe_status st = transcribe_diarize_session_init(model, &sp, &session); + if (st != TRANSCRIBE_OK) { + std::fprintf(stderr, "diarize session init: %s\n", transcribe_status_string(st)); + transcribe_model_free(model); + return EXIT_FAILURE; + } + + // --repeat N runs transcribe_diarize_run() N times for steady-state perf + // measurements. + for (int r = 0; r < args.repeat; ++r) { + st = transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), nullptr); + if (st != TRANSCRIBE_OK) { + break; + } + } + std::printf("run: %s\n", transcribe_status_string(st)); + bool output_ok = true; + if (st == TRANSCRIBE_OK) { + const int n = transcribe_diarize_n_segments(session); + std::printf("speaker segments: %d\n", n); + std::string lines; + for (int i = 0; i < n; ++i) { + transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + transcribe_diarize_get_segment(session, i, &row); + char line[64]; + std::snprintf(line, sizeof(line), "[%7.2f -> %7.2f] S%d\n", row.t0_ms / 1000.0, row.t1_ms / 1000.0, + row.speaker_id); + std::printf(" %s", line); + lines += line; + } + output_ok = write_output_file(output, args.output_path, lines.c_str()); + } + + transcribe_timings tm; + transcribe_timings_init(&tm); + transcribe_diarize_get_timings(session, &tm); + const double total_ms = tm.mel_ms + tm.encode_ms; + if (total_ms > 0.0 && duration_s > 0.0) { + std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", duration_s * 1000.0 / total_ms, total_ms, duration_s); + } + + transcribe_diarize_session_free(session); + transcribe_model_free(model); + return st == TRANSCRIBE_OK && output_ok ? EXIT_SUCCESS : EXIT_FAILURE; +} diff --git a/examples/cli/main.cpp b/examples/cli/main.cpp index 1867af445..0d5f4c0b9 100644 --- a/examples/cli/main.cpp +++ b/examples/cli/main.cpp @@ -5,17 +5,11 @@ // and batch modes, offline and streaming paths, and per-family knobs. // Run with --help for the full option list. -#include "transcribe.h" -#include "transcribe/parakeet.h" -#include "transcribe/voxtral_realtime.h" -#include "transcribe/whisper.h" +#include "cli.h" #include "wav.h" -#include #include #include -#include -#include #include #include #include @@ -24,6 +18,8 @@ #include #include +using transcribe_cli::cli_args; + namespace { bool parse_device_index(const char * text, int & out) { @@ -40,289 +36,6 @@ bool parse_device_index(const char * text, int & out) { return true; } -// Minimal JSON string escape: covers the characters MUST be escaped by -// the JSON spec (quote, backslash, control chars). Transcribed text is -// short UTF-8 in practice; we don't need unicode escaping. -std::string json_escape(const char * s) { - std::string out; - for (const char * p = s ? s : ""; *p; ++p) { - const unsigned char c = static_cast(*p); - if (c == '"') { - out += "\\\""; - } else if (c == '\\') { - out += "\\\\"; - } else if (c == '\n') { - out += "\\n"; - } else if (c == '\r') { - out += "\\r"; - } else if (c == '\t') { - out += "\\t"; - } else if (c < 0x20) { - char buf[8]; - std::snprintf(buf, sizeof(buf), "\\u%04x", c); - out += buf; - } else { - out += static_cast(c); - } - } - return out; -} - -// Shared row formatter for segments_json / batch_segments_json below. -std::string segment_row_json(const struct transcribe_segment & seg) { - std::string out; - char head[128]; - if (seg.speaker_id > 0) { - std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"text\":\"", - static_cast(seg.t0_ms), static_cast(seg.t1_ms), - static_cast(seg.speaker_id)); - } else { - std::snprintf(head, sizeof(head), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"text\":\"", - static_cast(seg.t0_ms), static_cast(seg.t1_ms)); - } - out += head; - out += json_escape(seg.text != nullptr ? seg.text : ""); - out += "\"}"; - return out; -} - -// Build the ",\"segments\":[...]" fragment when the context has real -// segment rows: segment timestamps, or speaker-attributed turns (which -// may carry no timing — granite's speaker-attribution task). Returns an -// empty string otherwise (the library still populates a single dummy -// entry when the kind is NONE, but with zero timing — don't pollute the -// JSON with it). -std::string segments_json(const transcribe_session * ctx) { - if (transcribe_returned_timestamp_kind(ctx) == TRANSCRIBE_TIMESTAMPS_NONE && - transcribe_n_speaker_segments(ctx) <= 0) { - return {}; - } - const int n_seg = transcribe_n_segments(ctx); - if (n_seg <= 0) { - return {}; - } - std::string out = ",\"segments\":["; - for (int s = 0; s < n_seg; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_get_segment(ctx, s, &seg); - out += segment_row_json(seg); - } - out += "]"; - return out; -} - -// Batch variant: segments JSON for utterance `i` of a transcribe_run_batch -// result, using the indexed transcribe_batch_* accessors. -std::string batch_segments_json(const transcribe_session * ctx, int i) { - if (transcribe_batch_returned_timestamp_kind(ctx, i) == TRANSCRIBE_TIMESTAMPS_NONE && - transcribe_batch_n_speaker_segments(ctx, i) <= 0) { - return {}; - } - const int n_seg = transcribe_batch_n_segments(ctx, i); - if (n_seg <= 0) { - return {}; - } - std::string out = ",\"segments\":["; - for (int s = 0; s < n_seg; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_batch_get_segment(ctx, i, s, &seg); - out += segment_row_json(seg); - } - out += "]"; - return out; -} - -// ",\"raw_text\":\"...\"" fragment: the model's pre-cleanup decode -// (transcribe_raw_text). Emitted only when it differs from the clean text, -// so JSONL stays compact for families with no post-processing. -std::string raw_text_json(const char * raw, const char * clean) { - if (raw == nullptr || raw[0] == '\0' || (clean != nullptr && std::strcmp(raw, clean) == 0)) { - return {}; - } - std::string out = ",\"raw_text\":\""; - out += json_escape(raw); - out += "\""; - return out; -} - -// A decode cut short before end-of-stream (budget or repetition stop): non-OK, -// but the partial transcript is preserved (see docs/input-limits.md). -bool is_cut_short(transcribe_status st) { - return st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED || st == TRANSCRIBE_ERR_OUTPUT_REPETITION; -} - -const char * cut_short_label(transcribe_status st) { - return st == TRANSCRIBE_ERR_OUTPUT_REPETITION ? "stopped repeating" : "truncated"; -} - -// ",\"speakers\":[...]" fragment: the "who spoke when" rows. Emitted only -// when the run produced speaker segments. p is omitted unless finite -// (NaN — "model provides no confidence" — is not representable in JSON). -std::string speaker_row_json(const struct transcribe_speaker_segment & row) { - char buf[160]; - if (std::isfinite(row.p)) { - std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d,\"p\":%.4f}", - static_cast(row.t0_ms), static_cast(row.t1_ms), - static_cast(row.speaker_id), static_cast(row.p)); - } else { - std::snprintf(buf, sizeof(buf), "{\"t0_ms\":%lld,\"t1_ms\":%lld,\"speaker_id\":%d}", - static_cast(row.t0_ms), static_cast(row.t1_ms), - static_cast(row.speaker_id)); - } - return buf; -} - -std::string speakers_json(const transcribe_session * ctx) { - const int n = transcribe_n_speaker_segments(ctx); - if (n <= 0) { - return {}; - } - std::string out = ",\"speakers\":["; - for (int s = 0; s < n; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - (void) transcribe_get_speaker_segment(ctx, s, &row); - out += speaker_row_json(row); - } - out += "]"; - return out; -} - -std::string batch_speakers_json(const transcribe_session * ctx, int i) { - const int n = transcribe_batch_n_speaker_segments(ctx, i); - if (n <= 0) { - return {}; - } - std::string out = ",\"speakers\":["; - for (int s = 0; s < n; ++s) { - if (s > 0) { - out += ","; - } - struct transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - (void) transcribe_batch_get_speaker_segment(ctx, i, s, &row); - out += speaker_row_json(row); - } - out += "]"; - return out; -} - -struct cli_args { - std::string wav_path; - std::string model_path; - std::string language; - std::string target_language; // --target-language: target lang for translation - std::string batch_file; // --batch: one wav path per line - int batch_size = 0; // --batch-size: >1 groups utterances into - // transcribe_run_batch calls (offline only). - // 0/1 keeps the per-file serial loop. - bool translate = false; - bool instruct = false; // --task instruct - bool quiet = false; - bool list_devices = false; // --list-devices: print devices and exit - bool batch_jsonl = false; // --batch-jsonl: output JSONL - std::string output_path; // -o/--output: write raw text here - int repeat = 1; - int n_threads = 0; // 0 = library default (all cores) - int n_ctx = 0; // 0 = model's true max; >0 lowers the cap - transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; - transcribe_backend_request backend = TRANSCRIBE_BACKEND_AUTO; - int device_index = -1; // --device N: -1 = auto, >=0 = exact registry device - transcribe_timestamp_kind timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; - - // Generic prompting (transcribe_run_params::vocabulary / prompt / prefix). - std::vector vocabulary; // --vocabulary TERMS / --vocabulary-file PATH - std::string prompt; // --prompt TEXT - std::string prefix; // --prefix TEXT - - // Whisper-family knobs. Ignored for non-Whisper models. - std::string initial_prompt; // --initial-prompt TEXT - bool whisper_set = false; - bool temperature_set = false; // --temperature F - float temperature = 0.0f; // tier-0 sampling temp - bool condition_on_prev_tokens = false; // --condition-on-prev-tokens - enum transcribe_whisper_prompt_condition prompt_condition = - TRANSCRIBE_WHISPER_PROMPT_FIRST_SEGMENT; // --prompt-condition first|all - - // SenseVoice / FunASR-Nano family knobs. The `--itn` flag is shared: - // it routes to whichever family the loaded model belongs to. Ignored - // by non-ITN-aware families. Unset leaves the library default in place, - // which differs per family (sensevoice: on; funasr-nano: off), so the - // initializer here is only read once --itn / --no-itn has been seen. - bool use_itn = false; // --itn / --no-itn - bool itn_set = false; - bool keep_special_tags = false; // --raw-tokens - - // Canary family knobs. Ignored by non-Canary families. - bool canary_pnc = true; // default: punctuation+caps on - bool canary_pnc_set = false; // --pnc / --no-pnc set this - - // Speaker diarization toggle (moss / granite-plus). Unset = library - // default (OFF). --diarize / --no-diarize set this. - bool diarize = true; - bool diarize_set = false; - - // Streaming demo: when > 0, the single-file path feeds the WAV - // through transcribe_stream_begin/feed/finalize in fixed-size - // ms-aligned chunks instead of one transcribe_run call. Requires - // the loaded model to advertise supports_streaming. Set by - // --stream-chunk-ms N. - int stream_chunk_ms = 0; - // Parakeet streaming: pick a right-context (lookahead) setting - // from the model's training menu. -1 = model default (max - // accuracy / max latency); 0/1/6/13 select the published - // nemotron-speech-streaming-en-0.6b settings. Set by - // --stream-att-right N. Ignored when stream_chunk_ms == 0. - int stream_att_right = -1; - // Parakeet buffered streaming (parakeet-unified-en-0.6b): override - // the (L, C, R) attention context tuple in milliseconds. -1 = use - // the model's default (highest-accuracy row of the training menu). - // Frame-aligned: the lib rounds each value down to the nearest - // post-subsample frame (80ms at 4x subsampling). Ignored when - // stream_chunk_ms == 0 or when the model is not buffered-streaming. - int stream_buf_left_ms = -1; - int stream_buf_chunk_ms = -1; - int stream_buf_right_ms = -1; - // Voxtral Realtime streaming: transcription delay in 12.5 Hz audio - // tokens (80 ms each). -1 = model default (6 = 480 ms). Set by - // --stream-voxtral-delay N. Ignored when stream_chunk_ms == 0 or when - // the model is not voxtral_realtime. - int stream_voxtral_delay = -1; - // Speculative-decode draft length passed through to - // transcribe_run_params::spec_k_drafts on the offline path. -1 = family - // default (each family picks its tuned K). 0 = explicitly off. >0 = - // explicit K. Silently ignored by families without - // supports_spec_decode. Set by --spec-k-drafts N. - int spec_k_drafts = -1; -}; - -// Point rp's generic prompting fields at args' storage; vocabulary_ptrs -// backs rp.vocabulary and must outlive the run. -void apply_prompting(const cli_args & args, transcribe_run_params & rp, std::vector & vocabulary_ptrs) { - if (args.instruct) { - rp.task = TRANSCRIBE_TASK_INSTRUCT; - } - vocabulary_ptrs.clear(); - for (const std::string & term : args.vocabulary) { - vocabulary_ptrs.push_back(term.c_str()); - } - rp.vocabulary = vocabulary_ptrs.empty() ? nullptr : vocabulary_ptrs.data(); - rp.n_vocabulary = static_cast(vocabulary_ptrs.size()); - rp.prompt = args.prompt.empty() ? nullptr : args.prompt.c_str(); - rp.prefix = args.prefix.empty() ? nullptr : args.prefix.c_str(); -} - void print_usage(const char * argv0) { std::fprintf(stderr, "usage: %s [options] audio.wav\n" @@ -341,7 +54,7 @@ void print_usage(const char * argv0) { " --target-language ISO target language for translation (e.g. de, es, fr)\n" " -q, --quiet suppress library log output\n" " -r, --repeat N run N times per file (benchmark)\n" - " -o, --output PATH write transcribed text to PATH (stdout unchanged)\n" + " -o, --output PATH write text or speaker segments to PATH (stdout unchanged)\n" " --threads N CPU threads (default: all cores)\n" " --n-ctx N session context/KV cap in tokens (bounds decoder\n" " KV memory; cannot extend the model): 0 = model\n" @@ -827,21 +540,53 @@ void log_cb(transcribe_log_level level, const char * msg, void * userdata) { std::fprintf(stderr, "%s %s%s", prefix, msg, (msg && *msg && msg[std::strlen(msg) - 1] == '\n') ? "" : "\n"); } -bool write_output_file(std::ofstream * output, const std::string & path, const char * text) { - if (output == nullptr) { - return true; +// One file: load the audio and (with -m) the model, print both, then hand the +// model to the driver for its role. The driver takes ownership of the model. +int run_file(const cli_args & args, std::ofstream * output) { + std::vector pcm; + std::string load_err; + if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { + std::fprintf(stderr, "wav: %s\n", load_err.c_str()); + return EXIT_FAILURE; } - const char * value = text != nullptr ? text : ""; - *output << value; - if (value[0] == '\0' || value[std::strlen(value) - 1] != '\n') { - *output << '\n'; + + const double duration_s = static_cast(pcm.size()) / 16000.0; + std::printf("audio: %s\n", args.wav_path.c_str()); + std::printf(" samples: %zu\n", pcm.size()); + std::printf(" duration: %.3f s\n", duration_s); + std::printf(" sample rate 16000 Hz mono float32\n"); + + if (args.model_path.empty()) { + std::printf("model: (none specified, skipping load)\n"); + return EXIT_SUCCESS; } - output->flush(); - if (!*output) { - std::fprintf(stderr, "error: cannot write %s\n", path.c_str()); - return false; + + struct transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = args.backend; + mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; + if (args.device_index >= 0 && mp.device == nullptr) { + std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); + return EXIT_FAILURE; } - return true; + struct transcribe_model * model = nullptr; + const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); + std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); + if (st != TRANSCRIBE_OK) { + return EXIT_FAILURE; + } + std::printf(" backend: %s\n", transcribe_model_backend(model)); + if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { + std::printf(" name: %s\n", dn); + } + if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { + std::printf(" license: %s\n", lic); + } + + if ((transcribe_model_roles(model) & TRANSCRIBE_ROLE_ASR) != 0) { + return transcribe_cli::run_asr_file(args, model, pcm, duration_s, output); + } + return transcribe_cli::run_diarize_file(args, model, pcm, duration_s, output); } } // namespace @@ -879,755 +624,10 @@ int main(int argc, char ** argv) { } output = &output_file; } - bool output_ok = true; - // Batch mode: --batch reads a file list, one wav path per line. Loads - // the model ONCE and reuses the context across all files. Outputs one - // JSONL line per file to stdout when --batch-jsonl is set, otherwise - // the same human-readable format as single-file mode. if (!args.batch_file.empty()) { - if (args.model_path.empty()) { - std::fprintf(stderr, "error: --batch requires --model\n"); - return EXIT_FAILURE; - } - - std::vector wav_paths; - { - std::ifstream fin(args.batch_file); - if (!fin) { - std::fprintf(stderr, "error: cannot open batch file %s\n", args.batch_file.c_str()); - return EXIT_FAILURE; - } - std::string line; - while (std::getline(fin, line)) { - while (!line.empty() && - (line.back() == '\n' || line.back() == '\r' || line.back() == ' ' || line.back() == '\t')) { - line.pop_back(); - } - if (!line.empty()) { - wav_paths.push_back(line); - } - } - } - if (wav_paths.empty()) { - std::fprintf(stderr, "error: batch file is empty\n"); - return EXIT_FAILURE; - } - if (!args.batch_jsonl) { - std::fprintf(stderr, "batch: %zu files\n", wav_paths.size()); - } - - struct transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; - if (args.device_index >= 0 && mp.device == nullptr) { - std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); - return EXIT_FAILURE; - } - struct transcribe_model * model = nullptr; - const transcribe_status load_st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); - if (load_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "model load: %s\n", transcribe_status_string(load_st)); - return EXIT_FAILURE; - } - - struct transcribe_session_params cp; - transcribe_session_params_init(&cp); - cp.n_threads = args.n_threads; - cp.n_ctx = args.n_ctx; - cp.kv_type = args.kv_type; - struct transcribe_session * ctx = nullptr; - const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); - if (init_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - struct transcribe_run_params rp; - transcribe_run_params_init(&rp); - if (args.translate) { - rp.task = TRANSCRIBE_TASK_TRANSLATE; - } - std::vector vocabulary_ptrs; - apply_prompting(args, rp, vocabulary_ptrs); - if (!args.language.empty()) { - rp.language = args.language.c_str(); - } - if (!args.target_language.empty()) { - rp.target_language = args.target_language.c_str(); - } - rp.timestamps = args.timestamps; - rp.spec_k_drafts = args.spec_k_drafts; - - if (args.itn_set) { - rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; - } - if (args.canary_pnc_set) { - rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; - } - if (args.diarize_set) { - rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; - } - - // Whisper run extension. Allocated outside rp's scope so its - // bytes outlive the per-file loop below; the library copies - // initial_prompt/prompt_tokens before transcribe_run returns, - // but rp aliases &wx.ext for the run call itself. - struct transcribe_whisper_run_ext wx; - transcribe_whisper_run_ext_init(&wx); - if (args.whisper_set) { - if (!args.initial_prompt.empty()) { - wx.initial_prompt = args.initial_prompt.c_str(); - } - wx.condition_on_prev_tokens = args.condition_on_prev_tokens; - if (args.temperature_set) { - wx.temperature = args.temperature; - } - wx.prompt_condition = args.prompt_condition; - if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { - rp.family = &wx.ext; - } - } - - if (args.keep_special_tags) { - rp.keep_special_tags = true; - } - - // Emit a batch header line once, before any per-file output. Carries - // the one-shot load time so downstream WER tooling can record it - // without parsing stderr. Per-file lines follow on subsequent lines. - if (args.batch_jsonl) { - struct transcribe_timings load_tm; - transcribe_timings_init(&load_tm); - (void) transcribe_get_timings(ctx, &load_tm); - std::printf("{\"type\":\"batch_header\",\"load_ms\":%.1f}\n", (double) load_tm.load_ms); - std::fflush(stdout); - } - - int n_ok = 0; - int n_truncated = 0; // result-bearing: hit the generation cap, partial hyp emitted - int n_repeating = 0; // result-bearing: stopped when the output looped, partial hyp emitted - int n_fail = 0; // no usable result (wav load / backend / unsupported / whole-batch) - - // Offline batched path: group up to batch_size utterances into one - // transcribe_run_batch call. Mutually exclusive with the streaming - // path (--stream-chunk-ms), which is per-utterance by construction. - // Per-file JSONL output is byte-identical in shape to the serial - // path so the WER harness (scripts/wer/run.py) consumes either. - const bool use_batched = args.batch_size > 1 && args.stream_chunk_ms <= 0; - if (use_batched) { - const size_t total = wav_paths.size(); - const size_t group = static_cast(args.batch_size); - for (size_t base = 0; base < total; base += group) { - const size_t end = std::min(total, base + group); - - // Load each wav in the group. A wav that fails to load is - // emitted as an error line and excluded from the batch, so - // its failure cannot abort its neighbours. - std::vector> pcms; - std::vector pcm_ptrs; - std::vector n_samps; - std::vector src_index; // -> wav_paths - for (size_t i = base; i < end; ++i) { - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav_paths[i], pcm, wav_err)) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav_paths[i].c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"wav: %s\"}\n", - file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); - } else { - std::fprintf(stderr, "SKIP %s: %s\n", wav_paths[i].c_str(), wav_err.c_str()); - } - ++n_fail; - std::fflush(stdout); - continue; - } - pcms.push_back(std::move(pcm)); - src_index.push_back(i); - } - for (auto & p : pcms) { - pcm_ptrs.push_back(p.data()); - n_samps.push_back(static_cast(p.size())); - } - if (pcms.empty()) { - continue; - } - - const transcribe_status bst = - transcribe_run_batch(ctx, pcm_ptrs.data(), n_samps.data(), static_cast(pcm_ptrs.size()), &rp); - if (bst != TRANSCRIBE_OK && transcribe_batch_n_results(ctx) == 0) { - // Whole-batch failure (no per-utterance results): emit an - // error line per file in the group and continue. - for (size_t k = 0; k < src_index.size(); ++k) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav_paths[src_index[k]].c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"%s\"}\n", - file_esc.c_str(), json_escape(transcribe_status_string(bst)).c_str()); - } - ++n_fail; - } - std::fflush(stdout); - continue; - } - - for (size_t k = 0; k < src_index.size(); ++k) { - const std::string & wav = wav_paths[src_index[k]]; - const transcribe_status ust = transcribe_batch_status(ctx, static_cast(k)); - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the - // partial transcript is preserved and readable via - // transcribe_batch_full_text (see transcribe.h). Emit it as the hyp - // so downstream tooling scores the partial rather than an empty - // string; the error field below still tags it so the stop stays - // visible. - const bool result_present = ust == TRANSCRIBE_OK || is_cut_short(ust); - const char * text = ""; - if (result_present) { - const char * t = transcribe_batch_full_text(ctx, static_cast(k)); - if (t && *t) { - text = t; - } - } - if (ust == TRANSCRIBE_OK) { - ++n_ok; - } else if (ust == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { - ++n_truncated; - } else if (ust == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - ++n_repeating; - } else { - ++n_fail; - } - if (args.batch_jsonl) { - const std::string escaped = json_escape(text); - std::string segments = batch_segments_json(ctx, static_cast(k)); - segments += batch_speakers_json(ctx, static_cast(k)); - segments += raw_text_json(transcribe_batch_raw_text(ctx, static_cast(k)), text); - std::string err_field; - if (ust != TRANSCRIBE_OK) { - err_field = ",\"error\":\""; - err_field += json_escape(transcribe_status_string(ust)); - err_field += "\""; - } - // Per-utterance timings: the batched encoder time is - // amortized across the batch, decode is real per-utt, - // so summing across utterances gives the true encode vs - // decode split for the whole batched run. - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_batch_get_timings(ctx, static_cast(k), &tm); - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"%s\"%s," - "\"mel_ms\":%.1f,\"encode_ms\":%.1f," - "\"decode_ms\":%.1f%s}\n", - file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, - (double) tm.encode_ms, (double) tm.decode_ms, err_field.c_str()); - } else { - std::printf("[%zu/%zu] %s", src_index[k] + 1, total, wav.c_str()); - if (ust == TRANSCRIBE_OK) { - std::printf("\n text: %s\n", text); - } else if (is_cut_short(ust)) { - std::printf(" (%s)\n text: %s\n", cut_short_label(ust), text); - } else { - std::printf(" ERROR: %s\n", transcribe_status_string(ust)); - } - } - output_ok = write_output_file(output, args.output_path, text) && output_ok; - std::fflush(stdout); - } - } - } else { - for (size_t i = 0; i < wav_paths.size(); ++i) { - const std::string & wav = wav_paths[i]; - - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav, pcm, wav_err)) { - if (args.batch_jsonl) { - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"\"," - "\"error\":\"wav: %s\"}\n", - file_esc.c_str(), json_escape(wav_err.c_str()).c_str()); - } else { - std::fprintf(stderr, "SKIP %s: %s\n", wav.c_str(), wav_err.c_str()); - } - ++n_fail; - std::fflush(stdout); - continue; - } - - // Run. When --stream-chunk-ms > 0, drive the streaming API - // for this utterance (begin/feed/finalize) so the WER - // harness can measure cache-aware streaming output. - transcribe_status run_st = TRANSCRIBE_OK; - if (args.stream_chunk_ms > 0) { - struct transcribe_stream_params sp; - transcribe_stream_params_init(&sp); - struct transcribe_parakeet_stream_ext pkt_sp; - transcribe_parakeet_stream_ext_init(&pkt_sp); - struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; - transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); - struct transcribe_voxtral_realtime_stream_ext vx_sp; - transcribe_voxtral_realtime_stream_ext_init(&vx_sp); - const bool want_cache_aware = (args.stream_att_right >= 0); - const bool want_buffered = - args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; - if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { - pkt_sp.att_context_right = args.stream_att_right; - sp.family = &pkt_sp.ext; - } else if (want_buffered && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { - pkt_buf_sp.left_ms = args.stream_buf_left_ms; - pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; - pkt_buf_sp.right_ms = args.stream_buf_right_ms; - sp.family = &pkt_buf_sp.ext; - } else if (args.stream_voxtral_delay != -1 && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { - vx_sp.num_delay_tokens = args.stream_voxtral_delay; - sp.family = &vx_sp.ext; - } - run_st = transcribe_stream_begin(ctx, &rp, &sp); - if (run_st == TRANSCRIBE_OK) { - const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); - size_t pos = 0; - while (pos < pcm.size()) { - const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); - struct transcribe_stream_update upd; - transcribe_stream_update_init(&upd); - run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); - if (run_st != TRANSCRIBE_OK) { - break; - } - pos += take; - } - if (run_st == TRANSCRIBE_OK) { - struct transcribe_stream_update fin_upd; - transcribe_stream_update_init(&fin_upd); - run_st = transcribe_stream_finalize(ctx, &fin_upd); - } - } - } else { - run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); - } - - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing: the partial - // transcript is preserved and readable via transcribe_full_text (see - // transcribe.h). Emit it as the hyp so downstream tooling scores the - // partial rather than an empty string; the error field below still - // tags it so the stop stays visible. - const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st); - const char * text = ""; - if (result_present) { - const char * t = transcribe_full_text(ctx); - if (t && *t) { - text = t; - } - } - if (run_st == TRANSCRIBE_OK) { - ++n_ok; - } else if (run_st == TRANSCRIBE_ERR_OUTPUT_TRUNCATED) { - ++n_truncated; - } else if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - ++n_repeating; - } else { - ++n_fail; - } - - // Output. When run_st != OK we still emit a JSON line so - // batch consumers see one record per input wav, but we tag - // it with the error string. Without this the wav-load error - // path (above) reports errors but transcribe_run failures - // (e.g. unsupported language) produce empty hyp_text with no - // error tag, so downstream tools count them as silent - // successes. - if (args.batch_jsonl) { - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_get_timings(ctx, &tm); - const std::string escaped = json_escape(text); - std::string segments = segments_json(ctx); - segments += speakers_json(ctx); - segments += raw_text_json(transcribe_raw_text(ctx), text); - std::string err_field; - if (run_st != TRANSCRIBE_OK) { - err_field = ",\"error\":\""; - err_field += json_escape(transcribe_status_string(run_st)); - err_field += "\""; - } - const std::string file_esc = json_escape(wav.c_str()); - std::printf( - "{\"file\":\"%s\",\"text\":\"%s\"%s," - "\"mel_ms\":%.1f,\"encode_ms\":%.1f," - "\"decode_ms\":%.1f%s}\n", - file_esc.c_str(), escaped.c_str(), segments.c_str(), (double) tm.mel_ms, (double) tm.encode_ms, - (double) tm.decode_ms, err_field.c_str()); - } else { - std::printf("[%zu/%zu] %s", i + 1, wav_paths.size(), wav.c_str()); - if (run_st == TRANSCRIBE_OK) { - std::printf("\n text: %s\n", text); - } else if (is_cut_short(run_st)) { - std::printf(" (%s)\n text: %s\n", cut_short_label(run_st), text); - } else { - std::printf(" ERROR: %s\n", transcribe_status_string(run_st)); - } - } - output_ok = write_output_file(output, args.output_path, text) && output_ok; - std::fflush(stdout); - } - } - - if (!args.batch_jsonl) { - std::fprintf(stderr, "batch: %d ok, %d truncated, %d stopped repeating, %d failed out of %zu\n", n_ok, - n_truncated, n_repeating, n_fail, wav_paths.size()); - } - - transcribe_session_free(ctx); - transcribe_model_free(model); - // OUTPUT_TRUNCATED / OUTPUT_REPETITION are result-bearing and do not fail the batch, but - // hard per-utterance failures must remain visible to automation. - return n_fail > 0 || !output_ok ? EXIT_FAILURE : EXIT_SUCCESS; - } - - // Single-file mode. - std::vector pcm; - std::string load_err; - if (!transcribe_cli::load_wav_mono_16k(args.wav_path, pcm, load_err)) { - std::fprintf(stderr, "wav: %s\n", load_err.c_str()); - return EXIT_FAILURE; + return transcribe_cli::run_asr_batch(args, output); } - const double duration_s = static_cast(pcm.size()) / 16000.0; - std::printf("audio: %s\n", args.wav_path.c_str()); - std::printf(" samples: %zu\n", pcm.size()); - std::printf(" duration: %.3f s\n", duration_s); - std::printf(" sample rate 16000 Hz mono float32\n"); - - if (!args.model_path.empty()) { - struct transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = args.backend; - mp.device = args.device_index >= 0 ? transcribe_device_get(args.device_index) : nullptr; - if (args.device_index >= 0 && mp.device == nullptr) { - std::fprintf(stderr, "error: --device index %d is not available\n", args.device_index); - return EXIT_FAILURE; - } - struct transcribe_model * model = nullptr; - const transcribe_status st = transcribe_model_load_file(args.model_path.c_str(), &mp, &model); - std::printf("model: %s -> %s\n", args.model_path.c_str(), transcribe_status_string(st)); - if (st != TRANSCRIBE_OK) { - return EXIT_FAILURE; - } - std::printf(" backend: %s\n", transcribe_model_backend(model)); - if (const char * dn = transcribe_model_meta_val_str(model, "general.name"); dn[0]) { - std::printf(" name: %s\n", dn); - } - if (const char * lic = transcribe_model_meta_val_str(model, "general.license"); lic[0]) { - std::printf(" license: %s\n", lic); - } - - struct transcribe_session_params cp; - transcribe_session_params_init(&cp); - cp.n_threads = args.n_threads; - cp.n_ctx = args.n_ctx; - cp.kv_type = args.kv_type; - struct transcribe_session * ctx = nullptr; - const transcribe_status init_st = transcribe_session_init(model, &cp, &ctx); - if (init_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "context init: %s\n", transcribe_status_string(init_st)); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - // Surface the effective input-length limit so it's obvious how much - // audio this session accepts (reflects --n-ctx). 0 means "no practical - // limit", which covers two different families: one that chunks long - // audio internally (FEATURE_LONG_FORM) and one that is genuinely - // unbounded and encodes the clip in a single pass. Distinguish them - // rather than asserting the first. See docs/input-limits.md. - { - struct transcribe_session_limits lim; - transcribe_session_limits_init(&lim); - if (transcribe_session_get_limits(ctx, &lim) == TRANSCRIBE_OK) { - if (lim.effective_max_audio_ms > 0) { - std::printf(" max audio: %.1f s", (double) lim.effective_max_audio_ms / 1000.0); - if (lim.effective_n_ctx > 0) { - std::printf(" (context %d tok, ~%lld MiB KV max)", lim.effective_n_ctx, - (long long) (lim.max_kv_bytes >> 20)); - } - std::printf("\n"); - } else if (lim.effective_n_ctx > 0) { - // Capped family whose context is too small to fit any audio - // plus a prompt (e.g. an aggressively low --n-ctx). - std::printf( - " max audio: ~0 s (context %d tok too small for " - "audio + prompt)\n", - lim.effective_n_ctx); - } else if (transcribe_model_supports(model, TRANSCRIBE_FEATURE_LONG_FORM)) { - std::printf(" max audio: unbounded (long audio chunked internally)\n"); - } else { - // No context cap and no chunker: the family encodes the - // whole clip in one pass (e.g. block-local attention, - // where cost is linear in audio length). - std::printf(" max audio: unbounded (whole clip in one pass)\n"); - } - } - } - - struct transcribe_run_params rp; - transcribe_run_params_init(&rp); - if (args.translate) { - rp.task = TRANSCRIBE_TASK_TRANSLATE; - } - std::vector vocabulary_ptrs; - apply_prompting(args, rp, vocabulary_ptrs); - if (!args.language.empty()) { - rp.language = args.language.c_str(); - } - if (!args.target_language.empty()) { - rp.target_language = args.target_language.c_str(); - } - rp.timestamps = args.timestamps; - rp.spec_k_drafts = args.spec_k_drafts; - - if (args.itn_set) { - rp.itn = args.use_itn ? TRANSCRIBE_ITN_MODE_ON : TRANSCRIBE_ITN_MODE_OFF; - } - if (args.canary_pnc_set) { - rp.pnc = args.canary_pnc ? TRANSCRIBE_PNC_MODE_ON : TRANSCRIBE_PNC_MODE_OFF; - } - if (args.diarize_set) { - rp.diarize = args.diarize ? TRANSCRIBE_DIARIZE_MODE_ON : TRANSCRIBE_DIARIZE_MODE_OFF; - } - - struct transcribe_whisper_run_ext wx; - transcribe_whisper_run_ext_init(&wx); - if (args.whisper_set) { - if (!args.initial_prompt.empty()) { - wx.initial_prompt = args.initial_prompt.c_str(); - } - wx.condition_on_prev_tokens = args.condition_on_prev_tokens; - if (args.temperature_set) { - wx.temperature = args.temperature; - } - wx.prompt_condition = args.prompt_condition; - if (transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN)) { - rp.family = &wx.ext; - } - } - - if (args.keep_special_tags) { - rp.keep_special_tags = true; - } - - // Streaming demo: drive transcribe_stream_begin/feed/finalize - // with fixed-size PCM chunks. Families with true per-feed - // partial decoding (moonshine_streaming) flip - // update.result_changed whenever the transcript advances; the - // CLI prints the live tentative text on each such feed. - // Families that only commit at finalize keep result_changed - // false until the finalize call. - transcribe_status run_st = TRANSCRIBE_OK; - if (args.stream_chunk_ms > 0) { - struct transcribe_capabilities caps; - transcribe_capabilities_init(&caps); - const transcribe_status caps_st = transcribe_model_get_capabilities(model, &caps); - if (caps_st != TRANSCRIBE_OK || !caps.supports_streaming) { - std::fprintf(stderr, - "stream: model does not advertise " - "supports_streaming; use a streaming-capable " - "model or drop --stream-chunk-ms\n"); - transcribe_session_free(ctx); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - const int chunk_samples = std::max(1, args.stream_chunk_ms * 16000 / 1000); - std::printf("stream: chunk=%d ms (%d samples)\n", args.stream_chunk_ms, chunk_samples); - - struct transcribe_stream_params sp; - transcribe_stream_params_init(&sp); - struct transcribe_parakeet_stream_ext pkt_sp; - transcribe_parakeet_stream_ext_init(&pkt_sp); - struct transcribe_parakeet_buffered_stream_ext pkt_buf_sp; - transcribe_parakeet_buffered_stream_ext_init(&pkt_buf_sp); - struct transcribe_voxtral_realtime_stream_ext vx_sp; - transcribe_voxtral_realtime_stream_ext_init(&vx_sp); - const bool want_cache_aware = (args.stream_att_right >= 0); - const bool want_buffered = - args.stream_buf_left_ms >= 0 || args.stream_buf_chunk_ms >= 0 || args.stream_buf_right_ms >= 0; - if (want_cache_aware && transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_STREAM)) { - pkt_sp.att_context_right = args.stream_att_right; - sp.family = &pkt_sp.ext; - std::printf("stream: att_context_right=%d\n", args.stream_att_right); - } else if (want_buffered && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_PARAKEET_BUFFERED_STREAM)) { - pkt_buf_sp.left_ms = args.stream_buf_left_ms; - pkt_buf_sp.chunk_ms = args.stream_buf_chunk_ms; - pkt_buf_sp.right_ms = args.stream_buf_right_ms; - sp.family = &pkt_buf_sp.ext; - std::printf("stream: buffered (L,C,R)_ms=(%d,%d,%d)\n", args.stream_buf_left_ms, - args.stream_buf_chunk_ms, args.stream_buf_right_ms); - } else if (args.stream_voxtral_delay != -1 && - transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, - TRANSCRIBE_EXT_KIND_VOXTRAL_REALTIME_STREAM)) { - vx_sp.num_delay_tokens = args.stream_voxtral_delay; - sp.family = &vx_sp.ext; - std::printf("stream: voxtral num_delay_tokens=%d\n", args.stream_voxtral_delay); - } - run_st = transcribe_stream_begin(ctx, &rp, &sp); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_begin: %s\n", transcribe_status_string(run_st)); - } else { - size_t pos = 0; - int feed_n = 0; - while (pos < pcm.size()) { - const size_t take = std::min(static_cast(chunk_samples), pcm.size() - pos); - struct transcribe_stream_update upd; - transcribe_stream_update_init(&upd); - run_st = transcribe_stream_feed(ctx, pcm.data() + pos, static_cast(take), &upd); - if (run_st != TRANSCRIBE_OK) { - std::fprintf(stderr, "stream_feed[%d]: %s\n", feed_n, transcribe_status_string(run_st)); - break; - } - pos += take; - std::printf(" feed[%2d]: input=%lld ms buffered=%lld ms", feed_n, - (long long) upd.input_received_ms, (long long) upd.buffered_ms); - if (upd.result_changed) { - const char * partial = transcribe_full_text(ctx); - std::printf(" partial=\"%s\"", (partial && *partial) ? partial : ""); - } - std::printf("\n"); - ++feed_n; - } - if (run_st == TRANSCRIBE_OK) { - struct transcribe_stream_update fin_upd; - transcribe_stream_update_init(&fin_upd); - run_st = transcribe_stream_finalize(ctx, &fin_upd); - std::printf( - " finalize: status=%s " - "revision=%d input=%lld ms committed=%lld ms\n", - transcribe_status_string(run_st), fin_upd.revision, (long long) fin_upd.input_received_ms, - (long long) fin_upd.audio_committed_ms); - } - } - } else { - // --repeat N runs transcribe_run() N times for steady-state - // perf measurements. - for (int r = 0; r < args.repeat; ++r) { - run_st = transcribe_run(ctx, pcm.data(), static_cast(pcm.size()), &rp); - if (run_st != TRANSCRIBE_OK) { - break; - } - } - } - std::printf("run: %s\n", transcribe_status_string(run_st)); - // OUTPUT_TRUNCATED, OUTPUT_REPETITION and ABORTED are non-OK but - // preserve the partial transcript (see docs/input-limits.md), so show - // the result for them too — just flagged. - const bool result_present = run_st == TRANSCRIBE_OK || is_cut_short(run_st) || run_st == TRANSCRIBE_ERR_ABORTED; - if (result_present) { - const char * text = transcribe_full_text(ctx); - std::printf("text: %s\n", (text && *text) ? text : "(empty)"); - output_ok = write_output_file(output, args.output_path, text) && output_ok; - - // A decode cut short before end-of-stream; the text above is - // incomplete. - if (run_st == TRANSCRIBE_ERR_OUTPUT_REPETITION) { - std::printf( - " note: decode stopped when the output began repeating " - "itself (repeats dropped); transcript is incomplete\n"); - } else if (transcribe_was_truncated(ctx)) { - std::printf( - " note: output truncated (hit the model's " - "context/generation cap before end-of-stream); " - "transcript is incomplete\n"); - } - - const char * dl = transcribe_detected_language(ctx); - if (dl && *dl) { - std::printf("detected-language: %s\n", dl); - } - - const transcribe_timestamp_kind ret_kind = transcribe_returned_timestamp_kind(ctx); - const int n_seg = transcribe_n_segments(ctx); - const bool has_spk = transcribe_n_speaker_segments(ctx) > 0; - if (n_seg > 0 && (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE || has_spk)) { - std::printf("segments: %d\n", n_seg); - for (int i = 0; i < n_seg; ++i) { - struct transcribe_segment seg; - transcribe_segment_init(&seg); - (void) transcribe_get_segment(ctx, i, &seg); - char spk[16] = ""; - if (seg.speaker_id > 0) { - std::snprintf(spk, sizeof(spk), "S%d: ", static_cast(seg.speaker_id)); - } - if (ret_kind != TRANSCRIBE_TIMESTAMPS_NONE) { - std::printf(" [%7.2f -> %7.2f] %s%s\n", seg.t0_ms / 1000.0, seg.t1_ms / 1000.0, spk, - (seg.text != nullptr) ? seg.text : ""); - } else { - // Speaker-attributed turns without timing (granite SAA). - std::printf(" %s%s\n", spk, (seg.text != nullptr) ? seg.text : ""); - } - } - } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_WORD || ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_wrd = transcribe_n_words(ctx); - std::printf("words: %d\n", n_wrd); - for (int i = 0; i < n_wrd; ++i) { - struct transcribe_word wrd; - transcribe_word_init(&wrd); - (void) transcribe_get_word(ctx, i, &wrd); - std::printf(" [%7.2f -> %7.2f] %s\n", wrd.t0_ms / 1000.0, wrd.t1_ms / 1000.0, - (wrd.text != nullptr) ? wrd.text : ""); - } - } - if (ret_kind == TRANSCRIBE_TIMESTAMPS_TOKEN) { - const int n_tok = transcribe_n_tokens(ctx); - std::printf("tokens: %d\n", n_tok); - for (int i = 0; i < n_tok; ++i) { - struct transcribe_token tok; - transcribe_token_init(&tok); - (void) transcribe_get_token(ctx, i, &tok); - std::printf(" [%7.2f -> %7.2f] p=%.3f %s\n", tok.t0_ms / 1000.0, tok.t1_ms / 1000.0, tok.p, - (tok.text != nullptr) ? tok.text : ""); - } - } - } - - transcribe_print_timings(ctx); - - { - struct transcribe_timings tm; - transcribe_timings_init(&tm); - (void) transcribe_get_timings(ctx, &tm); - const double total_ms = tm.mel_ms + tm.encode_ms + tm.decode_ms; - if (total_ms > 0.0 && duration_s > 0.0) { - std::printf(" realtime: %.0fx (%.1f ms for %.1f s)\n", (duration_s * 1000.0) / total_ms, total_ms, - duration_s); - } - } - - transcribe_session_free(ctx); - transcribe_model_free(model); - - if (run_st != TRANSCRIBE_OK || !output_ok) { - return EXIT_FAILURE; - } - } else { - std::printf("model: (none specified, skipping load)\n"); - } - - return EXIT_SUCCESS; + return run_file(args, output); } diff --git a/include/transcribe.abihash b/include/transcribe.abihash index 4d382de59..8fce33b22 100644 --- a/include/transcribe.abihash +++ b/include/transcribe.abihash @@ -1 +1 @@ -57b1af43650f195d +bd3273dabb25a1fe diff --git a/include/transcribe.h b/include/transcribe.h index 7c52267fd..c36df7b75 100644 --- a/include/transcribe.h +++ b/include/transcribe.h @@ -313,6 +313,8 @@ typedef enum { * the check. See docs/input-limits.md. */ TRANSCRIBE_ERR_OUTPUT_REPETITION = 19, + /* The model does not serve the role this call needs; see transcribe_model_roles(). */ + TRANSCRIBE_ERR_UNSUPPORTED_ROLE = 20, } transcribe_status; /* @@ -369,22 +371,26 @@ TRANSCRIBE_API const char * transcribe_version_commit(void); * is append-only; do not renumber existing values. */ typedef enum { - TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0, - TRANSCRIBE_ABI_SESSION_PARAMS = 1, - TRANSCRIBE_ABI_RUN_PARAMS = 2, - TRANSCRIBE_ABI_STREAM_PARAMS = 3, - TRANSCRIBE_ABI_CAPABILITIES = 4, - TRANSCRIBE_ABI_TIMINGS = 5, - TRANSCRIBE_ABI_SEGMENT = 6, - TRANSCRIBE_ABI_WORD = 7, - TRANSCRIBE_ABI_TOKEN = 8, - TRANSCRIBE_ABI_STREAM_UPDATE = 9, - TRANSCRIBE_ABI_STREAM_TEXT = 10, - TRANSCRIBE_ABI_SESSION_LIMITS = 11, - TRANSCRIBE_ABI_EXT = 12, - TRANSCRIBE_ABI_DEVICE_INFO = 13, - TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14, - TRANSCRIBE_ABI_BACKEND_INIT_PARAMS = 15, + TRANSCRIBE_ABI_MODEL_LOAD_PARAMS = 0, + TRANSCRIBE_ABI_SESSION_PARAMS = 1, + TRANSCRIBE_ABI_RUN_PARAMS = 2, + TRANSCRIBE_ABI_STREAM_PARAMS = 3, + TRANSCRIBE_ABI_CAPABILITIES = 4, + TRANSCRIBE_ABI_TIMINGS = 5, + TRANSCRIBE_ABI_SEGMENT = 6, + TRANSCRIBE_ABI_WORD = 7, + TRANSCRIBE_ABI_TOKEN = 8, + TRANSCRIBE_ABI_STREAM_UPDATE = 9, + TRANSCRIBE_ABI_STREAM_TEXT = 10, + TRANSCRIBE_ABI_SESSION_LIMITS = 11, + TRANSCRIBE_ABI_EXT = 12, + TRANSCRIBE_ABI_DEVICE_INFO = 13, + TRANSCRIBE_ABI_SPEAKER_SEGMENT = 14, + TRANSCRIBE_ABI_BACKEND_INIT_PARAMS = 15, + /* include/transcribe/diarize.h */ + TRANSCRIBE_ABI_DIARIZE_INFO = 16, + TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS = 17, + TRANSCRIBE_ABI_DIARIZE_PARAMS = 18, } transcribe_abi_struct; /* sizeof / alignof of the selected public struct, or 0 for an unknown id. @@ -668,9 +674,11 @@ TRANSCRIBE_API transcribe_status transcribe_ext_check(const struct transcribe_ex */ typedef enum { /* transcribe_run_params::family */ - TRANSCRIBE_EXT_SLOT_RUN = 0, + TRANSCRIBE_EXT_SLOT_RUN = 0, /* transcribe_stream_params::family */ - TRANSCRIBE_EXT_SLOT_STREAM = 1, + TRANSCRIBE_EXT_SLOT_STREAM = 1, + /* transcribe_diarize_params::family (include/transcribe/diarize.h) */ + TRANSCRIBE_EXT_SLOT_DIARIZE_RUN = 2, } transcribe_ext_slot; /* @@ -1345,6 +1353,25 @@ struct transcribe_capabilities { TRANSCRIBE_API void transcribe_capabilities_init(struct transcribe_capabilities * out); +/* + * Roles: the kinds of work a loaded model can do. ASR is this header's + * transcribe_session_* / transcribe_run* API. The other roles each have + * their own header under include/transcribe/ and their own session type. + * A model may serve more than one role. Values are bits; append-only. + */ +typedef enum { + TRANSCRIBE_ROLE_ASR = 1u << 0, + TRANSCRIBE_ROLE_DIARIZE = 1u << 1, +} transcribe_role; + +/* + * Bitmask of transcribe_role values the loaded model serves. Fixed at load + * and never changes for the model's lifetime. Returns 0 for NULL. Entry + * points that need a role the model lacks return + * TRANSCRIBE_ERR_UNSUPPORTED_ROLE. + */ +TRANSCRIBE_API uint32_t transcribe_model_roles(const struct transcribe_model * model); + /* * Read model capabilities into caller-owned storage. The caller * initializes *out_caps via transcribe_capabilities_init() (zero-fill); @@ -1352,9 +1379,12 @@ TRANSCRIBE_API void transcribe_capabilities_init(struct transcribe_capabilities * beyond the caller's struct_size untouched. * * Returns: - * TRANSCRIBE_ERR_INVALID_ARG model or out_caps is NULL. - * TRANSCRIBE_ERR_BAD_STRUCT_SIZE out_caps->struct_size is 0 or - * smaller than the library's minimum. + * TRANSCRIBE_ERR_INVALID_ARG model or out_caps is NULL. + * TRANSCRIBE_ERR_BAD_STRUCT_SIZE out_caps->struct_size is 0 or + * smaller than the library's minimum. + * TRANSCRIBE_ERR_UNSUPPORTED_ROLE the model does not serve + * TRANSCRIBE_ROLE_ASR. These are ASR + * capabilities; *out_caps is untouched. * * Pointer fields written by the library (e.g. `languages`) point at * model-owned storage and remain valid until transcribe_model_free(). @@ -1537,6 +1567,10 @@ TRANSCRIBE_API void transcribe_model_free(struct transcribe_model * model); * * params may be NULL for library defaults, or initialize a struct with * transcribe_session_params_init(). See transcribe_model_load_file. + * + * Returns TRANSCRIBE_ERR_UNSUPPORTED_ROLE when the model does not serve + * TRANSCRIBE_ROLE_ASR (see transcribe_model_roles); transcribe_open + * returns the same and frees the model it loaded. */ TRANSCRIBE_API transcribe_status transcribe_session_init(struct transcribe_model * model, const struct transcribe_session_params * params, @@ -1604,7 +1638,10 @@ TRANSCRIBE_API const struct transcribe_model * transcribe_get_model(const struct /* * Run one batch transcription. * - * pcm: mono float32 PCM samples in [-1.0, 1.0] at 16 kHz. + * pcm: mono float32 PCM samples in [-1.0, 1.0] at 16 kHz. Every + * sample must be finite: NaN or +-Inf returns + * TRANSCRIBE_ERR_INVALID_ARG and leaves the previous result + * in place. Silence is valid. * n_samples: number of samples in pcm. Must be strictly positive; * a non-positive count returns TRANSCRIBE_ERR_INVALID_ARG * (same rule as transcribe_stream_feed). @@ -1683,7 +1720,10 @@ TRANSCRIBE_API transcribe_status transcribe_run(struct transcribe_session * * or n_samples[i] <= 0) fails only that * utterance and is reported there. * TRANSCRIBE_ERR_INVALID_ARG session / pcm / n_samples NULL, n <= 0, - * or the session is in an ACTIVE stream. + * any utterance holds a non-finite sample + * (NaN / +-Inf; the previous result is + * preserved), or the session is in an + * ACTIVE stream. * TRANSCRIBE_ERR_BAD_STRUCT_SIZE params->struct_size below the minimum. * TRANSCRIBE_ERR_NOT_IMPLEMENTED the model has no run path at all. * ... plus the same shared-param rejections as transcribe_run @@ -2223,8 +2263,10 @@ TRANSCRIBE_API transcribe_status transcribe_stream_begin(struct transcribe_sessi * Feed PCM into the active stream. 16 kHz mono float32, same as * transcribe_run. * - * pcm must be non-null and n_samples must be strictly greater than - * zero. Polling the stream without supplying audio is unsupported — + * pcm must be non-null, n_samples must be strictly greater than zero, + * and every sample must be finite (NaN / +-Inf returns + * TRANSCRIBE_ERR_INVALID_ARG and the stream stays ACTIVE, unchanged). + * Polling the stream without supplying audio is unsupported — * use the stream accessors (transcribe_stream_revision, * transcribe_stream_get_text, transcribe_stream_n_committed_*, * transcribe_stream_last_status, transcribe_stream_get_state) to inspect diff --git a/include/transcribe/diarize.h b/include/transcribe/diarize.h new file mode 100644 index 000000000..4a4c2793a --- /dev/null +++ b/include/transcribe/diarize.h @@ -0,0 +1,95 @@ +/* + * include/transcribe/diarize.h - DIARIZE role: who spoke when. + * + * Includes transcribe.h; safe to include in C or C++ TUs. For models whose + * product is speaker turns rather than text (transcribe_model_roles() has + * TRANSCRIBE_ROLE_DIARIZE). ASR models that attribute speakers inside a + * transcript keep using transcribe_run_params::diarize instead. + * + * Open a transcribe_diarize_session on a loaded model, run, then read the + * speaker segments. Threading and lifetime rules: docs/roles.md. + */ + +#ifndef TRANSCRIBE_DIARIZE_H +#define TRANSCRIBE_DIARIZE_H + +#include "transcribe.h" + +#ifdef __cplusplus +extern "C" { +#endif + +struct transcribe_diarize_session; + +/* Static facts about a diarization model. */ +struct transcribe_diarize_info { + uint64_t struct_size; + int32_t sample_rate; /* input PCM rate (16000) */ + int32_t max_speakers; /* speaker_id is in [1, max_speakers] */ +}; + +struct transcribe_diarize_session_params { + uint64_t struct_size; + int32_t n_threads; /* 0 = library default */ +}; + +struct transcribe_diarize_params { + uint64_t struct_size; + /* Typed family extension on TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, or NULL. */ + const struct transcribe_ext * family; +}; + +TRANSCRIBE_API void transcribe_diarize_info_init(struct transcribe_diarize_info * out); +TRANSCRIBE_API void transcribe_diarize_session_params_init(struct transcribe_diarize_session_params * params); +TRANSCRIBE_API void transcribe_diarize_params_init(struct transcribe_diarize_params * params); + +/* UNSUPPORTED_ROLE when the model does not serve TRANSCRIBE_ROLE_DIARIZE. */ +TRANSCRIBE_API transcribe_status transcribe_diarize_get_info(const struct transcribe_model * model, + struct transcribe_diarize_info * out); + +/* params may be NULL for defaults. UNSUPPORTED_ROLE when the model does not + * serve TRANSCRIBE_ROLE_DIARIZE. On failure *out_session is NULL. */ +TRANSCRIBE_API transcribe_status +transcribe_diarize_session_init(struct transcribe_model * model, + const struct transcribe_diarize_session_params * params, + struct transcribe_diarize_session ** out_session); + +/* NULL is a no-op. */ +TRANSCRIBE_API void transcribe_diarize_session_free(struct transcribe_diarize_session * session); + +/* Polled during a run; returning true stops it with TRANSCRIBE_ERR_ABORTED. */ +TRANSCRIBE_API void transcribe_diarize_set_abort_callback(struct transcribe_diarize_session * session, + transcribe_abort_callback cb, + void * user_data); + +/* + * Diarize one recording: 16 kHz mono float32 PCM, every sample finite. + * params may be NULL for defaults. Replaces the previous result. Malformed + * input (NULL pointers, n_samples <= 0, NaN / Inf, a bad extension) returns + * an error before the previous result is touched. + */ +TRANSCRIBE_API transcribe_status transcribe_diarize_run(struct transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const struct transcribe_diarize_params * params); + +/* + * Speaker segments of the last run, grouped by speaker and time-ordered + * within a speaker; segments of different speakers may overlap. 0 before any + * run, and 0 after a run that passed input validation but then failed + * (aborted, backend error). An out-of-range index returns OK with a zeroed row. + */ +TRANSCRIBE_API int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session); +TRANSCRIBE_API transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session, + int i, + struct transcribe_speaker_segment * out); + +/* load_ms plus the last run's stage times. */ +TRANSCRIBE_API transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, + struct transcribe_timings * out); + +#ifdef __cplusplus +} +#endif + +#endif /* TRANSCRIBE_DIARIZE_H */ diff --git a/include/transcribe/extensions.h b/include/transcribe/extensions.h index b93bf1460..16c5684ad 100644 --- a/include/transcribe/extensions.h +++ b/include/transcribe/extensions.h @@ -14,6 +14,7 @@ #define TRANSCRIBE_EXTENSIONS_H #include "transcribe.h" +#include "transcribe/diarize.h" #include "transcribe/moonshine_streaming.h" #include "transcribe/parakeet.h" #include "transcribe/sortformer.h" diff --git a/include/transcribe/sortformer.h b/include/transcribe/sortformer.h index 751b11c88..a488671b0 100644 --- a/include/transcribe/sortformer.h +++ b/include/transcribe/sortformer.h @@ -1,23 +1,12 @@ /* * include/transcribe/sortformer.h - Sortformer-family public extension. * - * Includes transcribe.h; safe to include in C or C++ TUs. Holds the - * streaming-operating-point run extension, its kind constant, and its - * init function. - * - * Sortformer (diar_streaming_sortformer_4spk-v2.1) is a diarization-only model: a run - * produces no text; the product is the who-spoke-when rows read back via - * transcribe_n_speaker_segments / transcribe_get_speaker_segment - * (TRANSCRIBE_FEATURE_DIARIZATION). The compute core is streaming - * (AOSC speaker cache + FIFO) but the shipped entry point is the batch - * transcribe_run over a whole recording, so this extension lives on the - * RUN slot. A future push-audio entry point (transcribe_stream_begin/ - * feed) would register a separate STREAM-slot kind taking the same - * preset enum. - * - * Probe via transcribe_model_accepts_ext_kind(model, - * TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM) - * before pointing transcribe_run_params::family at the struct. + * Includes transcribe.h; safe to include in C or C++ TUs. Sortformer + * (diar_streaming_sortformer_4spk-v2.1) serves the DIARIZE role + * (include/transcribe/diarize.h). Its one extension picks the streaming + * operating point for transcribe_diarize_run; probe with + * transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, + * TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE). * * FourCC kinds are reserved in docs/extension-kinds.md. */ @@ -31,9 +20,6 @@ extern "C" { #endif -/* 'SFST' little-endian = 0x54534653 */ -#define TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM 0x54534653u - /* * Streaming operating point (latency / accuracy trade-off). * @@ -56,8 +42,8 @@ extern "C" { * (many small windows); see the family doc for * measured throughput. * - * Values outside the enum range are rejected by transcribe_run with - * TRANSCRIBE_ERR_INVALID_ARG before the previous result is cleared. + * Values outside the enum range are rejected by transcribe_diarize_run + * with TRANSCRIBE_ERR_INVALID_ARG before the previous result is cleared. */ typedef enum { TRANSCRIBE_SORTFORMER_PRESET_DEFAULT = 0, @@ -66,13 +52,18 @@ typedef enum { TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY = 3, } transcribe_sortformer_preset; -struct transcribe_sortformer_stream_ext { +/* 'SFDR' little-endian = 0x52444653 (DIARIZE_RUN slot) */ +#define TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE 0x52444653u + +/* transcribe_diarize_params::family: the operating point for + * transcribe_diarize_run. */ +struct transcribe_sortformer_diarize_ext { struct transcribe_ext ext; transcribe_sortformer_preset preset; }; -/* Fills ext.size/kind and preset = DEFAULT (GGUF-shipped cfg). */ -TRANSCRIBE_API void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * ext); +/* Fills ext.size/kind and preset = DEFAULT. */ +TRANSCRIBE_API void transcribe_sortformer_diarize_ext_init(struct transcribe_sortformer_diarize_ext * ext); #ifdef __cplusplus } diff --git a/scripts/catalog/render.py b/scripts/catalog/render.py index 44d3947f6..3a38d4f47 100755 --- a/scripts/catalog/render.py +++ b/scripts/catalog/render.py @@ -369,12 +369,12 @@ def doc_for(record: dict) -> pathlib.Path | None: def block_family_index(records: dict[str, dict], attrs: dict[str, str]) -> list[str]: """The root README's supported-models table: one row per documentation - page, listing the variants it covers. `transcribe=false` selects the - models that only diarize.""" - want = as_bool(attrs.get("transcribe"), True) + page, listing the variants it covers, for one role (`role=`, default + asr).""" + want = attrs.get("role", "asr") groups: dict[str, dict] = {} for variant, record in records.items(): - if bool(record.get("capabilities", {}).get("transcribe", {}).get("supported")) != want: + if record.get("role", "asr") != want: continue doc = doc_for(record) if doc is not None: diff --git a/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml b/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml index d9d6365bc..dc4a5be46 100644 --- a/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml +++ b/scripts/hf_cards/diar_streaming_sortformer_4spk-v2.1.yaml @@ -40,22 +40,19 @@ usage: | cmake -B build && cmake --build build ``` - Run on a 16 kHz mono WAV. This is a diarizer, not a transcription model — - use the JSONL output to get the speaker segments (the plain output's - `text:` line is empty by design): + Run on a 16 kHz mono WAV. This is a diarizer, not a transcription model: + the CLI prints speaker segments. ```bash - echo input.wav > files.txt - build/bin/transcribe-cli \ - -m diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf \ - --batch files.txt --batch-jsonl - # {"file":"input.wav","text":"","speakers":[{"t0_ms":320,"t1_ms":2400,"speaker_id":1},...]} + build/bin/transcribe-cli -m diar_streaming_sortformer_4spk-v2.1-Q8_0.gguf input.wav + # speaker segments: 5 + # [ 0.32 -> 2.40] S1 + # ... ``` - From the C API, read segments via `transcribe_n_speaker_segments` / - `transcribe_get_speaker_segment`, and select the streaming operating - point (latency / accuracy trade-off) with the run extension in - `include/transcribe/sortformer.h`. + From the C API, use the DIARIZE role (`include/transcribe/diarize.h`), + and select the streaming operating point (latency / accuracy trade-off) + with the extension in `include/transcribe/sortformer.h`. If your audio isn't already 16 kHz mono WAV, convert it first: diff --git a/scripts/validate.py b/scripts/validate.py index 43af1e3fb..cbd9fc717 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -548,6 +548,8 @@ def cmd_cpp(args: argparse.Namespace) -> int: f"with exit code {result.returncode}" ) transcript = parse_cli_transcript(result.stdout or "") + if transcript is None and "speaker segments:" in (result.stdout or ""): + continue # a diarizer has no transcript if transcript is None: raise SystemExit( f"error: cpp dump [{args.family}/{case_name}] did not emit a transcript line" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5b0931f71..081473822 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,6 +12,8 @@ add_library(transcribe transcribe.cpp + transcribe-asr.cpp + transcribe-diarize.cpp transcribe-loader.cpp transcribe-arch.cpp transcribe-meta.cpp @@ -63,7 +65,6 @@ add_library(transcribe arch/moss/decoder.cpp arch/sortformer/model.cpp arch/sortformer/stream.cpp - arch/sortformer/capabilities.cpp arch/sortformer/weights.cpp arch/voxtral/model.cpp arch/voxtral/capabilities.cpp diff --git a/src/arch/gigaam/mel.cpp b/src/arch/gigaam/mel.cpp index bf17ce6f1..8670af0aa 100644 --- a/src/arch/gigaam/mel.cpp +++ b/src/arch/gigaam/mel.cpp @@ -167,15 +167,7 @@ transcribe_status GigaamMelFrontend::compute(const float * pcm, } } }; - std::vector workers; - workers.reserve(n_workers > 0 ? static_cast(n_workers - 1) : 0); - for (int worker_id = 1; worker_id < n_workers; ++worker_id) { - workers.emplace_back(worker, worker_id); - } - worker(0); - for (auto & thread : workers) { - thread.join(); - } + transcribe::run_on_threads(n_workers, worker); // mel[m, t] = sum_k filterbank[m, k] * power[t, k]. #if TRANSCRIBE_HAS_BLAS diff --git a/src/arch/parakeet/multitalker.cpp b/src/arch/parakeet/multitalker.cpp index 5b9a29d12..0c5b18122 100644 --- a/src/arch/parakeet/multitalker.cpp +++ b/src/arch/parakeet/multitalker.cpp @@ -651,7 +651,7 @@ transcribe_status run_multitalker(ParakeetSession * pc, // Transcript-independent "who spoke when" rows from the diarizer preds // (same emission as the standalone sortformer family). - sf::probs_to_speaker_segments(pc, probs, T_diar, n_spk, diar_ms_per_frame, /*threshold=*/0.5f); + transcribe::probs_to_segments(probs.data(), T_diar, n_spk, diar_ms_per_frame, pc->speaker_segments); // ---- Elide to the caller's requested granularity (same convention // as decode_and_populate). ---- diff --git a/src/arch/sortformer/capabilities.cpp b/src/arch/sortformer/capabilities.cpp deleted file mode 100644 index 7c929ef27..000000000 --- a/src/arch/sortformer/capabilities.cpp +++ /dev/null @@ -1,35 +0,0 @@ -// arch/sortformer/capabilities.cpp - Sortformer capability defaults. -// -// Applied before transcribe::read_capability_kv (KV present overrides, -// KV absent keeps the default). Sortformer is a pure frame-level diarizer: -// no transcript, no translation, no transcript-timestamp capability. Its -// output is speaker segments via the transcript-independent speaker_segment -// surface, so DIARIZATION is a family invariant. - -#include "sortformer.h" - -namespace transcribe::sortformer { - -void apply_family_invariants(transcribe_model & model) { - transcribe_capabilities & caps = model.caps; - - // Fixed 16 kHz mel bank (NeMo AudioToMelSpectrogramPreprocessor). - caps.native_sample_rate = 16000; - - // Not a transcription model. - caps.supports_translate = false; - - // Diarization segment times are intrinsic; there is no transcript, so - // there is no transcript-timestamp capability to advertise. - caps.max_timestamp_kind = TRANSCRIBE_TIMESTAMPS_NONE; - - // The model emits speaker-attributed output (T x 4 activity -> "who - // spoke when" segments). run() populates the transcript-independent - // speaker_segment surface directly from the probs. - transcribe::set_feature(&model, TRANSCRIBE_FEATURE_DIARIZATION, true); - - // Abort callback honored at the top of each run. - transcribe::set_feature(&model, TRANSCRIBE_FEATURE_CANCELLATION, true); -} - -} // namespace transcribe::sortformer diff --git a/src/arch/sortformer/model.cpp b/src/arch/sortformer/model.cpp index d1c229a9d..b9d11f87b 100644 --- a/src/arch/sortformer/model.cpp +++ b/src/arch/sortformer/model.cpp @@ -12,6 +12,7 @@ #include "ggml.h" #include "gguf.h" #include "sortformer.h" +#include "transcribe-abi.h" #include "transcribe-arch.h" #include "transcribe-backend.h" #include "transcribe-batch-util.h" @@ -21,7 +22,6 @@ #include "transcribe-loader.h" #include "transcribe-log.h" #include "transcribe-mel.h" -#include "transcribe-meta.h" #include #include @@ -29,7 +29,6 @@ #include #include #include -#include #include #include #include @@ -70,8 +69,6 @@ DiarStreamScratch::~DiarStreamScratch() { } } -SortformerSession::~SortformerSession() = default; - namespace { // Map the Sortformer hparams onto the parakeet encoder hparams that @@ -400,8 +397,8 @@ conf::BlockView sf_block_view(const pk::ParakeetBlock & b) { return v; } -// Host sinusoidal rel-pos table [pos_len, d_model] (same as the offline -// builder in run(); factored for per-chunk reuse in the streaming path). +// Host sinusoidal rel-pos table [pos_len, d_model], shared by the offline +// and streaming paths. void fill_rel_pos_emb(std::vector & buf, std::vector & div_term, int pos_len, int d_model) { const int zero_index = (pos_len - 1) / 2; const float ln_10000 = std::log(10000.0f); @@ -540,16 +537,6 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par m->variant = loader.variant().empty() ? k_default_variant : loader.variant(); m->backend.clear(); - apply_family_invariants(*m); - m->caps.n_languages = 0; - m->caps.languages = nullptr; - - if (const transcribe_status st = read_capability_kv(loader.gguf(), m->caps); st != TRANSCRIBE_OK) { - return st; - } - if (const transcribe_status st = read_languages_kv(loader.gguf(), *m); st != TRANSCRIBE_OK) { - return st; - } if (const transcribe_status st = read_sortformer_hparams(loader.gguf(), m->hparams); st != TRANSCRIBE_OK) { return st; } @@ -624,6 +611,9 @@ transcribe_status load(Loader & loader, const transcribe_model_load_params * par return st; } + m->roles = TRANSCRIBE_ROLE_DIARIZE; + // The abort callback is honored (transcribe_diarize_set_abort_callback). + transcribe::set_feature(m.get(), TRANSCRIBE_FEATURE_CANCELLATION, true); m->t_load_us = ggml_time_us() - t_load_start; *out_model = m.release(); return TRANSCRIBE_OK; @@ -656,50 +646,6 @@ transcribe_status fuse_embedded_diar_bn(SortformerEmbedded & e, return fuse_conformer_bn_core(e.conformer.blocks, e.conformer_hp.enc_d_model, backend, out_ctx, out_buffer); } -transcribe_status init_context(transcribe_model * model, - const transcribe_session_params * params, - transcribe_session ** out_ctx) { - if (model->arch != &arch) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - auto pc = std::make_unique(); - pc->model = model; - pc->n_threads = params->n_threads; - pc->kv_type = params->kv_type; - *out_ctx = pc.release(); - return TRANSCRIBE_OK; -} - -// Threshold-based probs -> speaker segments (simple offline segmentation; -// the DER-grade dihard3 post-processing is a later, streaming-checkpoint -// concern). probs is row-major [T, n_spk]. Emits one contiguous run per -// speaker as a speaker_segment row. Non-static: the parakeet multitalker -// bundle path emits its transcript-independent view through this too. -void probs_to_speaker_segments(transcribe_session * pc, - const std::vector & probs, - int T, - int n_spk, - double ms_per_frame, - float threshold) { - for (int s = 0; s < n_spk; ++s) { - int run_start = -1; - for (int t = 0; t <= T; ++t) { - const bool active = (t < T) && (probs[static_cast(t) * n_spk + s] > threshold); - if (active && run_start < 0) { - run_start = t; - } else if (!active && run_start >= 0) { - transcribe_session::SpeakerSegmentEntry row; - row.t0_ms = static_cast(std::llround(run_start * ms_per_frame)); - row.t1_ms = static_cast(std::llround(t * ms_per_frame)); - row.speaker_id = s + 1; // 1-based - row.p = std::numeric_limits::quiet_NaN(); - pc->speaker_segments.push_back(row); - run_start = -1; - } - } - } -} - // Lazily create the persistent multi-backend scheduler shared by the offline // and streaming graphs. static transcribe_status ensure_sched(SortformerSession * pc, SortformerModel * pm) { @@ -811,7 +757,7 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & sc, const float * mel_buf, int mel_n_mels, int mel_n_frames, - transcribe_session * abort_session) { + transcribe::SessionCore * abort_session) { const int ed = chp.enc_d_model; const int n_spk = hp.max_speakers; const int sub = hp.enc_subsampling_factor; @@ -974,13 +920,14 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & sc, } // Family wrapper: resolve the operating point, run the core, trim, dump, -// and emit speaker segments on the session. +// and hand back the frame probabilities. static transcribe_status run_streaming(SortformerSession * pc, SortformerModel * pm, int mel_n_mels, int mel_n_frames, double ms_per_frame, - transcribe_sortformer_preset preset) { + transcribe_sortformer_preset preset, + DiarizeProbs & out) { const SortformerStreamParams P = resolve_stream_params(pm->hparams, preset); if (const transcribe_status st = ensure_sched(pc, pm); st != TRANSCRIBE_OK) { return st; @@ -1007,33 +954,21 @@ static transcribe_status run_streaming(SortformerSession * pc, static_cast(used_n) * n_spk, shape, 2, "diarize"); } - probs_to_speaker_segments(pc, pc->scratch.stream.total_preds, used_n, n_spk, ms_per_frame, /*threshold=*/0.5f); + out.probs.assign(pc->scratch.stream.total_preds.begin(), + pc->scratch.stream.total_preds.begin() + static_cast(used_n) * n_spk); + out.n_frames = used_n; + out.n_speakers = n_spk; + out.frame_ms = ms_per_frame; return TRANSCRIBE_OK; } -transcribe_status run(transcribe_session * session, - const float * pcm, - int n_samples, - const transcribe_run_params * params) { - auto * pc = static_cast(session); - auto * pm = static_cast(session->model); - - if (pc->poll_abort()) { - return TRANSCRIBE_ERR_ABORTED; - } - - // Streaming operating-point run ext (kind/size/range already validated - // by the dispatcher + run_validate pre-clear; re-check is belt-and-braces). - transcribe_sortformer_preset preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; - if (params != nullptr && params->family != nullptr) { - if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - sizeof(struct transcribe_sortformer_stream_ext)); - st != TRANSCRIBE_OK) { - return st; - } - preset = reinterpret_cast(params->family)->preset; - } - pc->clear_result(); +// Mel -> (optional offline parity dump) -> streaming forward. +static transcribe_status diarize_pcm(SortformerSession * pc, + SortformerModel * pm, + const float * pcm, + int n_samples, + transcribe_sortformer_preset preset, + DiarizeProbs & out) { transcribe::debug::init(); if (!pm->mel.has_value()) { @@ -1061,57 +996,81 @@ transcribe_status run(transcribe_session * session, } // Streaming AOSC/FIFO forward drives the product output + diar.probs. - if (const transcribe_status st = run_streaming(pc, pm, mel_n_mels, mel_n_frames, ms_per_frame, preset); - st != TRANSCRIBE_OK) { - return st; - } - - pc->result_kind = TRANSCRIBE_TIMESTAMPS_NONE; - pc->has_result = true; - return TRANSCRIBE_OK; + return run_streaming(pc, pm, mel_n_mels, mel_n_frames, ms_per_frame, preset, out); } -// Kind+slot probe. Sortformer ships one RUN-slot extension (the streaming -// operating-point preset); there is no STREAM-slot surface (no push-audio -// entry point yet — a future one registers a separate kind). +// Kind+slot probe: the operating-point preset on the DIARIZE_RUN slot. static bool accepts_ext_kind(const transcribe_model * model, transcribe_ext_slot slot, uint32_t kind) { - if (model == nullptr) { - return false; - } - if (slot != TRANSCRIBE_EXT_SLOT_RUN) { - return false; - } - return kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM; + return model != nullptr && slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && + kind == TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; } -// Pre-clear validation for the _RUN slot (see Arch::run_validate): reject a -// malformed ext or an out-of-range preset before the previous result -// snapshot is destroyed. -static transcribe_status run_validate(const transcribe_session * /*ctx*/, const transcribe_run_params * params) { - if (params == nullptr || params->family == nullptr) { - return TRANSCRIBE_OK; // NULL ext -> family defaults - } - if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, - sizeof(struct transcribe_sortformer_stream_ext)); - st != TRANSCRIBE_OK) { - return st; - } - const auto * ext = reinterpret_cast(params->family); - switch (ext->preset) { +// The caller's preset, read as a raw int (see transcribe::enum_field_raw): +// an out-of-range value must be rejected before it is loaded as the enum. +static int ext_preset_raw(const transcribe_diarize_params * params) { + return transcribe::enum_field_raw( + &reinterpret_cast(params->family)->preset); +} + +static transcribe_status check_preset(int preset) { + switch (preset) { case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: case TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY: case TRANSCRIBE_SORTFORMER_PRESET_HIGH_LATENCY: case TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY: return TRANSCRIBE_OK; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } +} + +// ---- DIARIZE role ---- + +static int diarize_max_speakers(const transcribe_model * model) { + return static_cast(model)->hparams.max_speakers; +} + +static transcribe_diarize_session * diarize_new_session() { + return new SortformerSession(); +} + +static transcribe_status diarize_run_validate(const transcribe_diarize_params * params) { + if (const transcribe_status st = transcribe_ext_check(params->family, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE, + sizeof(struct transcribe_sortformer_diarize_ext)); + st != TRANSCRIBE_OK) { + return st; + } + return check_preset(ext_preset_raw(params)); +} + +static transcribe_status diarize_run(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params, + DiarizeProbs & out) { + auto * pc = static_cast(session); + if (pc->poll_abort()) { + return TRANSCRIBE_ERR_ABORTED; } - return TRANSCRIBE_ERR_INVALID_ARG; + // Range-checked by diarize_run_validate before the dispatcher got here. + const transcribe_sortformer_preset preset = params->family != nullptr ? + static_cast(ext_preset_raw(params)) : + TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; + return diarize_pcm(pc, static_cast(session->model), pcm, n_samples, preset, out); } +static const DiarizeOps k_diarize_ops = { + /* .max_speakers = */ diarize_max_speakers, + /* .new_session = */ diarize_new_session, + /* .run_validate = */ diarize_run_validate, + /* .run = */ diarize_run, +}; + extern const Arch arch = { /* .name = */ "sortformer", /* .load = */ load, - /* .init_context = */ init_context, - /* .run = */ run, + /* .init_context = */ nullptr, + /* .run = */ nullptr, /* .run_batch = */ nullptr, /* .stream_validate = */ nullptr, /* .stream_begin = */ nullptr, @@ -1119,7 +1078,8 @@ extern const Arch arch = { /* .stream_finalize = */ nullptr, /* .stream_reset = */ nullptr, /* .accepts_ext_kind = */ accepts_ext_kind, - /* .run_validate = */ run_validate, + /* .run_validate = */ nullptr, + /* .diarize = */ &k_diarize_ops, }; } // namespace transcribe::sortformer @@ -1130,12 +1090,12 @@ extern const Arch arch = { // transcribe_ext header (size + kind) and the preset default. // --------------------------------------------------------------------------- -extern "C" void transcribe_sortformer_stream_ext_init(struct transcribe_sortformer_stream_ext * p) { +extern "C" void transcribe_sortformer_diarize_ext_init(struct transcribe_sortformer_diarize_ext * p) { if (p == nullptr) { return; } std::memset(p, 0, sizeof(*p)); p->ext.size = sizeof(*p); - p->ext.kind = TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM; + p->ext.kind = TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE; p->preset = TRANSCRIBE_SORTFORMER_PRESET_DEFAULT; } diff --git a/src/arch/sortformer/sortformer.h b/src/arch/sortformer/sortformer.h index bb31eeddc..dd1bebeba 100644 --- a/src/arch/sortformer/sortformer.h +++ b/src/arch/sortformer/sortformer.h @@ -14,10 +14,10 @@ #include "../parakeet/encoder.h" // build_encoder_graph #include "../parakeet/weights.h" // ParakeetHParams / ParakeetWeights (conformer) #include "transcribe-backend.h" +#include "transcribe-diarize.h" #include "transcribe-mel.h" #include "transcribe-model.h" -#include "transcribe-session.h" -#include "transcribe/sortformer.h" // public preset enum + run ext +#include "transcribe/sortformer.h" // public preset enum + diarize ext #include "weights.h" #include @@ -37,11 +37,6 @@ typedef struct ggml_backend_sched * ggml_backend_sched_t; namespace transcribe::sortformer { -// Family defaults, applied before transcribe::read_capability_kv (KV -// present overrides, KV absent keeps the default). Defined in -// capabilities.cpp. -void apply_family_invariants(transcribe_model & model); - // Concrete model. Owns ctx_meta (every weight tensor's data buffer); // the destructor frees it, invalidating every borrowed ggml_tensor* in // `conformer` / `weights`. @@ -186,19 +181,16 @@ struct DiarStreamScratch { ~DiarStreamScratch(); }; -// Concrete context. The per-call compute context and multi-backend -// scheduler are owned by the transcribe_session base (sched / compute_ctx). -struct SortformerSession final : public transcribe_session { - // Per-context scratch reused across runs. +// Concrete DIARIZE session. The per-call compute context and multi-backend +// scheduler are owned by the SessionCore base (sched / compute_ctx). +struct SortformerSession final : public transcribe_diarize_session { + // Per-session scratch reused across runs. std::vector mel_buf; std::vector probs_host; // [n_spk * T], read back from diar.preds // Streaming scratch (AOSC/FIFO path + rel-pos tables, shared with the // offline forward's pos_emb fill). DiarStreamScratch scratch; - - SortformerSession() = default; - ~SortformerSession() override; }; // ---- Embedded-diarizer surface (multitalker bundle) -------------------- // @@ -252,15 +244,6 @@ transcribe_status run_diar_streaming_core(DiarStreamScratch & const float * mel_buf, int mel_n_mels, int mel_n_frames, - transcribe_session * abort_session); - -// Threshold-based probs -> speaker_segment rows (probs row-major [T, n_spk], -// speaker_id 1-based, p = NaN). Shared with the multitalker bundle path. -void probs_to_speaker_segments(transcribe_session * session, - const std::vector & probs, - int T, - int n_spk, - double ms_per_frame, - float threshold); + transcribe::SessionCore * abort_session); } // namespace transcribe::sortformer diff --git a/src/arch/sortformer/stream.cpp b/src/arch/sortformer/stream.cpp index 643795b90..f6bb19ea0 100644 --- a/src/arch/sortformer/stream.cpp +++ b/src/arch/sortformer/stream.cpp @@ -83,8 +83,8 @@ SortformerStreamParams resolve_stream_params(const SortformerHParams & hp, trans p.spkcache_update_period = hp.stream_spkcache_update_period > 0 ? hp.stream_spkcache_update_period : p.spkcache_update_period; - // Public run-ext preset (DEFAULT keeps the GGUF cfg). Range-checked by - // run_validate before the dispatcher clears the previous result. + // Public diarize-ext preset (DEFAULT keeps the GGUF cfg). Range-checked by + // diarize_run_validate before the dispatcher clears the previous result. const char * ext_name = nullptr; switch (preset) { case TRANSCRIBE_SORTFORMER_PRESET_DEFAULT: diff --git a/src/transcribe-abi.h b/src/transcribe-abi.h index dc792feb1..dfe9fe163 100644 --- a/src/transcribe-abi.h +++ b/src/transcribe-abi.h @@ -1,19 +1,52 @@ // transcribe-abi.h - internal helpers for the size-aware public ABI. // -// Shared by the central dispatcher (transcribe.cpp) and per-family public -// accessors (e.g. arch/whisper/public.cpp) so the struct_size validation -// and copy-out truncation logic lives in exactly one place. Not part of -// the public API. +// Shared by the public dispatchers (transcribe.cpp, transcribe-asr.cpp) and +// per-family public accessors (e.g. arch/whisper/public.cpp) so the raw enum +// reads, struct_size validation and copy-out truncation logic live in +// exactly one place. Not part of the public API. #pragma once #include "transcribe.h" +#include #include #include +// End offset of `field`: the minimum struct_size accepted for a caller-owned +// struct is the prefix through the last field the library reads or writes. +#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) + namespace transcribe { +// Raw enum reads at the public ABI boundary +// +// C callers can store ANY int in an enum-typed ABI field; in C++ loading an +// out-of-range value through an enum-typed lvalue is UB (UBSan traps it). So +// every public entry point reads caller-supplied enum values as raw bytes +// first, validates the raw int, and only then converts to the enum type. All +// public enums are int-sized; the static_assert below pins one. + +inline int enum_field_raw(const void * field) { + int raw; + std::memcpy(&raw, field, sizeof(raw)); + return raw; +} + +static_assert(sizeof(transcribe_backend_request) == sizeof(int), + "public enums must be int-sized for raw boundary reads"); + +// Rejects NaN / +-Inf; every PCM entry point checks this before touching +// result or stream state. +inline bool pcm_is_finite(const float * pcm, int n_samples) { + for (int i = 0; i < n_samples; ++i) { + if (!std::isfinite(pcm[i])) { + return false; + } + } + return true; +} + // Strict size check for every caller-owned struct (inputs AND outputs): // struct_size smaller than the prefix the library relies on is rejected. // `0` is invalid — defaults are reached by passing NULL where the entry @@ -44,4 +77,28 @@ inline void copy_out_prefix(void * dst, const void * src, uint64_t caller_size, std::memcpy(dst, src, static_cast(n)); } +// The body of every public *_init(): zero-fill, then stamp struct_size. +template inline void init_sized(T * p) { + if (p != nullptr) { + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + } +} + +// Copy stage times (microseconds) out as transcribe_timings (milliseconds); +// out->struct_size is already checked. +inline void copy_out_timings(int64_t load_us, + int64_t mel_us, + int64_t encode_us, + int64_t decode_us, + struct transcribe_timings * out) { + transcribe_timings staged{}; + staged.struct_size = out->struct_size; + staged.load_ms = static_cast(load_us) / 1000.0f; + staged.mel_ms = static_cast(mel_us) / 1000.0f; + staged.encode_ms = static_cast(encode_us) / 1000.0f; + staged.decode_ms = static_cast(decode_us) / 1000.0f; + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); +} + } // namespace transcribe diff --git a/src/transcribe-api-guard.h b/src/transcribe-api-guard.h new file mode 100644 index 000000000..5c1254351 --- /dev/null +++ b/src/transcribe-api-guard.h @@ -0,0 +1,57 @@ +// transcribe-api-guard.h - exception guards for the public C ABI entry +// points (bad_alloc -> OOM, other exceptions -> BACKEND, teardown never +// throws). MSVC builds use /EHs so exceptions thrown by ggml C-ABI frames +// remain catchable here. Not part of the public API. + +#pragma once + +#include "transcribe-log.h" +#include "transcribe.h" + +#include +#include + +namespace transcribe { + +template transcribe_status api_guard_status(const char * fn_name, Fn && body) { + try { + return body(); + } catch (const std::bad_alloc &) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: out of memory", fn_name); + return TRANSCRIBE_ERR_OOM; + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); + return TRANSCRIBE_ERR_BACKEND; + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); + return TRANSCRIBE_ERR_BACKEND; + } +} + +// Value-returning variant: `on_throw` is the entry point's documented +// hard-failure value (0 devices, false, INT_MIN, ...). +template T api_guard_value(const char * fn_name, T on_throw, Fn && body) { + try { + return body(); + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); + return on_throw; + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); + return on_throw; + } +} + +// Teardown variant: log and swallow. Frees must never fail. +template void api_guard_void(const char * fn_name, Fn && body) { + try { + body(); + } catch (const std::exception & e) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught backend exception during teardown: %s", fn_name, + e.what()); + } catch (...) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught unknown backend exception during teardown", fn_name); + } +} + +} // namespace transcribe diff --git a/src/transcribe-arch.cpp b/src/transcribe-arch.cpp index fb768cbac..0840408b4 100644 --- a/src/transcribe-arch.cpp +++ b/src/transcribe-arch.cpp @@ -6,6 +6,9 @@ #include "transcribe-arch.h" +#include "transcribe-log.h" +#include "transcribe-model.h" + #include namespace transcribe { @@ -112,4 +115,36 @@ const Arch * find_arch(const char * name) { return nullptr; } +transcribe_status resolve_roles(transcribe_model * model) { + if (model == nullptr || model->arch == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + const Arch & arch = *model->arch; + const char * name = arch.name != nullptr ? arch.name : "(unknown)"; + const bool has_asr = arch.init_context != nullptr && arch.run != nullptr; + const bool has_diarize = arch.diarize != nullptr; + + if (model->roles == 0 && has_asr) { + model->roles = TRANSCRIBE_ROLE_ASR; + } + + const uint32_t known = TRANSCRIBE_ROLE_ASR | TRANSCRIBE_ROLE_DIARIZE; + const char * why = nullptr; + if (model->roles == 0) { + why = "serves no role"; + } else if ((model->roles & ~known) != 0) { + why = "sets an unknown role bit"; + } else if ((model->roles & TRANSCRIBE_ROLE_ASR) != 0 && !has_asr) { + why = "sets the ASR role without init_context / run hooks"; + } else if ((model->roles & TRANSCRIBE_ROLE_DIARIZE) != 0 && !has_diarize) { + why = "sets the DIARIZE role without a diarize ops table"; + } + if (why != nullptr) { + log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: arch '%s' %s (roles 0x%x)", name, why, + static_cast(model->roles)); + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + return TRANSCRIBE_OK; +} + } // namespace transcribe diff --git a/src/transcribe-arch.h b/src/transcribe-arch.h index c88d63801..6a6e0e0a0 100644 --- a/src/transcribe-arch.h +++ b/src/transcribe-arch.h @@ -14,6 +14,7 @@ namespace transcribe { class Loader; +struct DiarizeOps; // Per-family trait. Function pointers may be null when an entry point is // not yet implemented; the central dispatch converts null entries into @@ -129,10 +130,21 @@ struct Arch { // out of the guarantee). NULL is skipped; the generic header-size + kind // checks remain in force. transcribe_status (*run_validate)(const struct transcribe_session * ctx, const transcribe_run_params * params); + + // Ops tables for the non-ASR roles; nullptr = role not implemented. + const DiarizeOps * diarize; }; // Look up an architecture by name. Returns nullptr if no registered // family matches. const Arch * find_arch(const char * name); +// Called once on a freshly loaded model, before it reaches the caller. +// Defaults roles to ASR when the family left it 0 and the arch has ASR hooks, +// then checks that the mask is non-zero, has no unknown bits, and that every +// set role is backed by the arch (init_context + run for ASR, a non-null ops +// table otherwise). A violation is a family wiring bug: it is logged and the +// load fails with TRANSCRIBE_ERR_NOT_IMPLEMENTED. +transcribe_status resolve_roles(struct transcribe_model * model); + } // namespace transcribe diff --git a/src/transcribe-asr.cpp b/src/transcribe-asr.cpp new file mode 100644 index 000000000..ffbfa0c2a --- /dev/null +++ b/src/transcribe-asr.cpp @@ -0,0 +1,2390 @@ +// transcribe-asr.cpp - ASR public C API entry points + central dispatch. +// +// What lives here: +// - Session lifecycle (transcribe_session_init / _free, transcribe_open / +// _close) and the run, batch and streaming dispatchers that validate +// params and hand off to the per-family Arch hooks. +// - The stream text policy (committed / tentative text view). +// - ASR capabilities, session limits, tokenize, timings, and the result +// and batch-result accessors. +// - The init functions for the ASR params / output structs. +// +// What does NOT live here: library-level entry points (status, version, +// logging, backends, devices, model load / free and model queries; +// transcribe.cpp) and per-family load / context / run code +// (src/arch//...). The accessors read the base structs in +// transcribe-model.h / transcribe-session.h directly. + +#include "transcribe-abi.h" +#include "transcribe-api-guard.h" +#include "transcribe-arch.h" +#include "transcribe-batch-util.h" +#include "transcribe-log.h" +#include "transcribe-model.h" +#include "transcribe-prompting.h" +#include "transcribe-session.h" +#include "transcribe-tokenizer.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using transcribe::api_guard_status; +using transcribe::api_guard_value; +using transcribe::api_guard_void; +using transcribe::enum_field_raw; +using transcribe::pcm_is_finite; + +namespace { + +// Ordered rank for timestamp granularities (NONE < SEGMENT < WORD < TOKEN), +// used to compare a request against a family's advertised maximum. AUTO and +// any unknown value get rank -1. +int timestamp_rank(transcribe_timestamp_kind k) { + switch (k) { + case TRANSCRIBE_TIMESTAMPS_NONE: + return 0; + case TRANSCRIBE_TIMESTAMPS_SEGMENT: + return 1; + case TRANSCRIBE_TIMESTAMPS_WORD: + return 2; + case TRANSCRIBE_TIMESTAMPS_TOKEN: + return 3; + case TRANSCRIBE_TIMESTAMPS_AUTO: + return -1; + } + return -1; +} + +// Shared run-params validation used by transcribe_run and +// transcribe_stream_begin. Both call sites have already validated +// that session, session->model, and params are non-null. The translate-task +// rejection differs between the two (run mirrors supports_translate, +// streaming-begin rejects unconditionally in v1), so each caller +// applies its own translate check before reaching this helper. +transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params); + +transcribe_status validate_run_params_common(const transcribe_session * session, const transcribe_run_params * params) { + // Raw-validate every enum field before its first enum-typed load (see + // enum_field_raw). Once a field passes here, downstream typed reads — + // including the per-family handlers' — are defined. + switch (enum_field_raw(¶ms->task)) { + case TRANSCRIBE_TASK_TRANSCRIBE: + case TRANSCRIBE_TASK_TRANSLATE: + case TRANSCRIBE_TASK_INSTRUCT: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->timestamps)) { + case TRANSCRIBE_TIMESTAMPS_NONE: + case TRANSCRIBE_TIMESTAMPS_AUTO: + case TRANSCRIBE_TIMESTAMPS_SEGMENT: + case TRANSCRIBE_TIMESTAMPS_WORD: + case TRANSCRIBE_TIMESTAMPS_TOKEN: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->pnc)) { + case TRANSCRIBE_PNC_MODE_DEFAULT: + case TRANSCRIBE_PNC_MODE_OFF: + case TRANSCRIBE_PNC_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->itn)) { + case TRANSCRIBE_ITN_MODE_DEFAULT: + case TRANSCRIBE_ITN_MODE_OFF: + case TRANSCRIBE_ITN_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + switch (enum_field_raw(¶ms->diarize)) { + case TRANSCRIBE_DIARIZE_MODE_DEFAULT: + case TRANSCRIBE_DIARIZE_MODE_OFF: + case TRANSCRIBE_DIARIZE_MODE_ON: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { + const int req_rank = timestamp_rank(params->timestamps); + const int max_rank = timestamp_rank(session->model->caps.max_timestamp_kind); + if (req_rank > max_rank) { + return TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS; + } + } + if (params->language != nullptr && session->model->caps.n_languages > 0 && + session->model->caps.languages != nullptr) { + bool found = false; + for (int i = 0; i < session->model->caps.n_languages; ++i) { + const char * entry = session->model->caps.languages[i]; + if (entry != nullptr && std::strcmp(entry, params->language) == 0) { + found = true; + break; + } + } + if (!found) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + } + // Translation-target gate: for a TRANSLATE request naming a target + // language, reject up front when the model advertises a target set + // (n > 0) that does not include it. Mirrors the source-language check + // above and returns the same code. An empty/unadvertised set (old + // GGUFs, ASR-only models) makes this inert; family-level target/pair + // checks (e.g. canary's pivot pairs) still apply on top. + if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->target_language != nullptr && + session->model->caps.n_translate_target_languages > 0 && + session->model->caps.translate_target_languages != nullptr) { + bool found = false; + for (int i = 0; i < session->model->caps.n_translate_target_languages; ++i) { + const char * entry = session->model->caps.translate_target_languages[i]; + if (entry != nullptr && std::strcmp(entry, params->target_language) == 0) { + found = true; + break; + } + } + if (!found) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + } + // Translation-pair gate: when a GGUF advertises exact src>dst pairs and + // the caller supplied both sides, reject unsupported directions before + // family dispatch. A missing source hint keeps this inert because most + // families do not perform generic language detection here. + if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->language != nullptr && + params->target_language != nullptr && + !session->model->allows_translation_pair(params->language, params->target_language)) { + return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; + } + return validate_prompting(session->model, params); +} + +// Shape and hard-gate checks for the generic prompting fields, on a +// normalized view. Soft inputs a model ignores are removed later by +// prepare_prompting. +transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params) { + auto reject = [](transcribe_status st, const char * why) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_run: %s", why); + return st; + }; + if (params->n_vocabulary < 0 || (params->n_vocabulary > 0 && params->vocabulary == nullptr)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary is NULL or n_vocabulary is negative"); + } + for (int32_t i = 0; i < params->n_vocabulary; ++i) { + if (params->vocabulary[i] == nullptr) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary has a NULL entry"); + } + } + const bool has_prefix = transcribe::prompting::has_text(params->prefix); + if (params->task == TRANSCRIBE_TASK_INSTRUCT) { + if (!transcribe::has_feature(model, TRANSCRIBE_FEATURE_INSTRUCT)) { + return reject(TRANSCRIBE_ERR_UNSUPPORTED_TASK, + "this model does not support TRANSCRIBE_TASK_INSTRUCT (TRANSCRIBE_FEATURE_INSTRUCT)"); + } + // Output is free text: no target language and no alignment. A prefix + // as answer prefill is untested on every INSTRUCT family. + if (!transcribe::prompting::has_text(params->prompt)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT requires a non-empty prompt"); + } + if (params->target_language != nullptr) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT does not take a target_language"); + } + if (params->timestamps != TRANSCRIBE_TIMESTAMPS_NONE && params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT supports timestamps NONE or AUTO only"); + } + if (has_prefix) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, + "a transcript prefix is not supported with TRANSCRIBE_TASK_INSTRUCT"); + } + } + // Ignoring a prefix would make the output repeat the prefix's words and + // silently break callers that stitch text together, so it is a hard gate. + if (has_prefix && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)) { + return reject(TRANSCRIBE_ERR_INVALID_ARG, + "this model does not support a transcript prefix (TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)"); + } + return TRANSCRIBE_OK; +} + +// Rejects control-token literals in the prompting text before the result +// snapshot is cleared. Runs after strip_ignored_prompting, so ignored inputs +// are not rejected. +transcribe_status check_prompting_text(const transcribe_model * model, const transcribe_run_params * params) { + const transcribe::Tokenizer * tok = model->tokenizer(); + if (tok == nullptr) { + return TRANSCRIBE_OK; + } + for (int32_t i = 0; i < params->n_vocabulary; ++i) { + if (const transcribe_status st = + transcribe::prompting::check_plain_text(*tok, params->vocabulary[i], "vocabulary"); + st != TRANSCRIBE_OK) { + return st; + } + } + if (params->prompt != nullptr) { + if (const transcribe_status st = transcribe::prompting::check_plain_text(*tok, params->prompt, "prompt"); + st != TRANSCRIBE_OK) { + return st; + } + } + if (params->prefix != nullptr) { + return transcribe::prompting::check_plain_text(*tok, params->prefix, "prefix"); + } + return TRANSCRIBE_OK; +} + +// Full-size copy of a caller's run params: defaults first, then only the +// prefix the caller's struct_size covers, so every trailing field is +// readable (NULL/0 for an older caller). struct_size is preserved so +// has_field() gating still sees the caller's true layout. Idempotent. +void normalize_run_params(const transcribe_run_params * in, transcribe_run_params * out) { + transcribe_run_params_init(out); + std::memcpy(out, in, static_cast(std::min(in->struct_size, sizeof(*out)))); +} + +// Warn about, then remove, the soft prompting inputs this model ignores, so +// a family only ever sees inputs it should act on. Idempotent: the batch +// serial fallback re-enters run_one_inner per utterance. +void strip_ignored_prompting(const transcribe_model * model, transcribe_run_params * params) { + const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; + const bool instruct = params->task == TRANSCRIBE_TASK_INSTRUCT; + if (params->n_vocabulary > 0 && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_VOCABULARY)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "transcribe_run: model '%s' does not support vocabulary; ignoring %d term(s). Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_VOCABULARY) to pre-check.", + arch_name, params->n_vocabulary); + params->vocabulary = nullptr; + params->n_vocabulary = 0; + } + if (!instruct && transcribe::prompting::has_text(params->prompt) && + !transcribe::has_feature(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, + "transcribe_run: model '%s' has no context-prompt slot; ignoring prompt. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT) to pre-check, or " + "TRANSCRIBE_TASK_INSTRUCT on models with TRANSCRIBE_FEATURE_INSTRUCT.", + arch_name); + params->prompt = nullptr; + } + if (!transcribe::prompting::has_text(params->prompt)) { + params->prompt = nullptr; + } + if (!transcribe::prompting::has_text(params->prefix)) { + params->prefix = nullptr; + } +} + +// The pre-clear prompting step every entry point shares, on a validated +// normalized view. +transcribe_status prepare_prompting(const transcribe_model * model, transcribe_run_params * params) { + strip_ignored_prompting(model, params); + return check_prompting_text(model, params); +} + +} // namespace + +// Params init functions +// +// See transcribe.cpp for the zero-fill + struct_size convention. + +extern "C" void transcribe_session_params_init(struct transcribe_session_params * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { + if (p == nullptr) { + return; + } + std::memset(p, 0, sizeof(*p)); + p->struct_size = sizeof(*p); + p->spec_k_drafts = -1; // family default + // Default to AUTO (richest output compatible with the model and selected + // run tasks, resolved per-family) rather than the memset NONE. + p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; +} + +extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { + transcribe::init_sized(p); +} + +// Output struct init functions + +extern "C" void transcribe_capabilities_init(struct transcribe_capabilities * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_session_limits_init(struct transcribe_session_limits * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_stream_update_init(struct transcribe_stream_update * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_stream_text_init(struct transcribe_stream_text * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_segment_init(struct transcribe_segment * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_word_init(struct transcribe_word * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_token_init(struct transcribe_token * p) { + transcribe::init_sized(p); +} + +// Whisper telemetry + run-extension init and chunk-trace accessors live in +// arch/whisper/public.cpp so this dispatcher stays family-agnostic. Whisper +// run knobs are in transcribe_whisper_run_ext (via run_params::family); +// sensevoice/funasr_nano use_itn and canary pnc use the generic run_params +// itn/pnc enums. + +namespace { + +constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); +// run_params is the one 0.2.0 exception to the append-only rule: `diarize` +// was inserted mid-struct, shifting every field from `language` on by 8 +// bytes. A 0.1-layout caller's sizeof (64) equals FIELD_END(family) in the +// NEW layout, so a `family`-prefix minimum would admit stale callers and +// read their pointers as enums. Require through spec_k_drafts so every +// pre-0.2 caller that bypasses the SONAME/abihash checks (dlopen by path, +// stale static link, hand-rolled FFI) gets BAD_STRUCT_SIZE instead. +constexpr size_t k_min_run_params_size = TRANSCRIBE_FIELD_END(transcribe_run_params, spec_k_drafts); +constexpr size_t k_min_stream_params_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, family); +constexpr size_t k_stream_params_commit_policy_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, commit_policy); +constexpr size_t k_stream_params_agreement_n_size = + TRANSCRIBE_FIELD_END(transcribe_stream_params, stable_prefix_agreement_n); +constexpr size_t k_min_stream_update_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, buffered_ms); +constexpr size_t k_stream_update_committed_changed_size = + TRANSCRIBE_FIELD_END(transcribe_stream_update, committed_changed); +constexpr size_t k_stream_update_tentative_changed_size = + TRANSCRIBE_FIELD_END(transcribe_stream_update, tentative_changed); +constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); +constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); +constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); +constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); +constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); +constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); +constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); +// k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with +// the chunk-trace accessor that uses it. + +using transcribe::check_input_struct_size; +using transcribe::check_struct_size; +using transcribe::copy_out_prefix; + +static bool has_field(uint64_t struct_size, size_t field_end) { + return struct_size >= static_cast(field_end); +} + +// Takes the RAW integer, not the enum: a C caller can store any int in the +// struct's enum-typed field, and in C++ loading an out-of-range value through +// the enum lvalue is UB (UBSan trips on it). Callers memcpy the bytes into an +// int and validate before the first enum-typed load. +static bool valid_stream_commit_policy(int policy) { + switch (policy) { + case TRANSCRIBE_STREAM_COMMIT_AUTO: + case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: + case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: + return true; + } + return false; +} + +} // namespace + +namespace { + +enum class StreamStablePrefixImpl { + GenericTextAgreement, + FamilyTokenAgreement, + FamilyNativeCommit, +}; + +static StreamStablePrefixImpl stream_stable_prefix_impl_for_arch(const transcribe::Arch * arch) { + // Single source of truth for the stable-prefix implementation used by + // AUTO and explicit STABLE_PREFIX. Add new streaming families here + // deliberately; otherwise they use the generic text-agreement fallback. + struct Entry { + const char * arch_name; + StreamStablePrefixImpl impl; + }; + + static constexpr Entry k_defaults[] = { + // Parakeet publishes native committed chunks; agreement_n does not + // add useful evidence on top of the family-provided boundary. + { "parakeet", StreamStablePrefixImpl::FamilyNativeCommit }, + // Moonshine re-decodes the full prefix and its family boundary is + // token-id agreement; the family applies stable_prefix_agreement_n. + { "moonshine_streaming", StreamStablePrefixImpl::FamilyTokenAgreement }, + }; + + const char * name = arch != nullptr && arch->name != nullptr ? arch->name : ""; + for (const Entry & entry : k_defaults) { + if (std::strcmp(name, entry.arch_name) == 0) { + return entry.impl; + } + } + return StreamStablePrefixImpl::GenericTextAgreement; +} + +static size_t utf8_floor_boundary(const std::string & s, size_t pos) { + if (pos >= s.size()) { + return s.size(); + } + while (pos > 0 && (static_cast(s[pos]) & 0xC0u) == 0x80u) { + --pos; + } + return pos; +} + +static size_t common_prefix_bytes(const std::string & a, const std::string & b) { + const size_t n = std::min(a.size(), b.size()); + size_t i = 0; + while (i < n && a[i] == b[i]) { + ++i; + } + return utf8_floor_boundary(a, i); +} + +static bool starts_with_bytes(const std::string & s, const std::string & prefix) { + return s.size() >= prefix.size() && s.compare(0, prefix.size(), prefix) == 0; +} + +static size_t token_prefix_raw_bytes(const transcribe_session * session, int n_tokens) { + if (session == nullptr || n_tokens <= 0 || session->tokens.empty()) { + return 0; + } + const int capped = std::min(n_tokens, static_cast(session->tokens.size())); + std::string prefix; + prefix.reserve(session->full_text.size()); + for (int i = 0; i < capped; ++i) { + prefix += session->tokens[static_cast(i)].text; + } + if (starts_with_bytes(session->full_text, prefix)) { + return prefix.size(); + } + if (!prefix.empty() && prefix.front() == ' ' && (session->full_text.empty() || session->full_text.front() != ' ')) { + const std::string normalized_prefix = prefix.substr(1); + if (starts_with_bytes(session->full_text, normalized_prefix)) { + return normalized_prefix.size(); + } + return common_prefix_bytes(session->full_text, normalized_prefix); + } + return common_prefix_bytes(session->full_text, prefix); +} + +static size_t family_candidate_raw_prefix_bytes(const transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + if (session->n_committed_tokens > 0) { + return token_prefix_raw_bytes(session, session->n_committed_tokens); + } + if (session->n_committed_words > 0 && session->n_committed_words >= static_cast(session->words.size())) { + return session->full_text.size(); + } + if (session->n_committed_segments > 0 && + session->n_committed_segments >= static_cast(session->segments.size())) { + return session->full_text.size(); + } + return 0; +} + +static size_t generic_text_stable_prefix_candidate_raw_bytes(transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + // Library default when the caller leaves stable_prefix_agreement_n at + // 0; documented as "currently 3" on transcribe_stream_params. + static constexpr uint32_t k_default_stable_prefix_agreement_n = 3; + uint32_t agreement_n = session->stream_stable_prefix_agreement_n; + if (agreement_n == 0) { + agreement_n = k_default_stable_prefix_agreement_n; + } + if (agreement_n <= 1) { + return session->full_text.size(); + } + + session->stream_raw_history.push_back(session->full_text); + while (session->stream_raw_history.size() > agreement_n) { + session->stream_raw_history.pop_front(); + } + if (session->stream_raw_history.size() < agreement_n) { + return 0; + } + + size_t prefix_n = session->stream_raw_history.front().size(); + for (const auto & text : session->stream_raw_history) { + prefix_n = std::min(prefix_n, common_prefix_bytes(session->stream_raw_history.front(), text)); + } + return utf8_floor_boundary(session->full_text, prefix_n); +} + +static size_t selected_stable_prefix_candidate_raw_bytes(transcribe_session * session) { + if (session == nullptr) { + return 0; + } + const transcribe::Arch * arch = session->model != nullptr ? session->model->arch : nullptr; + switch (stream_stable_prefix_impl_for_arch(arch)) { + case StreamStablePrefixImpl::FamilyTokenAgreement: + case StreamStablePrefixImpl::FamilyNativeCommit: + return family_candidate_raw_prefix_bytes(session); + case StreamStablePrefixImpl::GenericTextAgreement: + return generic_text_stable_prefix_candidate_raw_bytes(session); + } + return 0; +} + +static bool append_committed_raw_prefix(transcribe_session * session, size_t candidate_bytes) { + if (session == nullptr || !session->has_result) { + return false; + } + const std::string & raw = session->full_text; + const size_t old_boundary = utf8_floor_boundary( + raw, std::min(static_cast(session->stream_raw_tentative_start_bytes), raw.size())); + const size_t candidate = utf8_floor_boundary(raw, std::min(candidate_bytes, raw.size())); + if (candidate <= old_boundary) { + return false; + } + if (old_boundary != session->stream_committed_text.size()) { + return false; + } + if (raw.compare(0, old_boundary, session->stream_committed_text.data(), old_boundary) != 0) { + return false; + } + session->stream_committed_text.append(raw.data() + old_boundary, candidate - old_boundary); + session->stream_raw_tentative_start_bytes = static_cast(candidate); + return true; +} + +static bool finalize_committed_text(transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return false; + } + if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE || + session->stream_committed_text.empty()) { + const bool changed = session->stream_committed_text != session->full_text; + session->stream_committed_text = session->full_text; + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return changed; + } + if (!starts_with_bytes(session->full_text, session->stream_committed_text)) { + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return false; + } + const size_t old_n = session->stream_committed_text.size(); + if (session->full_text.size() > old_n) { + session->stream_committed_text.append(session->full_text.data() + old_n, session->full_text.size() - old_n); + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return true; + } + session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); + return false; +} + +struct StreamTextDelta { + bool committed_changed = false; + bool tentative_changed = false; +}; + +static StreamTextDelta apply_stream_text_policy(transcribe_session * session, bool is_finalize) { + StreamTextDelta delta; + if (session == nullptr) { + return delta; + } + const std::string prev_committed = session->stream_committed_text; + const std::string prev_tentative = session->stream_tentative_text; + + if (is_finalize) { + delta.committed_changed = finalize_committed_text(session); + session->stream_tentative_text.clear(); + } else if (session->has_result) { + size_t candidate = 0; + switch (session->stream_commit_policy) { + case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: + candidate = 0; + break; + case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: + case TRANSCRIBE_STREAM_COMMIT_AUTO: + default: + candidate = selected_stable_prefix_candidate_raw_bytes(session); + break; + } + delta.committed_changed = append_committed_raw_prefix(session, candidate); + + const size_t boundary = utf8_floor_boundary( + session->full_text, std::min(static_cast(session->stream_raw_tentative_start_bytes), + session->full_text.size())); + session->stream_tentative_text.assign(session->full_text.data() + boundary, + session->full_text.size() - boundary); + } else { + session->stream_tentative_text.clear(); + } + + delta.committed_changed = delta.committed_changed || session->stream_committed_text != prev_committed; + delta.tentative_changed = session->stream_tentative_text != prev_tentative; + return delta; +} + +static void publish_stream_update_tail(transcribe_stream_update * update, + bool committed_changed, + bool tentative_changed) { + if (update == nullptr) { + return; + } + if (has_field(update->struct_size, k_stream_update_committed_changed_size)) { + update->committed_changed = committed_changed; + } + if (has_field(update->struct_size, k_stream_update_tentative_changed_size)) { + update->tentative_changed = tentative_changed; + } +} + +// Compute the observable-change verdict, advance the revision counter +// when the result moved, and publish revision / result_changed / +// committed_changed / tentative_changed onto the caller's update. Shared +// verbatim by feed and finalize so the result_changed semantics live in +// exactly one place; the path-specific work (ON_FINALIZE counter zeroing +// and audio_committed_ms on feed, is_final and the lifecycle transition +// on finalize) stays in the callers. +static void publish_observable_delta(transcribe_session * session, + transcribe_stream_update * update, + int32_t prev_revision, + const std::string & prev_full_text, + bool prev_has_result, + const StreamTextDelta & text_delta) { + const bool raw_changed = session->has_result != prev_has_result || session->full_text != prev_full_text; + const bool observable_changed = raw_changed || text_delta.committed_changed || text_delta.tentative_changed || + (update != nullptr && update->result_changed); + if (observable_changed && session->stream_revision == prev_revision) { + session->stream_revision += 1; + } + if (update != nullptr) { + update->result_changed = + update->result_changed || observable_changed || session->stream_revision != prev_revision; + update->revision = session->stream_revision; + publish_stream_update_tail(update, text_delta.committed_changed, text_delta.tentative_changed); + } +} + +// Advisory pnc/itn warning. Emits a WARN when a non-DEFAULT request hits a +// model that does not support runtime control of that axis, then returns so +// the dispatcher proceeds with best-effort semantics. The reserved +// TRANSCRIBE_ERR_UNSUPPORTED_PNC / _ITN codes are NOT returned today +// (placeholders for a future opt-in strict mode). The message includes the +// arch + variant strings to pinpoint which model dropped the request. +void warn_unsupported_advisory(const struct transcribe_model * model, const struct transcribe_run_params * rp) { + if (model == nullptr || rp == nullptr) { + return; + } + const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; + const char * variant = model->variant.c_str(); + if (variant == nullptr || variant[0] == '\0') { + variant = "(unknown)"; + } + + // Defense in depth: keep raw reads here even though every dispatcher + // validates before warning. A future call-site reorder must not turn a + // malformed C enum into a typed C++ load. + const int pnc_raw = enum_field_raw(&rp->pnc); + const int itn_raw = enum_field_raw(&rp->itn); + const int diarize_raw = enum_field_raw(&rp->diarize); + + char buf[512]; + if (pnc_raw != TRANSCRIBE_PNC_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_PNC)) { + const char * req = (pnc_raw == TRANSCRIBE_PNC_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested pnc=%s but model '%s' " + "(variant '%s') does not support pnc control; output will use " + "the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_PNC) to " + "pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } + if (itn_raw != TRANSCRIBE_ITN_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_ITN)) { + const char * req = (itn_raw == TRANSCRIBE_ITN_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested itn=%s but model '%s' " + "(variant '%s') does not support itn control; output will use " + "the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_ITN) to " + "pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } + if (diarize_raw != TRANSCRIBE_DIARIZE_MODE_DEFAULT && + !transcribe::has_feature(model, TRANSCRIBE_FEATURE_DIARIZATION)) { + const char * req = (diarize_raw == TRANSCRIBE_DIARIZE_MODE_ON) ? "ON" : "OFF"; + std::snprintf(buf, sizeof(buf), + "transcribe_run: caller requested diarize=%s but model '%s' " + "(variant '%s') does not support diarization control; output " + "will use the model's default behavior. Use " + "transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION) " + "to pre-check.", + req, arch_name, variant); + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s", buf); + } +} + +} // namespace + +// Lifecycle + +static transcribe_status transcribe_session_init_impl(struct transcribe_model * model, + const struct transcribe_session_params * params, + struct transcribe_session ** out_session) { + if (out_session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out_session = nullptr; + if (model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if ((model->roles & TRANSCRIBE_ROLE_ASR) == 0) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + // NULL params means "all defaults" (see transcribe_model_load_file). + struct transcribe_session_params params_defaults; + transcribe_session_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_context_params_size); st != TRANSCRIBE_OK) { + return st; + } + // n_threads contract from include/transcribe.h: 0 means "library + // picks a sensible default", positive means "use this many." A + // negative value is undefined input, not a documented sentinel — + // reject it here so individual family handlers don't have to + // re-check the same condition. + if (params->n_threads < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Raw-validate kv_type before the families' first enum-typed load of it + // (see enum_field_raw): an unknown value from a C caller is a clean + // error here, never UB downstream. + switch (enum_field_raw(¶ms->kv_type)) { + case TRANSCRIBE_KV_TYPE_AUTO: + case TRANSCRIBE_KV_TYPE_F32: + case TRANSCRIBE_KV_TYPE_F16: + break; + default: + return TRANSCRIBE_ERR_INVALID_ARG; + } + // n_ctx is a trailing field (appended after kv_type), so only read it + // when the caller's struct_size actually covers it; an older caller's + // smaller struct leaves it at the default 0 = "model max". A negative + // value is undefined input, not a documented sentinel. The family that + // honors n_ctx clamps a too-large value down to the model maximum, so + // only the negative case is rejected here. See include/transcribe.h. + if (has_field(params->struct_size, offsetof(struct transcribe_session_params, n_ctx) + sizeof(params->n_ctx)) && + params->n_ctx < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // The model carries its own arch dispatch pointer. A model that + // came from a load() call always has arch set; the null check is + // defensive against a hypothetical malformed model object. + if (model->arch == nullptr || model->arch->init_context == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + return model->arch->init_context(model, params, out_session); +} + +static void transcribe_session_free_impl(struct transcribe_session * session) { + // Free the session and, if it owns its model (transcribe_open path), + // free the model too. The owns_model flag is set only by + // transcribe_open; sessions created via the two-step + // transcribe_session_init borrow the model and leave it alone here. + // + // Capture before the session is destroyed; reading session->model + // after delete would be use-after-free. + if (session == nullptr) { + return; + } + struct transcribe_model * owned = session->owns_model ? session->model : nullptr; + delete session; + transcribe_model_free(owned); // NULL is a no-op +} + +// Convenience: open / close / get_model +// +// transcribe_open bundles load + session_init and hands back a session that +// owns its model (owns_model = true). transcribe_close is a thin alias for +// transcribe_session_free; both honor owns_model and free the owned model +// after the session. + +static transcribe_status transcribe_open_impl(const char * path, + const struct transcribe_model_load_params * load_params, + const struct transcribe_session_params * session_params, + struct transcribe_session ** out_session) { + if (out_session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out_session = nullptr; + + // NULL params are forwarded as-is: the callees already treat NULL as + // "all defaults", so no substitution is needed here. + struct transcribe_model * model = nullptr; + if (const auto st = transcribe_model_load_file(path, load_params, &model); st != TRANSCRIBE_OK) { + return st; + } + + struct transcribe_session * session = nullptr; + if (const auto st = transcribe_session_init(model, session_params, &session); st != TRANSCRIBE_OK) { + // Load succeeded but session init failed: own the cleanup so the + // caller sees the same all-or-nothing contract as the two-step + // API. No partial state leaks. + transcribe_model_free(model); + return st; + } + + // The convenience session owns the model it loaded; either + // transcribe_session_free or transcribe_close (its alias) will free + // both. This is the only place owns_model is set. + session->owns_model = true; + *out_session = session; + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_close(struct transcribe_session * session) { + // Alias for transcribe_session_free, kept for source compatibility + // with the prior split-API shape. Both honor owns_model. + transcribe_session_free(session); +} + +extern "C" const struct transcribe_model * transcribe_get_model(const struct transcribe_session * session) { + if (session == nullptr) { + return nullptr; + } + return session->model; +} + +extern "C" void transcribe_set_abort_callback(struct transcribe_session * session, + transcribe_abort_callback cb, + void * user_data) { + if (session == nullptr) { + return; + } + session->abort_cb = cb; + session->abort_userdata = user_data; +} + +extern "C" bool transcribe_was_aborted(const struct transcribe_session * session) { + if (session == nullptr) { + return false; + } + return session->was_aborted; +} + +extern "C" bool transcribe_was_truncated(const struct transcribe_session * session) { + if (session == nullptr) { + return false; + } + return session->was_truncated; +} + +// Streaming dispatcher +// +// State transitions are managed entirely here; hooks see ACTIVE on entry to +// begin/feed/finalize and never observe transitions. Hooks own the +// per-utterance result data (tokens/words/segments, committed counts, audio +// cursors, stream_revision); the dispatcher owns lifecycle state, +// last_status, and the public committed/tentative text view. clear_result() +// wipes the snapshot but never touches stream_state. + +// A family stream hook that throws must still end the stream: mark it FAILED +// with the status the api_guard will report for the exception (OOM for +// bad_alloc, BACKEND otherwise), then let the exception reach the guard. +template static transcribe_status call_stream_hook(transcribe_session * session, Fn && hook) { + try { + return hook(); + } catch (const std::bad_alloc &) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = TRANSCRIBE_ERR_OOM; + throw; + } catch (...) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = TRANSCRIBE_ERR_BACKEND; + throw; + } +} + +static transcribe_status transcribe_stream_begin_impl(struct transcribe_session * session, + const struct transcribe_run_params * run_params, + const struct transcribe_stream_params * stream_params) { + if (session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // NULL run/stream params each mean "all defaults". + struct transcribe_run_params run_params_defaults; + transcribe_run_params_init(&run_params_defaults); + struct transcribe_stream_params stream_params_defaults; + transcribe_stream_params_init(&stream_params_defaults); + if (run_params == nullptr) { + run_params = &run_params_defaults; + } + if (stream_params == nullptr) { + stream_params = &stream_params_defaults; + } + if (const auto st = check_input_struct_size(run_params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + // Full-size view (see normalize_run_params); the family hook gets a + // further copy whose strings the library owns, built below. + struct transcribe_run_params run_params_view; + normalize_run_params(run_params, &run_params_view); + run_params = &run_params_view; + if (const auto st = check_input_struct_size(stream_params->struct_size, k_min_stream_params_size); + st != TRANSCRIBE_OK) { + return st; + } + // Read the caller's commit_policy as raw bytes and validate BEFORE the + // first load through the enum-typed lvalue: C callers can legally store + // any int there, and an out-of-range enum load is UB in C++. + int commit_policy_raw = TRANSCRIBE_STREAM_COMMIT_AUTO; + uint32_t stable_prefix_agreement_n = 0; + if (has_field(stream_params->struct_size, k_stream_params_commit_policy_size)) { + static_assert(sizeof(stream_params->commit_policy) == sizeof(int), + "commit_policy must be int-sized for the raw read"); + std::memcpy(&commit_policy_raw, &stream_params->commit_policy, sizeof(commit_policy_raw)); + } + if (!valid_stream_commit_policy(commit_policy_raw)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const transcribe_stream_commit_policy commit_policy = + static_cast(commit_policy_raw); + if (has_field(stream_params->struct_size, k_stream_params_agreement_n_size)) { + stable_prefix_agreement_n = stream_params->stable_prefix_agreement_n; + if (stable_prefix_agreement_n > 32) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + // Caller must finalize or reset before starting a new stream. + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->model == nullptr || session->model->arch == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + // Capability gate: the model must advertise streaming AND the + // family must wire the full required hook set. begin / feed / + // finalize come as a triple — a partially-wired family that + // accepts begin would otherwise let the caller enter ACTIVE and + // then get stuck on NOT_IMPLEMENTED at the first feed. stream_reset + // remains optional; the dispatcher's clear_result + state wipe is + // sufficient when a family does not need to release per-utterance + // buffers explicitly. + if (!session->model->caps.supports_streaming || session->model->arch->stream_begin == nullptr || + session->model->arch->stream_feed == nullptr || session->model->arch->stream_finalize == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + if (const transcribe_status st = validate_run_params_common(session, run_params); st != TRANSCRIBE_OK) { + return st; + } + + // v1 rejects TRANSLATE unconditionally for streaming. A future + // family that supports streaming translate would loosen this in + // its stream_begin hook, but the central dispatcher refuses + // upfront so partially-wired callers fail fast. + if (run_params->task == TRANSCRIBE_TASK_TRANSLATE || run_params->task == TRANSCRIBE_TASK_INSTRUCT) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + // A prefix is forced decoder text for one utterance's opening; a stream + // has no fixed opening to force it onto. + if (transcribe::prompting::has_text(run_params->prefix)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_stream_begin: a transcript prefix is not accepted " + "for streaming"); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + if (stream_params->family != nullptr) { + if (stream_params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_STREAM, + stream_params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + + // Advisory warn for pnc/itn requests against models that don't + // expose the corresponding runtime toggle. Emitted before + // clear_result so the pre-hook "snapshot preserved on rejection" + // contract is undisturbed. + warn_unsupported_advisory(session->model, run_params); + if (const transcribe_status st = prepare_prompting(session->model, &run_params_view); st != TRANSCRIBE_OK) { + return st; + } + + // Optional family preflight: validates extension field values + // (e.g. parakeet's (L, C, R) menu) without mutating state. On + // non-OK return the previous snapshot and lifecycle are preserved + // so a caller-side typo does not destroy the prior utterance. + if (session->model->arch->stream_validate != nullptr) { + if (const transcribe_status st = session->model->arch->stream_validate(session, run_params, stream_params); + st != TRANSCRIBE_OK) { + return st; + } + } + + // All checks pass — clear the previous snapshot and hand off. + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + session->stream_state = TRANSCRIBE_STREAM_ACTIVE; + session->stream_commit_policy = commit_policy; + session->stream_stable_prefix_agreement_n = stable_prefix_agreement_n; + + // Hand the family hook a params view whose pointers the LIBRARY owns. + // Families may capture `*run_params` for the stream's lifetime + // (parakeet re-reads .language on every feed), and the public contract + // lets the caller free every params pointer once begin returns — so the + // strings are copied into session storage here and the view repointed + // at it. `family` (the run-slot extension) is nulled in the view: no + // family reads it on the stream path today, and per the ext copy-out + // contract a retained pointer to it would dangle; a future family that + // wants a run-slot ext at stream begin must plumb it deliberately. + session->stream_language_owned = run_params->language != nullptr ? run_params->language : ""; + session->stream_target_language_owned = run_params->target_language != nullptr ? run_params->target_language : ""; + // Copied from the normalized view (never the caller's struct, whose + // allocation may end before the trailing fields); the view keeps the + // caller's struct_size, so has_field() gating still sees its layout. + struct transcribe_run_params run_params_owned = run_params_view; + run_params_owned.language = run_params->language != nullptr ? session->stream_language_owned.c_str() : nullptr; + run_params_owned.target_language = + run_params->target_language != nullptr ? session->stream_target_language_owned.c_str() : nullptr; + run_params_owned.family = nullptr; + // Generic prompting strings, same ownership rule. Only non-empty terms + // are kept, so the view's count matches the owned array. + session->stream_vocabulary_owned = transcribe::prompting::terms(run_params); + session->stream_vocabulary_ptrs.clear(); + for (const std::string & term : session->stream_vocabulary_owned) { + session->stream_vocabulary_ptrs.push_back(term.c_str()); + } + session->stream_prompt_owned = run_params->prompt != nullptr ? run_params->prompt : ""; + run_params_owned.vocabulary = + session->stream_vocabulary_ptrs.empty() ? nullptr : session->stream_vocabulary_ptrs.data(); + run_params_owned.n_vocabulary = static_cast(session->stream_vocabulary_ptrs.size()); + run_params_owned.prompt = run_params->prompt != nullptr ? session->stream_prompt_owned.c_str() : nullptr; + run_params_owned.prefix = nullptr; + + const transcribe_status st = call_stream_hook( + session, [&] { return session->model->arch->stream_begin(session, &run_params_owned, stream_params); }); + if (st != TRANSCRIBE_OK) { + // Family hook rejected the begin (config it does not understand, + // memory allocation failure, etc.). Roll lifecycle back to + // FAILED so the caller can read last_status; leave the cleared + // result snapshot in place. + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + } + return st; +} + +// Helper: validate the caller's stream_update struct_size and reset +// every field except struct_size to a clean state before handing the +// update buffer to the family hook. Returns OK when update is NULL +// (the update buffer is optional) or when the size passes validation. +static transcribe_status prepare_stream_update(struct transcribe_stream_update * update) { + if (update == nullptr) { + return TRANSCRIBE_OK; + } + if (const auto st = check_struct_size(update->struct_size, k_min_stream_update_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = update->struct_size; + transcribe_stream_update staged{}; + staged.struct_size = caller_size; + copy_out_prefix(update, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * session, + const float * pcm, + int n_samples, + struct transcribe_stream_update * update) { + if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { + return st; + } + if (session == nullptr || pcm == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Zero-length feed is rejected: feed exists to consume audio, + // and callers that just want to inspect state use the stream + // accessors. n_samples < 0 is plain garbage. + if (n_samples <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Rejected before the hook, so the stream stays ACTIVE and usable. + if (!pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // begin already confirmed model/arch/hook; we re-check defensively + // so a malformed context (never happens in practice from a real + // begin) does not deref a null function pointer. + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->stream_feed == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + const int32_t prev_revision = session->stream_revision; + const std::string prev_full_text = session->full_text; + const bool prev_has_result = session->has_result; + + const transcribe_status st = + call_stream_hook(session, [&] { return session->model->arch->stream_feed(session, pcm, n_samples, update); }); + if (st != TRANSCRIBE_OK) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + return st; + } + const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/false); + if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { + session->n_committed_tokens = 0; + session->n_committed_words = 0; + session->n_committed_segments = 0; + } + publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); + if (update != nullptr && session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { + update->audio_committed_ms = 0; + } + return st; +} + +static transcribe_status transcribe_stream_finalize_impl(struct transcribe_session * session, + struct transcribe_stream_update * update) { + if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { + return st; + } + if (session == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session->model == nullptr || session->model->arch == nullptr || + session->model->arch->stream_finalize == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + const int32_t prev_revision = session->stream_revision; + const std::string prev_full_text = session->full_text; + const bool prev_has_result = session->has_result; + + const transcribe_status st = + call_stream_hook(session, [&] { return session->model->arch->stream_finalize(session, update); }); + if (update != nullptr) { + // Force is_final true regardless of family hook return; the + // marker describes the call site, not the result. Set after + // the hook so the family cannot accidentally clear it. + update->is_final = true; + } + if (st != TRANSCRIBE_OK) { + session->stream_state = TRANSCRIBE_STREAM_FAILED; + session->stream_last_status = st; + return st; + } + const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/true); + publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); + session->stream_state = TRANSCRIBE_STREAM_FINISHED; + return TRANSCRIBE_OK; +} + +static void transcribe_stream_reset_impl(struct transcribe_session * session) { + if (session == nullptr) { + return; + } + // Family hook releases per-utterance state and clears any buffered + // audio contents while keeping the allocations. A family without + // streaming just has no hook installed; reset becomes a pure + // dispatcher state wipe. + // The wipe below runs even if the hook throws, so reset always ends + // IDLE; the exception still reaches the api_guard afterwards. + std::exception_ptr hook_error; + if (session->model != nullptr && session->model->arch != nullptr && session->model->arch->stream_reset != nullptr) { + try { + session->model->arch->stream_reset(session); + } catch (...) { + hook_error = std::current_exception(); + } + } + session->clear_result(); + session->stream_state = TRANSCRIBE_STREAM_IDLE; + // was_aborted is per-stream; reset re-arms it the same way begin + // does so a caller that resets after an abort starts clean. + session->was_aborted = false; + if (hook_error) { + std::rethrow_exception(hook_error); + } +} + +extern "C" enum transcribe_stream_state transcribe_stream_get_state(const struct transcribe_session * session) { + if (session == nullptr) { + return TRANSCRIBE_STREAM_IDLE; + } + return session->stream_state; +} + +extern "C" int transcribe_stream_revision(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->stream_revision; +} + +extern "C" int transcribe_stream_n_committed_segments(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_segments; +} + +extern "C" int transcribe_stream_n_committed_words(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_words; +} + +extern "C" int transcribe_stream_n_committed_tokens(const struct transcribe_session * session) { + if (session == nullptr) { + return 0; + } + return session->n_committed_tokens; +} + +extern "C" transcribe_status transcribe_stream_last_status(const struct transcribe_session * session) { + if (session == nullptr) { + return TRANSCRIBE_OK; + } + return session->stream_last_status; +} + +extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_session * session, + struct transcribe_stream_text * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_stream_text_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_stream_text staged{}; + staged.struct_size = caller_size; + if (session != nullptr && session->has_result) { + staged.full_text = session->full_text.c_str(); + staged.full_text_bytes = static_cast(session->full_text.size()); + staged.committed_text = session->stream_committed_text.c_str(); + staged.committed_text_bytes = static_cast(session->stream_committed_text.size()); + staged.tentative_text = session->stream_tentative_text.c_str(); + staged.tentative_text_bytes = static_cast(session->stream_tentative_text.size()); + staged.raw_tentative_start_bytes = + std::min(session->stream_raw_tentative_start_bytes, staged.full_text_bytes); + } else { + staged.full_text = ""; + staged.committed_text = ""; + staged.tentative_text = ""; + } + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +// Shared one-utterance run body. Does NOT touch session->batch_results, so +// the batch dispatcher can call it once per utterance inside a loop without +// erasing already-accumulated entries; the public transcribe_run wrapper +// below clears batch_results once before delegating here. Every early +// return preserves the previous result snapshot exactly as the original +// transcribe_run contract documented (see the inline comments). +// `committed` (optional) is set true once the call passes the pre-clear +// gates and commits to replacing the result. The caller uses it to decide +// whether compute scratch needs releasing. +static transcribe_status run_one_inner(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params, + bool * committed = nullptr) { + // Parameter-shape validation runs first and does not touch session + // state. A caller that passes NULL pointers or a non-positive sample + // count gets ERR_INVALID_ARG back without any visible side effect + // — including the session's result fields, which are preserved + // across a malformed call. This is the narrower half of the + // "transcribe_run replaces the previous result" contract: the + // replacement happens when the call is well-formed enough that + // the dispatcher is willing to dereference session. A call that + // isn't well-formed isn't considered to have "run". + if (session == nullptr || pcm == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // n_samples must be strictly positive, matching transcribe_stream_feed. + // There is no meaningful "transcribe zero samples" operation; a + // zero-length batch is treated as a caller error, not an empty run. + if (n_samples <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // NULL params means "all defaults" (transcribe vs translate, no + // timestamps, etc.). A well-formed default run is not a malformed + // call, so it proceeds and replaces the previous result. + struct transcribe_run_params params_defaults; + transcribe_run_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + // Everything downstream reads this full-size view, never the caller's + // struct: an older caller's allocation may end before the prompting + // fields. Strings stay caller-owned; the call is synchronous. + struct transcribe_run_params params_view; + normalize_run_params(params, ¶ms_view); + params = ¶ms_view; + // A run cannot replace an active stream's results — that would + // strand the in-flight stream's per-family state. Caller must + // finalize or reset first. FINISHED and FAILED both fall through; + // the run path below resets stream_state to IDLE. + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + + // Pre-clear family-extension SHAPE validation: a run ext with a bad + // transcribe_ext header size or a kind the model does not accept must + // NOT wipe the previous result snapshot (mirrors the + // transcribe_stream_begin contract). When model is null we skip the + // pre-clear check and let the post-clear NOT_IMPLEMENTED path handle it. + // + // This guarantee covers ext shape (size/kind) and the run-param checks + // below. A family that defers deeper value validation to run() (whisper + // does this for prompt semantics) can still reject a correctly-shaped + // ext after the snapshot is cleared; full pre-clear safety requires + // validating values in run_validate. + if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + + // Validate before advisory warning so typed enum reads are proven safe. + // A rejection preserves the previous snapshot. Skipped when model is + // null — the post-clear NOT_IMPLEMENTED path signals that directly. + if (session->model != nullptr) { + // Pre-clear run-param validation. A caller-side param bug + // (out-of-range enum, timestamp granularity finer than the + // model's max, unsupported language, or TRANSLATE against a + // model that doesn't support it) must NOT wipe the previous + // result snapshot — the same contract transcribe_stream_begin + // honors for its run params. These checks read model->caps and + // are therefore guarded by model != null; the degenerate + // model-null case falls through to the post-clear + // NOT_IMPLEMENTED path, which keeps its existing snapshot-wipe + // behavior. + if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { + return st; + } + warn_unsupported_advisory(session->model, params); + if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { + return st; + } + + // Family run-ext validation (the _RUN analogue of stream_validate), + // the final pre-clear gate. Runs AFTER the run-param checks above, + // mirroring transcribe_stream_begin's ordering (dispatcher param + // checks, then the family validate hook, then clear). The family + // alone knows its per-kind minimum struct size; whisper enforces + // that here. A non-OK return preserves the previous snapshot. A + // family with no _RUN ext leaves this hook NULL. Called regardless + // of whether params->family is set (whisper's hook treats a NULL + // family as "defaults", returning OK). + if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { + if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + // Result-replacement contract: everything past this point either + // succeeds and writes a fresh result, or fails and leaves the + // context in the documented "no result" sentinel state. Clear + // eagerly so the one remaining downstream rejection path — + // NOT_IMPLEMENTED on an incomplete arch — inherits the sentinel + // without having to remember to call clear_result() itself. + // Caller-param rejections (enum range, timestamp ceiling, + // language, TRANSLATE support) are validated ABOVE, before this + // clear, so a malformed call preserves the previous snapshot + // exactly as transcribe_stream_begin does. + // + // Per-family run() handlers call clear_result() again after + // their own front-matter checks succeed; that call is now + // redundant but idempotent, and removing it is a refactor + // deferred to a later pass. + if (committed != nullptr) { + *committed = true; + } + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + // Force stream_state to IDLE: clear_result deliberately preserves + // lifecycle state, but a well-formed transcribe_run subsumes any + // prior FINISHED/FAILED stream — after a one-shot run the context + // is no longer meaningfully in a streaming lifecycle. + session->stream_state = TRANSCRIBE_STREAM_IDLE; + + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + return session->model->arch->run(session, pcm, n_samples, params); +} + +static transcribe_status transcribe_run_impl(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params) { + // A well-formed single run has the same result view as a one-item batch: + // reset batch_results so the transcribe_batch_* accessors fall back to + // reading the scratch slot as utterance 0. + // + // Guard the deref with the SAME pre-deref conditions run_one_inner uses + // (non-null session + non-null pcm + positive n_samples). A malformed + // call must not touch the session at all — the API smoke test probes + // this with a fake (session *)0x1 and a NULL pcm / non-positive + // n_samples, expecting INVALID_ARG with no dereference. + // Non-finite PCM is malformed: reject before batch_results is touched. + if (pcm != nullptr && n_samples > 0 && !pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (session != nullptr && pcm != nullptr && n_samples > 0) { + session->batch_results.clear(); + } + // run_one_inner arms the guard at its commit point, so a pre-clear + // rejection never releases and everything after (family error, abort, + // throw) always does. + transcribe::ScratchReleaseGuard scratch_release; + scratch_release.session = session; + return run_one_inner(session, pcm, n_samples, params, &scratch_release.armed); +} + +// Batch run (offline) +// +// transcribe_run_batch validates the shared run_params ONCE, then either +// delegates to the family's batched run_batch() fast path or falls back to +// run_one_inner() per utterance. Either way the result is N entries in +// session->batch_results read back via the transcribe_batch_* accessors. + +namespace { + +// Pad batch_results out to `n` entries with explicit aborted failures. +// Called only on an aborted batch: synthesized slots carry +// TRANSCRIBE_ERR_ABORTED ("did not complete because the batch was aborted"), +// so the result-set view has one slot per input utterance +// (transcribe_batch_n_results() == n) regardless of family or path. +void pad_batch_results_aborted(transcribe_session * s, int n) { + while (s->batch_results.size() < static_cast(n)) { + transcribe_session::ResultSet rs; + rs.status = TRANSCRIBE_ERR_ABORTED; + s->batch_results.push_back(std::move(rs)); + } +} + +// Restore the scratch slot from a ResultSet so the single-result accessors +// stay coherent after a batch run (they reflect utterance 0). This is the +// inverse of transcribe_session::capture_result minus the per-run timing +// accumulators — keep the result fields in sync with that method when adding +// one (raw_text was once missed here). +void restore_scratch_from_result(transcribe_session * s, const transcribe_session::ResultSet & rs) { + s->tokens = rs.tokens; + s->words = rs.words; + s->segments = rs.segments; + s->speaker_segments = rs.speaker_segments; + s->full_text = rs.full_text; + s->raw_text = rs.raw_text; + s->detected_language = rs.detected_language; + s->result_kind = rs.result_kind; + s->has_result = rs.has_result; +} + +} // namespace + +static transcribe_status transcribe_run_batch_impl(struct transcribe_session * session, + const float * const * pcm, + const int * n_samples, + int n, + const struct transcribe_run_params * params) { + // Top-level argument shape. The arrays themselves must be present and + // n strictly positive; an individual malformed utterance (pcm[i] NULL + // or n_samples[i] <= 0) is a per-utterance failure handled in the loop, + // not a whole-batch error. As with transcribe_run, these checks do not + // mutate the session, so a malformed call preserves the prior result. + if (session == nullptr || pcm == nullptr || n_samples == nullptr || n <= 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // Non-finite samples in any utterance reject the whole batch here, before + // anything is cleared (unlike a NULL / empty utterance, which is a + // per-utterance failure): the families' batched hooks never see them. + for (int i = 0; i < n; ++i) { + if (pcm[i] != nullptr && n_samples[i] > 0 && !pcm_is_finite(pcm[i], n_samples[i])) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_run_batch: utterance %d has non-finite samples (NaN / Inf)", i); + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + struct transcribe_run_params params_defaults; + transcribe_run_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { + return st; + } + struct transcribe_run_params params_view; + normalize_run_params(params, ¶ms_view); + params = ¶ms_view; + if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + // One shared params across different audio: a transcript prefix can + // only describe one of them. + if (transcribe::prompting::has_text(params->prefix)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_run_batch: a transcript prefix is per-utterance " + "and is not accepted in a batch"); + return TRANSCRIBE_ERR_INVALID_ARG; + } + + // Shared-param validation, ONCE, mirroring transcribe_run's pre-clear + // gates (ext shape/kind, pnc/itn advisory, enum range, timestamp + // ceiling, language, TRANSLATE support, run_validate). A rejection here + // preserves the previous result snapshot — nothing has been cleared. + if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + } + if (session->model != nullptr) { + if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { + return st; + } + warn_unsupported_advisory(session->model, params); + if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { + return TRANSCRIBE_ERR_UNSUPPORTED_TASK; + } + if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { + return st; + } + if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { + if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; + } + + // Past this point we commit to producing a fresh batch result. + session->clear_result(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + session->was_aborted = false; + session->was_truncated = false; + session->stopped_on_repetition = false; + session->stream_state = TRANSCRIBE_STREAM_IDLE; + session->batch_results.clear(); + + // Release the compute scratch once the batch has run, whichever path it + // took (see transcribe_session::release_scratch). Once per call, not per + // utterance, so the serial fallback keeps its workspace across the loop. + transcribe::ScratchReleaseGuard scratch_release; + scratch_release.session = session; + scratch_release.armed = true; + + // Fast path: a family with a batched compute graph owns the whole loop. + if (session->model->arch->run_batch != nullptr) { + const transcribe_status st = session->model->arch->run_batch(session, pcm, n_samples, n, params); + // On abort the hook may retain only completed results; synthesize any + // missing slots so the result-set view always exposes n entries. + if (st == TRANSCRIBE_ERR_ABORTED) { + pad_batch_results_aborted(session, n); + } + // Keep the single accessors coherent with utterance 0. + if (!session->batch_results.empty()) { + restore_scratch_from_result(session, session->batch_results.front()); + } + return st; + } + + // Generic serial fallback: run each utterance in turn and snapshot it. + // Correct for every family; only the per-dispatch device throughput of + // a real run_batch() is forgone. run_one_inner re-validates the shared + // params (idempotent) before the family run(). + session->batch_results.reserve(static_cast(n)); + const transcribe_status batch_status = transcribe::run_batch_serial( + session, pcm, n_samples, n, [&](const float * p, int ns) { return run_one_inner(session, p, ns, params); }); + + // On abort the loop can break early, leaving fewer than n entries; + // synthesize any missing slots so the result-set view always exposes n + // entries (same invariant the fast path holds). + if (batch_status == TRANSCRIBE_ERR_ABORTED) { + pad_batch_results_aborted(session, n); + } + + // Restore the scratch slot to mirror utterance 0 for the single-result + // accessors (n >= 1 guarantees at least one entry, but be defensive). + if (!session->batch_results.empty()) { + restore_scratch_from_result(session, session->batch_results.front()); + } + return batch_status; +} + +// Model introspection +// +// These accessors read directly from the base struct; per-family load() +// fills caps / variant / backend before returning success. + +extern "C" transcribe_status transcribe_model_get_capabilities(const struct transcribe_model * model, + struct transcribe_capabilities * out_caps) { + if (model == nullptr || out_caps == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out_caps->struct_size, k_min_capabilities_size); st != TRANSCRIBE_OK) { + return st; + } + // These are ASR capabilities: a model without the ASR role gets an + // error, not a zeroed struct that reads as "supports nothing". + if ((model->roles & TRANSCRIBE_ROLE_ASR) == 0) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + // Preserve the caller-declared size, then write only the prefix + // that fits in both the caller's buffer and this library's view; + // any tail bytes stay as the caller initialized them (zero, by + // the init function's zero-fill contract). + // + // Pointer fields written into the caller buffer (e.g. languages) + // remain model-owned and valid until transcribe_model_free(). + const uint64_t caller_size = out_caps->struct_size; + transcribe_capabilities staged = model->caps; + staged.struct_size = caller_size; + copy_out_prefix(out_caps, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_session_get_limits(const struct transcribe_session * session, + struct transcribe_session_limits * out) { + if (session == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_session_limits_size); st != TRANSCRIBE_OK) { + return st; + } + + transcribe_session_limits staged; + transcribe_session_limits_init(&staged); + + const transcribe_model * model = session->model; + const auto & lb = model->limits; + + if (lb.has_context_cap && lb.model_max_ctx > 0) { + // effective_n_ctx = model max, lowered (never raised) by the session + // n_ctx cap. This mirrors the per-family context_ceiling helpers. + int32_t eff = lb.model_max_ctx; + if (session->n_ctx > 0 && session->n_ctx < eff) { + eff = session->n_ctx; + } + staged.effective_n_ctx = eff; + + // effective_max_audio_ms: for families whose audio tokens consume the + // decoder context, invert the input gate at the effective ceiling so + // the audio bound tracks n_ctx. For families whose audio bound is the + // encoder (audio_from_caps), the audio limit is independent of the + // decoder context, so report caps.max_audio_ms unchanged. Either way + // it is advisory (representative prompt), not an exact per-call bound. + if (lb.audio_from_caps) { + staged.effective_max_audio_ms = model->caps.max_audio_ms; + } else { + const int64_t audio_tokens = (int64_t) eff - lb.prompt_overhead - lb.gen_reserve; + staged.effective_max_audio_ms = (audio_tokens > 0 && lb.ms_per_audio_token > 0.0) ? + (int64_t) ((double) audio_tokens * lb.ms_per_audio_token) : + 0; + } + + // max_kv_bytes: worst-case single-utterance KV allocation at the + // effective ceiling, exact for the session's kv_type. The families + // resolve AUTO (and F16) to f16 for the KV cache and use f32 only for + // an explicit F32 request, so the byte size is 4/elem for F32 and + // 2/elem otherwise. This is the ceiling for one utterance, not the + // per-run allocation (the cache grows to fit input); transcribe_run_batch + // allocates roughly batch_size x this. + const int64_t kv_bytes_per_elem = (session->kv_type == TRANSCRIBE_KV_TYPE_F32) ? 4 : 2; + staged.max_kv_bytes = lb.kv_elems_per_ctx_token * (int64_t) eff * kv_bytes_per_elem; + } else { + // Unbounded / soft-window family: no decoder context cap. Report the + // soft window (if any) from caps.max_audio_ms, independent of n_ctx. + staged.effective_n_ctx = 0; + staged.effective_max_audio_ms = model->caps.max_audio_ms; + staged.max_kv_bytes = 0; + } + + const uint64_t caller_size = out->struct_size; + staged.struct_size = caller_size; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static int transcribe_tokenize_impl(const struct transcribe_model * model, + const char * text, + int32_t * tokens, + size_t n_max) { + if (model == nullptr || text == nullptr) { + return INT_MIN; + } + const transcribe::Tokenizer * tok = model->tokenizer(); + if (tok == nullptr) { + return INT_MIN; + } + + std::vector ids; + if (tok->encode(std::string(text), ids) != TRANSCRIBE_OK) { + return INT_MIN; + } + + const size_t n = ids.size(); + if (n > static_cast(INT_MAX)) { + return INT_MIN; + } + if (n > n_max) { + // Negative-of-N retry contract (mirrors whisper.cpp). Guard + // against -n overflowing int range for pathologically large + // vocabularies; fall back to INT_MIN + 1 so the caller still + // gets a hard-failure code rather than a stale pointer. + if (n > static_cast(INT_MAX)) { + return INT_MIN; + } + return -static_cast(n); + } + if (tokens == nullptr && n > 0) { + return INT_MIN; + } + for (size_t i = 0; i < n; ++i) { + tokens[i] = ids[i]; + } + return static_cast(n); +} + +// Timings +// +// Accumulators live on the base classes (transcribe_model::t_load_us and +// transcribe_session::t_{mel,encode,decode}_us), populated by per-family +// load() / run(). These accessors convert microseconds to milliseconds. + +extern "C" transcribe_status transcribe_get_timings(const struct transcribe_session * session, + struct transcribe_timings * out_timings) { + if (session == nullptr || out_timings == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out_timings->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + transcribe::copy_out_timings(session->model != nullptr ? session->model->t_load_us : 0, session->t_mel_us, + session->t_encode_us, session->t_decode_us, out_timings); + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_print_timings(const struct transcribe_session * session) { + if (session == nullptr) { + return; + } + struct transcribe_timings t; + transcribe_timings_init(&t); + (void) transcribe_get_timings(session, &t); + char buf[256]; + + std::snprintf(buf, sizeof(buf), + "timings: load=%.2f ms mel=%.2f ms " + "encode=%.2f ms decode=%.2f ms", + t.load_ms, t.mel_ms, t.encode_ms, t.decode_ms); + // Surfaced even in the common dev case (transcribe-cli without an + // installed sink) via the stderr fallback; an explicit + // transcribe_log_set(NULL, ...) silences it like everything else. + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_INFO, "%s", buf); +} + +extern "C" void transcribe_reset_timings(struct transcribe_session * session) { + if (session == nullptr) { + return; + } + // load_us is intentionally left alone — it's a model-scoped fact, + // not a per-call accumulator. + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; +} + +// Result accessors +// +// These read from the base context's result storage (tokens / words / +// segments / full_text / result_kind / has_result), populated by the +// per-family run() driver. Out-of-range indices and pre-run access return +// safe sentinels per the public contract. + +extern "C" const char * transcribe_full_text(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->full_text.c_str(); +} + +extern "C" const char * transcribe_raw_text(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->raw_text.c_str(); +} + +extern "C" const char * transcribe_detected_language(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return ""; + } + return session->detected_language.c_str(); +} + +extern "C" transcribe_timestamp_kind transcribe_returned_timestamp_kind(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return TRANSCRIBE_TIMESTAMPS_NONE; + } + return session->result_kind; +} + +extern "C" int transcribe_n_segments(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->segments.size()); +} + +extern "C" int transcribe_n_words(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->words.size()); +} + +extern "C" int transcribe_n_tokens(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->tokens.size()); +} + +extern "C" int transcribe_n_speaker_segments(const struct transcribe_session * session) { + if (session == nullptr || !session->has_result) { + return 0; + } + return static_cast(session->speaker_segments.size()); +} + +// Result accessors - per-item rows +// +// Copy-out accessors backed by the context's segments / words / tokens +// vectors, writing only the prefix that fits the caller's struct_size. +// Out-of-range index or pre-run access leaves the caller's struct zero-init +// and still returns OK, so callers can use text!=NULL as the "row present" +// signal without branching on status. +// +// `text` pointers in the staged structs alias the context's std::string +// storage; the library treats that as session-owned, valid until the next +// transcribe_run / transcribe_stream_begin / transcribe_stream_reset / +// transcribe_session_free on the same context (see the public header). + +extern "C" transcribe_status transcribe_get_segment(const struct transcribe_session * session, + int i, + struct transcribe_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->segments.size()) { + return TRANSCRIBE_OK; + } + const auto & s = session->segments[static_cast(i)]; + transcribe_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.first_word = s.first_word; + staged.n_words = s.n_words; + staged.first_token = s.first_token; + staged.n_tokens = s.n_tokens; + staged.text = s.text.c_str(); + staged.speaker_id = s.speaker_id; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_word(const struct transcribe_session * session, + int i, + struct transcribe_word * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_word zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->words.size()) { + return TRANSCRIBE_OK; + } + const auto & w = session->words[static_cast(i)]; + transcribe_word staged{}; + staged.struct_size = caller_size; + staged.t0_ms = w.t0_ms; + staged.t1_ms = w.t1_ms; + staged.seg_index = w.seg_index; + staged.first_token = w.first_token; + staged.n_tokens = w.n_tokens; + staged.text = w.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_token(const struct transcribe_session * session, + int i, + struct transcribe_token * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_token zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->tokens.size()) { + return TRANSCRIBE_OK; + } + const auto & t = session->tokens[static_cast(i)]; + transcribe_token staged{}; + staged.struct_size = caller_size; + staged.id = t.id; + staged.p = t.p; + staged.t0_ms = t.t0_ms; + staged.t1_ms = t.t1_ms; + staged.seg_index = t.seg_index; + staged.word_index = t.word_index; + staged.text = t.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_get_speaker_segment(const struct transcribe_session * session, + int i, + struct transcribe_speaker_segment * out) { + const bool in_range = session != nullptr && session->has_result && i >= 0 && + static_cast(i) < session->speaker_segments.size(); + return transcribe::copy_out_speaker_segment(in_range ? &session->speaker_segments[static_cast(i)] : nullptr, + out); +} + +// Batch result accessors +// +// These index session->batch_results when a batch run populated it, and +// otherwise synthesize utterance 0 from the scratch slot after +// transcribe_run. They share the staging shape of the single-result +// accessors above; only the source vector differs. + +namespace { + +// Non-owning view of one utterance's result. valid == false means "no such +// utterance / empty result" and the accessors return their safe sentinels. +struct BatchResultView { + const std::vector * tokens = nullptr; + const std::vector * words = nullptr; + const std::vector * segments = nullptr; + const std::vector * speaker_segments = nullptr; + const std::string * full_text = nullptr; + const std::string * raw_text = nullptr; + const std::string * detected_language = nullptr; + transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; + bool valid = false; +}; + +int batch_result_count(const transcribe_session * s) { + if (s == nullptr) { + return 0; + } + if (!s->batch_results.empty()) { + return static_cast(s->batch_results.size()); + } + return s->has_result ? 1 : 0; +} + +BatchResultView batch_result_view(const transcribe_session * s, int i) { + BatchResultView v; + if (s == nullptr || i < 0) { + return v; + } + if (!s->batch_results.empty()) { + if (static_cast(i) >= s->batch_results.size()) { + return v; + } + const auto & rs = s->batch_results[static_cast(i)]; + if (!rs.has_result) { + return v; // individually-failed / empty utterance + } + v.tokens = &rs.tokens; + v.words = &rs.words; + v.segments = &rs.segments; + v.speaker_segments = &rs.speaker_segments; + v.full_text = &rs.full_text; + v.raw_text = &rs.raw_text; + v.detected_language = &rs.detected_language; + v.result_kind = rs.result_kind; + v.valid = true; + return v; + } + if (i == 0 && s->has_result) { + v.tokens = &s->tokens; + v.words = &s->words; + v.segments = &s->segments; + v.speaker_segments = &s->speaker_segments; + v.full_text = &s->full_text; + v.raw_text = &s->raw_text; + v.detected_language = &s->detected_language; + v.result_kind = s->result_kind; + v.valid = true; + } + return v; +} + +} // namespace + +extern "C" int transcribe_batch_n_results(const struct transcribe_session * session) { + return batch_result_count(session); +} + +extern "C" transcribe_status transcribe_batch_status(const struct transcribe_session * session, int i) { + if (session == nullptr || i < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (!session->batch_results.empty()) { + if (static_cast(i) >= session->batch_results.size()) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + return session->batch_results[static_cast(i)].status; + } + if (i == 0 && session->has_result) { + return TRANSCRIBE_OK; + } + return TRANSCRIBE_ERR_INVALID_ARG; +} + +extern "C" const char * transcribe_batch_full_text(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.full_text->c_str() : ""; +} + +extern "C" const char * transcribe_batch_raw_text(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.raw_text->c_str() : ""; +} + +extern "C" const char * transcribe_batch_detected_language(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.detected_language->c_str() : ""; +} + +extern "C" transcribe_timestamp_kind transcribe_batch_returned_timestamp_kind(const struct transcribe_session * session, + int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? v.result_kind : TRANSCRIBE_TIMESTAMPS_NONE; +} + +extern "C" int transcribe_batch_n_segments(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.segments->size()) : 0; +} + +extern "C" int transcribe_batch_n_words(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.words->size()) : 0; +} + +extern "C" int transcribe_batch_n_tokens(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.tokens->size()) : 0; +} + +extern "C" transcribe_status transcribe_batch_get_segment(const struct transcribe_session * session, + int i, + int j, + struct transcribe_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_segment zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.segments->size()) { + return TRANSCRIBE_OK; + } + const auto & s = (*v.segments)[static_cast(j)]; + transcribe_segment staged{}; + staged.struct_size = caller_size; + staged.t0_ms = s.t0_ms; + staged.t1_ms = s.t1_ms; + staged.first_word = s.first_word; + staged.n_words = s.n_words; + staged.first_token = s.first_token; + staged.n_tokens = s.n_tokens; + staged.text = s.text.c_str(); + staged.speaker_id = s.speaker_id; + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_batch_get_word(const struct transcribe_session * session, + int i, + int j, + struct transcribe_word * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_word zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.words->size()) { + return TRANSCRIBE_OK; + } + const auto & w = (*v.words)[static_cast(j)]; + transcribe_word staged{}; + staged.struct_size = caller_size; + staged.t0_ms = w.t0_ms; + staged.t1_ms = w.t1_ms; + staged.seg_index = w.seg_index; + staged.first_token = w.first_token; + staged.n_tokens = w.n_tokens; + staged.text = w.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" transcribe_status transcribe_batch_get_token(const struct transcribe_session * session, + int i, + int j, + struct transcribe_token * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { + return st; + } + const uint64_t caller_size = out->struct_size; + transcribe_token zero{}; + zero.struct_size = caller_size; + copy_out_prefix(out, &zero, caller_size, sizeof(zero)); + const BatchResultView v = batch_result_view(session, i); + if (!v.valid || j < 0 || static_cast(j) >= v.tokens->size()) { + return TRANSCRIBE_OK; + } + const auto & t = (*v.tokens)[static_cast(j)]; + transcribe_token staged{}; + staged.struct_size = caller_size; + staged.id = t.id; + staged.p = t.p; + staged.t0_ms = t.t0_ms; + staged.t1_ms = t.t1_ms; + staged.seg_index = t.seg_index; + staged.word_index = t.word_index; + staged.text = t.text.c_str(); + copy_out_prefix(out, &staged, caller_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +extern "C" int transcribe_batch_n_speaker_segments(const struct transcribe_session * session, int i) { + const BatchResultView v = batch_result_view(session, i); + return v.valid ? static_cast(v.speaker_segments->size()) : 0; +} + +extern "C" transcribe_status transcribe_batch_get_speaker_segment(const struct transcribe_session * session, + int i, + int j, + struct transcribe_speaker_segment * out) { + const BatchResultView v = batch_result_view(session, i); + const bool in_range = v.valid && j >= 0 && static_cast(j) < v.speaker_segments->size(); + return transcribe::copy_out_speaker_segment(in_range ? &(*v.speaker_segments)[static_cast(j)] : nullptr, + out); +} + +extern "C" transcribe_status transcribe_batch_get_timings(const struct transcribe_session * session, + int i, + struct transcribe_timings * out) { + if (session == nullptr || out == nullptr || i < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + int64_t mel_us = 0, enc_us = 0, dec_us = 0; + if (!session->batch_results.empty()) { + if (static_cast(i) >= session->batch_results.size()) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const auto & rs = session->batch_results[static_cast(i)]; + mel_us = rs.t_mel_us; + enc_us = rs.t_encode_us; + dec_us = rs.t_decode_us; + } else if (i == 0) { + mel_us = session->t_mel_us; + enc_us = session->t_encode_us; + dec_us = session->t_decode_us; + } else { + return TRANSCRIBE_ERR_INVALID_ARG; + } + transcribe::copy_out_timings(session->model != nullptr ? session->model->t_load_us : 0, mel_us, enc_us, dec_us, + out); + return TRANSCRIBE_OK; +} + +// C ABI forwarders. Entry points that allocate, reach ggml/driver state, +// or transfer ownership route through api_guard_* here. Other public symbols +// above are kept nothrow by construction: pure POD/c_str reads, prefix +// copies, plain stores, or contained callback emission. + +extern "C" transcribe_status transcribe_session_init(struct transcribe_model * model, + const struct transcribe_session_params * params, + struct transcribe_session ** out_session) { + const transcribe_status st = api_guard_status( + "transcribe_session_init", [&] { return transcribe_session_init_impl(model, params, out_session); }); + // Boundary-owned postcondition: failure => *out_session == NULL. + if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { + transcribe_session_free(*out_session); + *out_session = nullptr; + } + return st; +} + +extern "C" void transcribe_session_free(struct transcribe_session * session) { + api_guard_void("transcribe_session_free", [&] { transcribe_session_free_impl(session); }); +} + +extern "C" transcribe_status transcribe_open(const char * path, + const struct transcribe_model_load_params * load_params, + const struct transcribe_session_params * session_params, + struct transcribe_session ** out_session) { + const transcribe_status st = api_guard_status( + "transcribe_open", [&] { return transcribe_open_impl(path, load_params, session_params, out_session); }); + // Boundary-owned postcondition: failure => *out_session == NULL. + if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { + transcribe_session_free(*out_session); + *out_session = nullptr; + } + return st; +} + +extern "C" transcribe_status transcribe_run(struct transcribe_session * session, + const float * pcm, + int n_samples, + const struct transcribe_run_params * params) { + return api_guard_status("transcribe_run", [&] { return transcribe_run_impl(session, pcm, n_samples, params); }); +} + +extern "C" transcribe_status transcribe_run_batch(struct transcribe_session * session, + const float * const * pcm, + const int * n_samples, + int n, + const struct transcribe_run_params * params) { + return api_guard_status("transcribe_run_batch", + [&] { return transcribe_run_batch_impl(session, pcm, n_samples, n, params); }); +} + +extern "C" transcribe_status transcribe_stream_begin(struct transcribe_session * session, + const struct transcribe_run_params * run_params, + const struct transcribe_stream_params * stream_params) { + return api_guard_status("transcribe_stream_begin", + [&] { return transcribe_stream_begin_impl(session, run_params, stream_params); }); +} + +extern "C" transcribe_status transcribe_stream_feed(struct transcribe_session * session, + const float * pcm, + int n_samples, + struct transcribe_stream_update * update) { + return api_guard_status("transcribe_stream_feed", + [&] { return transcribe_stream_feed_impl(session, pcm, n_samples, update); }); +} + +extern "C" transcribe_status transcribe_stream_finalize(struct transcribe_session * session, + struct transcribe_stream_update * update) { + return api_guard_status("transcribe_stream_finalize", + [&] { return transcribe_stream_finalize_impl(session, update); }); +} + +extern "C" void transcribe_stream_reset(struct transcribe_session * session) { + api_guard_void("transcribe_stream_reset", [&] { transcribe_stream_reset_impl(session); }); +} + +extern "C" int transcribe_tokenize(const struct transcribe_model * model, + const char * text, + int32_t * tokens, + size_t n_max) { + return api_guard_value("transcribe_tokenize", INT_MIN, + [&] { return transcribe_tokenize_impl(model, text, tokens, n_max); }); +} diff --git a/src/transcribe-batch-util.cpp b/src/transcribe-batch-util.cpp index 5fa509b63..e0d389bef 100644 --- a/src/transcribe-batch-util.cpp +++ b/src/transcribe-batch-util.cpp @@ -110,15 +110,7 @@ bool parallel_for_all(int n, int n_threads, const std::function & wor } }; - std::vector pool; - pool.reserve(static_cast(n_threads - 1)); - for (int w = 0; w < n_threads - 1; ++w) { - pool.emplace_back(worker); - } - worker(); // the calling thread participates - for (auto & th : pool) { - th.join(); - } + run_on_threads(n_threads, [&](int) { worker(); }); // the calling thread participates return all_ok.load(std::memory_order_relaxed); } diff --git a/src/transcribe-batch-util.h b/src/transcribe-batch-util.h index cb2586d6d..7aeef25b6 100644 --- a/src/transcribe-batch-util.h +++ b/src/transcribe-batch-util.h @@ -13,7 +13,9 @@ #include #include +#include #include +#include #include struct ggml_tensor; @@ -32,6 +34,55 @@ namespace transcribe { // falling back to the per-utterance path). bool parallel_for_all(int n, int n_threads, const std::function & work); +// Run `fn(tid)` once for every tid in [0, n_threads): tid 0 on the calling +// thread, the rest on their own threads. Every launched thread is joined on +// every path, and exceptions reach the caller instead of std::terminate: +// - a thread that fails to launch stops further launches, tid 0 is not run, +// and the launch error is rethrown once the launched threads have joined; +// - otherwise, if any fn(tid) threw, the lowest tid's exception is rethrown +// after all threads have joined. +// n_threads <= 1 runs fn(0) inline. `Thread` is a test seam (fault injection +// for thread launch); production code uses the default. +template void run_on_threads(int n_threads, Fn && fn) { + if (n_threads <= 1) { + fn(0); + return; + } + std::vector errors(static_cast(n_threads)); + auto guarded = [&fn, &errors](int tid) { + try { + fn(tid); + } catch (...) { + errors[static_cast(tid)] = std::current_exception(); + } + }; + std::vector pool; + pool.reserve(static_cast(n_threads - 1)); + std::exception_ptr launch_error; + for (int tid = 1; tid < n_threads; ++tid) { + try { + pool.emplace_back(guarded, tid); + } catch (...) { + launch_error = std::current_exception(); + break; + } + } + if (!launch_error) { + guarded(0); + } + for (auto & th : pool) { + th.join(); + } + if (launch_error) { + std::rethrow_exception(launch_error); + } + for (const auto & e : errors) { + if (e) { + std::rethrow_exception(e); + } + } +} + // Default CPU thread count for the ggml backends and the host parallel-for // when the caller passes n_threads <= 0. Counts the CPUs the process may // actually run on (the affinity mask via sched_getaffinity on Linux / diff --git a/src/transcribe-diarize.cpp b/src/transcribe-diarize.cpp new file mode 100644 index 000000000..370e38fa7 --- /dev/null +++ b/src/transcribe-diarize.cpp @@ -0,0 +1,223 @@ +// transcribe-diarize.cpp - DIARIZE role C ABI (include/transcribe/diarize.h) +// and the shared probs -> speaker segments step. + +#include "transcribe-diarize.h" + +#include "transcribe-abi.h" +#include "transcribe-api-guard.h" +#include "transcribe-arch.h" +#include "transcribe-model.h" + +#include +#include + +using transcribe::api_guard_status; +using transcribe::api_guard_void; +using transcribe::check_input_struct_size; +using transcribe::check_struct_size; +using transcribe::copy_out_prefix; + +namespace { + +constexpr size_t k_min_info_size = TRANSCRIBE_FIELD_END(transcribe_diarize_info, max_speakers); +constexpr size_t k_min_session_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_session_params, n_threads); +constexpr size_t k_min_params_size = TRANSCRIBE_FIELD_END(transcribe_diarize_params, family); +constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); + +const transcribe::DiarizeOps * diarize_ops(const transcribe_model * model) { + return (model->roles & TRANSCRIBE_ROLE_DIARIZE) != 0 ? model->arch->diarize : nullptr; +} + +} // namespace + +void transcribe::probs_to_segments(const float * probs, + int n_frames, + int n_speakers, + double frame_ms, + std::vector & out) { + for (int s = 0; s < n_speakers; ++s) { + int run_start = -1; + for (int t = 0; t <= n_frames; ++t) { + const bool active = t < n_frames && probs[static_cast(t) * n_speakers + s] > 0.5f; + if (active && run_start < 0) { + run_start = t; + } else if (!active && run_start >= 0) { + SpeakerSegmentEntry row; + row.t0_ms = static_cast(std::llround(run_start * frame_ms)); + row.t1_ms = static_cast(std::llround(t * frame_ms)); + row.speaker_id = s + 1; + out.push_back(row); + run_start = -1; + } + } + } +} + +extern "C" void transcribe_diarize_info_init(struct transcribe_diarize_info * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_diarize_session_params_init(struct transcribe_diarize_session_params * p) { + transcribe::init_sized(p); +} + +extern "C" void transcribe_diarize_params_init(struct transcribe_diarize_params * p) { + transcribe::init_sized(p); +} + +static transcribe_status diarize_get_info_impl(const transcribe_model * model, transcribe_diarize_info * out) { + if (model == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_info_size); st != TRANSCRIBE_OK) { + return st; + } + const transcribe::DiarizeOps * ops = diarize_ops(model); + if (ops == nullptr) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + transcribe_diarize_info staged{}; + staged.struct_size = out->struct_size; + staged.sample_rate = 16000; + staged.max_speakers = ops->max_speakers(model); + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +static transcribe_status diarize_session_init_impl(transcribe_model * model, + const transcribe_diarize_session_params * params, + transcribe_diarize_session ** out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out = nullptr; + if (model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + const transcribe::DiarizeOps * ops = diarize_ops(model); + if (ops == nullptr) { + return TRANSCRIBE_ERR_UNSUPPORTED_ROLE; + } + transcribe_diarize_session_params defaults; + transcribe_diarize_session_params_init(&defaults); + if (params == nullptr) { + params = &defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_session_params_size); st != TRANSCRIBE_OK) { + return st; + } + if (params->n_threads < 0) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + *out = ops->new_session(); + (*out)->model = model; + (*out)->n_threads = params->n_threads; + return TRANSCRIBE_OK; +} + +static transcribe_status diarize_run_impl(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params) { + // Everything up to the commit point leaves the previous result intact. + if (session == nullptr || pcm == nullptr || n_samples <= 0 || !transcribe::pcm_is_finite(pcm, n_samples)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + transcribe_diarize_params defaults; + transcribe_diarize_params_init(&defaults); + if (params == nullptr) { + params = &defaults; + } + if (const auto st = check_input_struct_size(params->struct_size, k_min_params_size); st != TRANSCRIBE_OK) { + return st; + } + const transcribe::DiarizeOps * ops = diarize_ops(session->model); + if (params->family != nullptr) { + if (params->family->size < sizeof(struct transcribe_ext)) { + return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; + } + if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, params->family->kind)) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (ops->run_validate != nullptr) { + if (const auto st = ops->run_validate(params); st != TRANSCRIBE_OK) { + return st; + } + } + } + + session->segments.clear(); + session->t_mel_us = 0; + session->t_encode_us = 0; + session->t_decode_us = 0; + transcribe::ScratchReleaseGuard release{ session, true }; + + transcribe::DiarizeProbs out; + if (const auto st = ops->run(session, pcm, n_samples, params, out); st != TRANSCRIBE_OK) { + return st; + } + std::vector segments; + transcribe::probs_to_segments(out.probs.data(), out.n_frames, out.n_speakers, out.frame_ms, segments); + session->segments.swap(segments); + return TRANSCRIBE_OK; +} + +extern "C" void transcribe_diarize_set_abort_callback(struct transcribe_diarize_session * session, + transcribe_abort_callback cb, + void * user_data) { + if (session != nullptr) { + session->abort_cb = cb; + session->abort_userdata = user_data; + } +} + +extern "C" int transcribe_diarize_n_segments(const struct transcribe_diarize_session * session) { + return session != nullptr ? static_cast(session->segments.size()) : 0; +} + +extern "C" transcribe_status transcribe_diarize_get_segment(const struct transcribe_diarize_session * session, + int i, + struct transcribe_speaker_segment * out) { + const bool in_range = i >= 0 && i < transcribe_diarize_n_segments(session); + return transcribe::copy_out_speaker_segment(in_range ? &session->segments[static_cast(i)] : nullptr, out); +} + +extern "C" transcribe_status transcribe_diarize_get_timings(const struct transcribe_diarize_session * session, + struct transcribe_timings * out) { + if (session == nullptr || out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { + return st; + } + transcribe::copy_out_timings(session->model->t_load_us, session->t_mel_us, session->t_encode_us, + session->t_decode_us, out); + return TRANSCRIBE_OK; +} + +// C ABI forwarders for the entry points that allocate, compute, or transfer +// ownership; the ones above are nothrow by construction. + +extern "C" transcribe_status transcribe_diarize_get_info(const struct transcribe_model * model, + struct transcribe_diarize_info * out) { + return api_guard_status("transcribe_diarize_get_info", [&] { return diarize_get_info_impl(model, out); }); +} + +extern "C" transcribe_status transcribe_diarize_session_init(struct transcribe_model * model, + const struct transcribe_diarize_session_params * params, + struct transcribe_diarize_session ** out) { + return api_guard_status("transcribe_diarize_session_init", + [&] { return diarize_session_init_impl(model, params, out); }); +} + +extern "C" void transcribe_diarize_session_free(struct transcribe_diarize_session * session) { + api_guard_void("transcribe_diarize_session_free", [&] { delete session; }); +} + +extern "C" transcribe_status transcribe_diarize_run(struct transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const struct transcribe_diarize_params * params) { + return api_guard_status("transcribe_diarize_run", + [&] { return diarize_run_impl(session, pcm, n_samples, params); }); +} diff --git a/src/transcribe-diarize.h b/src/transcribe-diarize.h new file mode 100644 index 000000000..8e315e664 --- /dev/null +++ b/src/transcribe-diarize.h @@ -0,0 +1,49 @@ +// transcribe-diarize.h - internal DIARIZE role surface: the session base, +// the per-arch ops table, and the shared probs -> segments step. +// +// Families compute frame probabilities; the role dispatcher +// (transcribe-diarize.cpp) turns them into speaker segments. + +#pragma once + +#include "transcribe-session-core.h" +#include "transcribe/diarize.h" + +#include + +struct transcribe_diarize_session : transcribe::SessionCore { + std::vector segments; // last successful run's rows +}; + +namespace transcribe { + +// One run's family output: probs is row-major [n_frames, n_speakers]. +struct DiarizeProbs { + std::vector probs; + int n_frames = 0; + int n_speakers = 0; + double frame_ms = 0.0; +}; + +struct DiarizeOps { + int (*max_speakers)(const transcribe_model * model); + transcribe_diarize_session * (*new_session)(); + // Pure pre-clear check of params->family (already kind/size-checked); + // NULL = nothing to check. + transcribe_status (*run_validate)(const transcribe_diarize_params * params); + transcribe_status (*run)(transcribe_diarize_session * session, + const float * pcm, + int n_samples, + const transcribe_diarize_params * params, + DiarizeProbs & out); +}; + +// Append one segment per contiguous run of probs > 0.5 per speaker, +// speaker-major and time-ordered (speaker_id 1-based, p = NaN). +void probs_to_segments(const float * probs, + int n_frames, + int n_speakers, + double frame_ms, + std::vector & out); + +} // namespace transcribe diff --git a/src/transcribe-mel.cpp b/src/transcribe-mel.cpp index 085197e0a..0893486d6 100644 --- a/src/transcribe-mel.cpp +++ b/src/transcribe-mel.cpp @@ -490,21 +490,10 @@ transcribe_status MelFrontend::compute(const float * pcm, // Per-thread worker dispatcher. Strided frame assignment (stride = // stft_threads) keeps load balanced and per-thread memory access - // patterns mostly sequential. + // patterns mostly sequential. run_on_threads joins every thread and + // hands worker exceptions back to the caller. auto run_threaded = [&](auto && worker) { - if (stft_threads <= 1) { - worker(0); - return; - } - std::vector pool; - pool.reserve(static_cast(stft_threads - 1)); - for (int tid = 1; tid < stft_threads; ++tid) { - pool.emplace_back(worker, tid); - } - worker(0); - for (auto & th : pool) { - th.join(); - } + run_on_threads(stft_threads, worker); }; if (!n_fft_is_pow2) { @@ -579,6 +568,14 @@ transcribe_status MelFrontend::compute(const float * pcm, FFTSetupD fft_setup = vDSP_create_fftsetupD(log2n, FFT_RADIX2); const size_t half_n = static_cast(n_fft / 2); + // Destroyed on every exit, including a worker exception rethrown by + // run_threaded. + struct FftSetupGuard { + FFTSetupD setup; + + ~FftSetupGuard() { vDSP_destroy_fftsetupD(setup); } + } fft_setup_guard{ fft_setup }; + auto worker = [&](int tid) { std::vector fft_real(half_n); std::vector fft_imag(half_n); @@ -603,7 +600,6 @@ transcribe_status MelFrontend::compute(const float * pcm, } }; run_threaded(worker); - vDSP_destroy_fftsetupD(fft_setup); #else // Hand-rolled radix-2 Cooley-Tukey FFT fallback for non-Apple. auto worker = [&](int tid) { diff --git a/src/transcribe-model.cpp b/src/transcribe-model.cpp index ea14c46ec..8bf79f23c 100644 --- a/src/transcribe-model.cpp +++ b/src/transcribe-model.cpp @@ -12,16 +12,18 @@ transcribe_model::~transcribe_model() = default; // Scheduler first, then the compute context (release_compute_scratch keeps -// that order). Runs after the derived destructor; see the header note. -transcribe_session::~transcribe_session() { +// that order). Runs after the derived destructors; see the header note. +transcribe::SessionCore::~SessionCore() { transcribe::release_compute_scratch(sched, compute_ctx); } -void transcribe_session::release_scratch() noexcept { +void transcribe::SessionCore::release_scratch() noexcept { transcribe::release_compute_scratch(sched, compute_ctx); on_scratch_released(); } +transcribe_session::~transcribe_session() = default; + void transcribe_session::clear_result() { tokens.clear(); words.clear(); diff --git a/src/transcribe-model.h b/src/transcribe-model.h index ce522cd9b..e37c439ae 100644 --- a/src/transcribe-model.h +++ b/src/transcribe-model.h @@ -41,6 +41,10 @@ struct transcribe_model { // for per-call dispatch (init_context, run). const transcribe::Arch * arch = nullptr; + // transcribe_role bits; set by load() (0 = ASR when the arch has ASR + // hooks) and validated by resolve_roles. + uint32_t roles = 0; + // Identification, both surfaced via the public string accessors. // variant is whatever the family decided (loader leaves it empty if // stt.variant was absent; the family supplies a default). @@ -85,7 +89,7 @@ struct transcribe_model { // Basis for the session-level limits query (transcribe_session_get_limits). // A hard-context-cap family fills this at load() — the same place it - // computes caps.max_audio_ms — so the generic query in transcribe.cpp can + // computes caps.max_audio_ms — so the generic query in transcribe-asr.cpp can // recompute the effective limits for any session n_ctx without a // per-family hook. Left zero by unbounded / soft-window families (which // have no decoder context cap): zero model_max_ctx => the query reports diff --git a/src/transcribe-session-core.h b/src/transcribe-session-core.h new file mode 100644 index 000000000..6fd7f7c32 --- /dev/null +++ b/src/transcribe-session-core.h @@ -0,0 +1,158 @@ +// transcribe-session-core.h - internal base shared by every role session. +// +// SessionCore holds what is not specific to any role: the borrowed model, +// the thread count, the per-run ggml compute scratch, the abort callback and +// the stage timings. transcribe_session (ASR) derives from it, and so does +// each role's session type, so scratch release (Handy #2000), cancellation +// and timings behave the same for every role. + +#pragma once + +#include "transcribe-abi.h" +#include "transcribe.h" + +#include +#include + +struct transcribe_model; + +// ggml handle types for the base-owned compute scratch. Forward-declared so +// this internal header does not pull the ggml headers into every includer +// (the public transcribe.h never sees them); transcribe-backend.h uses the +// same pattern for ggml_backend_t. +struct ggml_context; +struct ggml_backend_sched; +typedef struct ggml_backend_sched * ggml_backend_sched_t; + +namespace transcribe { + +// "Who spoke when" rows; may overlap in time. t0_ms == t1_ms == 0 means the +// family attributes text but has no timing for the turn. +struct SpeakerSegmentEntry { + int64_t t0_ms = 0; + int64_t t1_ms = 0; + int32_t speaker_id = 0; // 1-based + float p = NAN; +}; + +struct SessionCore { + // The model this session was constructed from. Borrowed pointer: + // the caller is required (per the public threading contract) to keep + // the model alive for the lifetime of every derived session. + transcribe_model * model = nullptr; + + // Cached n_threads value the caller passed at init time. 0 means + // "library picks a sensible default" (matches the factory). + int n_threads = 0; + + // Stage timings of the last run (microseconds). + int64_t t_mel_us = 0; + int64_t t_encode_us = 0; + int64_t t_decode_us = 0; + + // Abort callback; run drivers call poll_abort() at chunk / step + // boundaries and return TRANSCRIBE_ERR_ABORTED when it fires. + transcribe_abort_callback abort_cb = nullptr; + void * abort_userdata = nullptr; + bool was_aborted = false; + + bool poll_abort() { + if (abort_cb != nullptr && abort_cb(abort_userdata)) { + was_aborted = true; + return true; + } + return false; + } + + // Per-run ggml compute scratch, owned by the base so every family + // releases it the same way and none can forget to: the backend + // scheduler (whose graph allocator only ever grows) and the no_alloc + // graph context. Families create both lazily inside their run / stream + // hooks and use them directly; the base frees them in release_scratch + // and in its destructor (scheduler first, then context). + ggml_backend_sched_t sched = nullptr; + ggml_context * compute_ctx = nullptr; + + // Release the per-run ggml compute scratch (sched, then compute_ctx; + // see transcribe::release_compute_scratch), then let the family drop + // pointers that lived in them (on_scratch_released). + // The dispatcher calls this after every offline transcribe_run or + // transcribe_run_batch that passes pre-clear validation and reaches its + // commit point, whether family execution succeeds, fails, or throws. + // Without this, a single long utterance would pin the scheduler's + // high-water mark in backend compute memory for the session's lifetime + // (Handy #2000). Families re-create the scheduler lazily on the next run. + // The measured recreation cost is about 1 ms per run on Metal and up to + // about 10 ms on CPU for families that reserve a worst-case decoder + // workspace each run, such as Canary and Cohere. + // + // Host-side vectors that scale with input length, such as mel buffers, + // encoder host copies, and positional banks, deliberately retain their + // capacity. Family KV caches sized by the last batch are also retained; + // this hook releases only the ggml scheduler and compute context. + // + // Streaming entry points do not invoke this hook. A streaming session + // keeps its scheduler until a later offline run or session destruction. + // Parakeet and Voxtral Realtime use per-chunk bounded stream workspaces; + // Moonshine Streaming's decode graph cross-attends over the complete + // committed stream, so its workspace grows with total stream length. + // Must not throw. Non-virtual: a family cannot opt out of the release. + void release_scratch() noexcept; + + SessionCore() = default; + // Frees sched / compute_ctx after the derived destructors have run. The + // scheduler owns only its own allocator buffers and references + // model-owned backends, so freeing it after the family's KV caches and + // stream buffers is order-independent. + virtual ~SessionCore(); + + SessionCore(const SessionCore &) = delete; + SessionCore & operator=(const SessionCore &) = delete; + SessionCore(SessionCore &&) = delete; + SessionCore & operator=(SessionCore &&) = delete; + + protected: + // Family hook, called by release_scratch after sched / compute_ctx are + // freed: null out tensors borrowed from the freed context (encoder_out) + // or reset capacity bookkeeping (whisper compute_ctx_size). Most + // families need nothing. Not called from the base destructor because + // derived members are already destroyed by then. Must not throw. + virtual void on_scratch_released() noexcept {} +}; + +// Calls release_scratch on scope exit once armed, so release also happens +// when a family hook throws and the api_guard unwinds. +struct ScratchReleaseGuard { + SessionCore * session = nullptr; + bool armed = false; + + ~ScratchReleaseGuard() { + if (armed && session != nullptr) { + session->release_scratch(); + } + } +}; + +// Copy a speaker-segment row out (a zeroed row when s is NULL), after the +// struct_size check every speaker-segment accessor shares. +inline transcribe_status copy_out_speaker_segment(const SpeakerSegmentEntry * s, transcribe_speaker_segment * out) { + if (out == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; + } + if (const auto st = check_struct_size(out->struct_size, TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p)); + st != TRANSCRIBE_OK) { + return st; + } + transcribe_speaker_segment staged{}; + staged.struct_size = out->struct_size; + if (s != nullptr) { + staged.t0_ms = s->t0_ms; + staged.t1_ms = s->t1_ms; + staged.speaker_id = s->speaker_id; + staged.p = s->p; + } + copy_out_prefix(out, &staged, out->struct_size, sizeof(staged)); + return TRANSCRIBE_OK; +} + +} // namespace transcribe diff --git a/src/transcribe-session.h b/src/transcribe-session.h index e3984b7e3..8f4ff9646 100644 --- a/src/transcribe-session.h +++ b/src/transcribe-session.h @@ -10,25 +10,15 @@ #pragma once +#include "transcribe-session-core.h" #include "transcribe.h" -#include #include #include #include #include #include -struct transcribe_model; - -// ggml handle types for the base-owned compute scratch. Forward-declared so -// this internal header does not pull the ggml headers into every includer -// (the public transcribe.h never sees them); transcribe-backend.h uses the -// same pattern for ggml_backend_t. -struct ggml_context; -struct ggml_backend_sched; -typedef struct ggml_backend_sched * ggml_backend_sched_t; - // Read transcribe_session_params::n_ctx with a struct_size guard. n_ctx is // a trailing field appended after kv_type, so an older caller's smaller // struct may not include it; in that case (or a NULL params) the default 0 @@ -47,12 +37,9 @@ inline int32_t transcribe_session_params_n_ctx(const struct transcribe_session_p return params->n_ctx; } -struct transcribe_session { - // The model this session was constructed from. Borrowed pointer: - // the caller is required (per the public threading contract) to keep - // the model alive for the lifetime of every derived session. - transcribe_model * model = nullptr; - +// Common members (model, n_threads, timings, abort callback, compute +// scratch) live on transcribe::SessionCore; see transcribe-session-core.h. +struct transcribe_session : transcribe::SessionCore { // True only for sessions created via transcribe_open(), which loads // and therefore owns its model. Both transcribe_session_free() and // transcribe_close() (now an alias) read this flag and free the @@ -61,10 +48,6 @@ struct transcribe_session { // freed independently by the caller via transcribe_model_free(). bool owns_model = false; - // Cached n_threads value the caller passed at init time. 0 means - // "library picks a sensible default" (matches the factory). - int n_threads = 0; - // Cached n_ctx value the caller passed at init time (decoder context // cap in tokens). 0 means "use the model's true maximum from GGUF". // A positive value lowers the ceiling to bound KV-cache memory and is @@ -81,13 +64,6 @@ struct transcribe_session { // for byte accounting only F32 differs (4 bytes/elem vs 2). transcribe_kv_type kv_type = TRANSCRIBE_KV_TYPE_AUTO; - // Per-call timings, populated by the most recent transcribe_run. - // Surfaced via the public transcribe_get_timings accessor; reset - // by transcribe_reset_timings. - int64_t t_mel_us = 0; - int64_t t_encode_us = 0; - int64_t t_decode_us = 0; - // Result storage (family-agnostic; populated by per-family run()). // A flat backing array per level (segments / words / tokens) with // forward/backward indices between levels. result_kind is the @@ -125,16 +101,9 @@ struct transcribe_session { int32_t speaker_id = 0; // 1-based; 0 = no attribution }; - // "Who spoke when" rows (diarization). Populated only when a run - // resolves diarize ON for a supporting family; may overlap in time. - // t0_ms == t1_ms == 0 means the family attributes text but has no - // timing information for the turn. - struct SpeakerSegmentEntry { - int64_t t0_ms = 0; - int64_t t1_ms = 0; - int32_t speaker_id = 0; // 1-based - float p = NAN; - }; + // Hoisted to transcribe::SpeakerSegmentEntry so every session type + // that reports speaker segments shares one row type. + using SpeakerSegmentEntry = transcribe::SpeakerSegmentEntry; std::vector tokens; std::vector words; @@ -218,24 +187,6 @@ struct transcribe_session { transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; bool has_result = false; - // Abort / cancellation (set via transcribe_set_abort_callback). - // run() drivers call poll_abort() at chunk / decode-step boundaries; - // a callback returning true sets was_aborted and the run returns - // TRANSCRIBE_ERR_ABORTED with partial segments preserved. was_aborted - // is cleared at the top of every transcribe_run, NOT by clear_result - // (the partial result may be deliberately retained). - transcribe_abort_callback abort_cb = nullptr; - void * abort_userdata = nullptr; - bool was_aborted = false; - - bool poll_abort() { - if (abort_cb != nullptr && abort_cb(abort_userdata)) { - was_aborted = true; - return true; - } - return false; - } - // Set by a run() driver when decode stops at the model's context / // position cap before end-of-stream (output truncated; partial result // retained). Surfaced via transcribe_was_truncated(); cleared at the @@ -306,58 +257,11 @@ struct transcribe_session { void clear_result(); - // Per-run ggml compute scratch, owned by the base so every family - // releases it the same way and none can forget to: the backend - // scheduler (whose graph allocator only ever grows) and the no_alloc - // graph context. Families create both lazily inside their run / stream - // hooks and use them directly; the base frees them in release_scratch - // and in its destructor (scheduler first, then context). - ggml_backend_sched_t sched = nullptr; - ggml_context * compute_ctx = nullptr; - - // Release the per-run ggml compute scratch (sched, then compute_ctx; - // see transcribe::release_compute_scratch), then let the family drop - // pointers that lived in them (on_scratch_released). - // The dispatcher calls this after every offline transcribe_run or - // transcribe_run_batch that passes pre-clear validation and reaches its - // commit point, whether family execution succeeds, fails, or throws. - // Without this, a single long utterance would pin the scheduler's - // high-water mark in backend compute memory for the session's lifetime - // (Handy #2000). Families re-create the scheduler lazily on the next run. - // The measured recreation cost is about 1 ms per run on Metal and up to - // about 10 ms on CPU for families that reserve a worst-case decoder - // workspace each run, such as Canary and Cohere. - // - // Host-side vectors that scale with input length, such as mel buffers, - // encoder host copies, and positional banks, deliberately retain their - // capacity. Family KV caches sized by the last batch are also retained; - // this hook releases only the ggml scheduler and compute context. - // - // Streaming entry points do not invoke this hook. A streaming session - // keeps its scheduler until a later offline run or session destruction. - // Parakeet and Voxtral Realtime use per-chunk bounded stream workspaces; - // Moonshine Streaming's decode graph cross-attends over the complete - // committed stream, so its workspace grows with total stream length. - // Must not throw. Non-virtual: a family cannot opt out of the release. - void release_scratch() noexcept; - transcribe_session() = default; - // Frees sched / compute_ctx after the derived destructor has run. The - // scheduler owns only its own allocator buffers and references - // model-owned backends, so freeing it after the family's KV caches and - // stream buffers is order-independent. - virtual ~transcribe_session(); + ~transcribe_session() override; transcribe_session(const transcribe_session &) = delete; transcribe_session & operator=(const transcribe_session &) = delete; transcribe_session(transcribe_session &&) = delete; transcribe_session & operator=(transcribe_session &&) = delete; - - protected: - // Family hook, called by release_scratch after sched / compute_ctx are - // freed: null out tensors borrowed from the freed context (encoder_out) - // or reset capacity bookkeeping (whisper compute_ctx_size). Most - // families need nothing. Not called from the base destructor because - // derived members are already destroyed by then. Must not throw. - virtual void on_scratch_released() noexcept {} }; diff --git a/src/transcribe.cpp b/src/transcribe.cpp index 8f7089c80..940a9004f 100644 --- a/src/transcribe.cpp +++ b/src/transcribe.cpp @@ -1,17 +1,19 @@ -// transcribe.cpp - public C API entry points + central dispatch. +// transcribe.cpp - library-level public C API entry points + model dispatch. // // What lives here: -// - The C entry points declared in include/transcribe.h. -// - The single dispatch site mapping an opened GGUF to its per-family -// Arch handler (transcribe_model_load_file). +// - Status strings, version, and ABI struct-size metadata. // - The log-sink publication / emission helpers. -// - The factory functions that initialize each public params struct. +// - Backend module loading and device discovery. +// - The single dispatch site mapping an opened GGUF to its per-family +// Arch handler (transcribe_model_load_file), model free, and the +// model-level queries (arch / variant / metadata / backend / device, +// extension-kind and feature probes). +// - The init functions for the params / output structs shared by every +// role (model load params, device info, timings, speaker segments). // -// What does NOT live here: GGUF reading (transcribe-loader), per-family -// load / context / run code (src/arch//...), and ggml-specific -// code. The introspection accessors read the base structs in -// transcribe-model.h / transcribe-session.h directly; per-family code owns -// load(), init_context(), and run(). +// What does NOT live here: the ASR session API (sessions, run, batch, +// streaming, results; transcribe-asr.cpp), GGUF reading (transcribe-loader), +// and per-family load / context / run code (src/arch//...). #include "transcribe.h" @@ -19,17 +21,14 @@ #include "ggml-backend.h" #include "ggml.h" // ggml_log_set: route ggml diagnostics into our sink #include "transcribe-abi.h" +#include "transcribe-api-guard.h" #include "transcribe-arch.h" #include "transcribe-backend.h" -#include "transcribe-batch-util.h" #include "transcribe-loader.h" #include "transcribe-log.h" #include "transcribe-model.h" #include "transcribe-path.h" -#include "transcribe-prompting.h" -#include "transcribe-session.h" -#include "transcribe-tokenizer.h" -#include "transcribe/whisper.h" +#include "transcribe/diarize.h" #if defined(TRANSCRIBE_GGML_BACKEND_DL) && defined(_WIN32) # ifndef WIN32_LEAN_AND_MEAN @@ -46,73 +45,23 @@ #include #include #include -#include -#include #include #include #include #include #include #include -#include #include #include #include #include -#include #include #include -#include - -// Public C ABI entry points must not leak C++ exceptions. The guarded -// paths below map bad_alloc to OOM, other exceptions to BACKEND, and make -// teardown best-effort/no-throw. MSVC builds use /EHs so exceptions thrown -// by ggml C-ABI frames remain catchable here. - -namespace { - -template transcribe_status api_guard_status(const char * fn_name, Fn && body) { - try { - return body(); - } catch (const std::bad_alloc &) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: out of memory", fn_name); - return TRANSCRIBE_ERR_OOM; - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); - return TRANSCRIBE_ERR_BACKEND; - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); - return TRANSCRIBE_ERR_BACKEND; - } -} - -// Value-returning variant: `on_throw` is the entry point's documented -// hard-failure value (0 devices, false, INT_MIN, ...). -template T api_guard_value(const char * fn_name, T on_throw, Fn && body) { - try { - return body(); - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught backend exception: %s", fn_name, e.what()); - return on_throw; - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "%s: caught unknown backend exception", fn_name); - return on_throw; - } -} - -// Teardown variant: log and swallow. Frees must never fail. -template void api_guard_void(const char * fn_name, Fn && body) { - try { - body(); - } catch (const std::exception & e) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught backend exception during teardown: %s", fn_name, - e.what()); - } catch (...) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, "%s: caught unknown backend exception during teardown", fn_name); - } -} -} // namespace +using transcribe::api_guard_status; +using transcribe::api_guard_value; +using transcribe::api_guard_void; +using transcribe::enum_field_raw; // Status @@ -161,6 +110,8 @@ extern "C" const char * transcribe_status_string(int status) { return "output truncated: decode hit the context/generation cap before end-of-stream"; case TRANSCRIBE_ERR_OUTPUT_REPETITION: return "output repetition: decode stopped when the output began repeating itself"; + case TRANSCRIBE_ERR_UNSUPPORTED_ROLE: + return "model does not serve the requested role"; default: return "unknown status"; } @@ -183,23 +134,6 @@ extern "C" const char * transcribe_version_commit(void) { return TRANSCRIBE_COMMIT; } -// Raw enum reads at the public ABI boundary -// -// C callers can store ANY int in an enum-typed ABI field; in C++ loading an -// out-of-range value through an enum-typed lvalue is UB (UBSan traps it). So -// every public entry point reads caller-supplied enum values as raw bytes -// first, validates the raw int, and only then converts to the enum type. All -// public enums are int-sized; the static_assert below pins one. - -static inline int enum_field_raw(const void * field) { - int raw; - std::memcpy(&raw, field, sizeof(raw)); - return raw; -} - -static_assert(sizeof(transcribe_backend_request) == sizeof(int), - "public enums must be int-sized for raw boundary reads"); - // ABI metadata: sizeof/alignof for the public structs, so a binding can // verify its layout against this build. Keep these switches in sync with the // transcribe_abi_struct enum. @@ -238,6 +172,12 @@ extern "C" size_t transcribe_abi_struct_size(transcribe_abi_struct which) { return sizeof(struct transcribe_speaker_segment); case TRANSCRIBE_ABI_BACKEND_INIT_PARAMS: return sizeof(struct transcribe_backend_init_params); + case TRANSCRIBE_ABI_DIARIZE_INFO: + return sizeof(struct transcribe_diarize_info); + case TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: + return sizeof(struct transcribe_diarize_session_params); + case TRANSCRIBE_ABI_DIARIZE_PARAMS: + return sizeof(struct transcribe_diarize_params); } return 0; // unknown id: "cannot verify", never a real size } @@ -276,262 +216,16 @@ extern "C" size_t transcribe_abi_struct_align(transcribe_abi_struct which) { return alignof(struct transcribe_speaker_segment); case TRANSCRIBE_ABI_BACKEND_INIT_PARAMS: return alignof(struct transcribe_backend_init_params); + case TRANSCRIBE_ABI_DIARIZE_INFO: + return alignof(struct transcribe_diarize_info); + case TRANSCRIBE_ABI_DIARIZE_SESSION_PARAMS: + return alignof(struct transcribe_diarize_session_params); + case TRANSCRIBE_ABI_DIARIZE_PARAMS: + return alignof(struct transcribe_diarize_params); } return 0; } -namespace { - -// Ordered rank for timestamp granularities (NONE < SEGMENT < WORD < TOKEN), -// used to compare a request against a family's advertised maximum. AUTO and -// any unknown value get rank -1. -int timestamp_rank(transcribe_timestamp_kind k) { - switch (k) { - case TRANSCRIBE_TIMESTAMPS_NONE: - return 0; - case TRANSCRIBE_TIMESTAMPS_SEGMENT: - return 1; - case TRANSCRIBE_TIMESTAMPS_WORD: - return 2; - case TRANSCRIBE_TIMESTAMPS_TOKEN: - return 3; - case TRANSCRIBE_TIMESTAMPS_AUTO: - return -1; - } - return -1; -} - -// Shared run-params validation used by transcribe_run and -// transcribe_stream_begin. Both call sites have already validated -// that session, session->model, and params are non-null. The translate-task -// rejection differs between the two (run mirrors supports_translate, -// streaming-begin rejects unconditionally in v1), so each caller -// applies its own translate check before reaching this helper. -transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params); - -transcribe_status validate_run_params_common(const transcribe_session * session, const transcribe_run_params * params) { - // Raw-validate every enum field before its first enum-typed load (see - // enum_field_raw). Once a field passes here, downstream typed reads — - // including the per-family handlers' — are defined. - switch (enum_field_raw(¶ms->task)) { - case TRANSCRIBE_TASK_TRANSCRIBE: - case TRANSCRIBE_TASK_TRANSLATE: - case TRANSCRIBE_TASK_INSTRUCT: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->timestamps)) { - case TRANSCRIBE_TIMESTAMPS_NONE: - case TRANSCRIBE_TIMESTAMPS_AUTO: - case TRANSCRIBE_TIMESTAMPS_SEGMENT: - case TRANSCRIBE_TIMESTAMPS_WORD: - case TRANSCRIBE_TIMESTAMPS_TOKEN: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->pnc)) { - case TRANSCRIBE_PNC_MODE_DEFAULT: - case TRANSCRIBE_PNC_MODE_OFF: - case TRANSCRIBE_PNC_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->itn)) { - case TRANSCRIBE_ITN_MODE_DEFAULT: - case TRANSCRIBE_ITN_MODE_OFF: - case TRANSCRIBE_ITN_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - switch (enum_field_raw(¶ms->diarize)) { - case TRANSCRIBE_DIARIZE_MODE_DEFAULT: - case TRANSCRIBE_DIARIZE_MODE_OFF: - case TRANSCRIBE_DIARIZE_MODE_ON: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { - const int req_rank = timestamp_rank(params->timestamps); - const int max_rank = timestamp_rank(session->model->caps.max_timestamp_kind); - if (req_rank > max_rank) { - return TRANSCRIBE_ERR_UNSUPPORTED_TIMESTAMPS; - } - } - if (params->language != nullptr && session->model->caps.n_languages > 0 && - session->model->caps.languages != nullptr) { - bool found = false; - for (int i = 0; i < session->model->caps.n_languages; ++i) { - const char * entry = session->model->caps.languages[i]; - if (entry != nullptr && std::strcmp(entry, params->language) == 0) { - found = true; - break; - } - } - if (!found) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - } - // Translation-target gate: for a TRANSLATE request naming a target - // language, reject up front when the model advertises a target set - // (n > 0) that does not include it. Mirrors the source-language check - // above and returns the same code. An empty/unadvertised set (old - // GGUFs, ASR-only models) makes this inert; family-level target/pair - // checks (e.g. canary's pivot pairs) still apply on top. - if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->target_language != nullptr && - session->model->caps.n_translate_target_languages > 0 && - session->model->caps.translate_target_languages != nullptr) { - bool found = false; - for (int i = 0; i < session->model->caps.n_translate_target_languages; ++i) { - const char * entry = session->model->caps.translate_target_languages[i]; - if (entry != nullptr && std::strcmp(entry, params->target_language) == 0) { - found = true; - break; - } - } - if (!found) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - } - // Translation-pair gate: when a GGUF advertises exact src>dst pairs and - // the caller supplied both sides, reject unsupported directions before - // family dispatch. A missing source hint keeps this inert because most - // families do not perform generic language detection here. - if (params->task == TRANSCRIBE_TASK_TRANSLATE && params->language != nullptr && - params->target_language != nullptr && - !session->model->allows_translation_pair(params->language, params->target_language)) { - return TRANSCRIBE_ERR_UNSUPPORTED_LANGUAGE; - } - return validate_prompting(session->model, params); -} - -// Shape and hard-gate checks for the generic prompting fields, on a -// normalized view. Soft inputs a model ignores are removed later by -// prepare_prompting. -transcribe_status validate_prompting(const transcribe_model * model, const transcribe_run_params * params) { - auto reject = [](transcribe_status st, const char * why) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_run: %s", why); - return st; - }; - if (params->n_vocabulary < 0 || (params->n_vocabulary > 0 && params->vocabulary == nullptr)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary is NULL or n_vocabulary is negative"); - } - for (int32_t i = 0; i < params->n_vocabulary; ++i) { - if (params->vocabulary[i] == nullptr) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "vocabulary has a NULL entry"); - } - } - const bool has_prefix = transcribe::prompting::has_text(params->prefix); - if (params->task == TRANSCRIBE_TASK_INSTRUCT) { - if (!transcribe::has_feature(model, TRANSCRIBE_FEATURE_INSTRUCT)) { - return reject(TRANSCRIBE_ERR_UNSUPPORTED_TASK, - "this model does not support TRANSCRIBE_TASK_INSTRUCT (TRANSCRIBE_FEATURE_INSTRUCT)"); - } - // Output is free text: no target language and no alignment. A prefix - // as answer prefill is untested on every INSTRUCT family. - if (!transcribe::prompting::has_text(params->prompt)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT requires a non-empty prompt"); - } - if (params->target_language != nullptr) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT does not take a target_language"); - } - if (params->timestamps != TRANSCRIBE_TIMESTAMPS_NONE && params->timestamps != TRANSCRIBE_TIMESTAMPS_AUTO) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, "TRANSCRIBE_TASK_INSTRUCT supports timestamps NONE or AUTO only"); - } - if (has_prefix) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, - "a transcript prefix is not supported with TRANSCRIBE_TASK_INSTRUCT"); - } - } - // Ignoring a prefix would make the output repeat the prefix's words and - // silently break callers that stitch text together, so it is a hard gate. - if (has_prefix && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)) { - return reject(TRANSCRIBE_ERR_INVALID_ARG, - "this model does not support a transcript prefix (TRANSCRIBE_FEATURE_TRANSCRIPT_PREFIX)"); - } - return TRANSCRIBE_OK; -} - -// Rejects control-token literals in the prompting text before the result -// snapshot is cleared. Runs after strip_ignored_prompting, so ignored inputs -// are not rejected. -transcribe_status check_prompting_text(const transcribe_model * model, const transcribe_run_params * params) { - const transcribe::Tokenizer * tok = model->tokenizer(); - if (tok == nullptr) { - return TRANSCRIBE_OK; - } - for (int32_t i = 0; i < params->n_vocabulary; ++i) { - if (const transcribe_status st = - transcribe::prompting::check_plain_text(*tok, params->vocabulary[i], "vocabulary"); - st != TRANSCRIBE_OK) { - return st; - } - } - if (params->prompt != nullptr) { - if (const transcribe_status st = transcribe::prompting::check_plain_text(*tok, params->prompt, "prompt"); - st != TRANSCRIBE_OK) { - return st; - } - } - if (params->prefix != nullptr) { - return transcribe::prompting::check_plain_text(*tok, params->prefix, "prefix"); - } - return TRANSCRIBE_OK; -} - -// Full-size copy of a caller's run params: defaults first, then only the -// prefix the caller's struct_size covers, so every trailing field is -// readable (NULL/0 for an older caller). struct_size is preserved so -// has_field() gating still sees the caller's true layout. Idempotent. -void normalize_run_params(const transcribe_run_params * in, transcribe_run_params * out) { - transcribe_run_params_init(out); - std::memcpy(out, in, static_cast(std::min(in->struct_size, sizeof(*out)))); -} - -// Warn about, then remove, the soft prompting inputs this model ignores, so -// a family only ever sees inputs it should act on. Idempotent: the batch -// serial fallback re-enters run_one_inner per utterance. -void strip_ignored_prompting(const transcribe_model * model, transcribe_run_params * params) { - const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; - const bool instruct = params->task == TRANSCRIBE_TASK_INSTRUCT; - if (params->n_vocabulary > 0 && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_VOCABULARY)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, - "transcribe_run: model '%s' does not support vocabulary; ignoring %d term(s). Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_VOCABULARY) to pre-check.", - arch_name, params->n_vocabulary); - params->vocabulary = nullptr; - params->n_vocabulary = 0; - } - if (!instruct && transcribe::prompting::has_text(params->prompt) && - !transcribe::has_feature(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_WARN, - "transcribe_run: model '%s' has no context-prompt slot; ignoring prompt. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_CONTEXT_PROMPT) to pre-check, or " - "TRANSCRIBE_TASK_INSTRUCT on models with TRANSCRIBE_FEATURE_INSTRUCT.", - arch_name); - params->prompt = nullptr; - } - if (!transcribe::prompting::has_text(params->prompt)) { - params->prompt = nullptr; - } - if (!transcribe::prompting::has_text(params->prefix)) { - params->prefix = nullptr; - } -} - -// The pre-clear prompting step every entry point shares, on a validated -// normalized view. -transcribe_status prepare_prompting(const transcribe_model * model, transcribe_run_params * params) { - strip_ignored_prompting(model, params); - return check_prompting_text(model, params); -} - -} // namespace - // Logging // // Two-atomic log sink. Publication stores userdata (relaxed) then cb @@ -692,39 +386,7 @@ void log_msg(transcribe_log_level level, const char * fmt, ...) { // memory could hit the zero case); callers reach defaults via NULL only. extern "C" void transcribe_model_load_params_init(struct transcribe_model_load_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_session_params_init(struct transcribe_session_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_run_params_init(struct transcribe_run_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - p->spec_k_drafts = -1; // family default - // Default to AUTO (richest output compatible with the model and selected - // run tasks, resolved per-family) rather than the memset NONE. - p->timestamps = TRANSCRIBE_TIMESTAMPS_AUTO; -} - -extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } // Output struct init functions @@ -735,101 +397,25 @@ extern "C" void transcribe_stream_params_init(struct transcribe_stream_params * // struct_size is the correct empty state. struct_size == 0 is rejected by // the accessors as a "you forgot to init the buffer" error. -extern "C" void transcribe_capabilities_init(struct transcribe_capabilities * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_session_limits_init(struct transcribe_session_limits * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - extern "C" void transcribe_timings_init(struct transcribe_timings * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_stream_update_init(struct transcribe_stream_update * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_stream_text_init(struct transcribe_stream_text * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_segment_init(struct transcribe_segment * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_device_info_init(struct transcribe_device_info * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } extern "C" void transcribe_backend_init_params_init(struct transcribe_backend_init_params * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); - p->allowed_backends = TRANSCRIBE_BACKEND_MASK_ALL; -} - -extern "C" void transcribe_word_init(struct transcribe_word * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); -} - -extern "C" void transcribe_token_init(struct transcribe_token * p) { - if (p == nullptr) { - return; + transcribe::init_sized(p); + if (p != nullptr) { + p->allowed_backends = TRANSCRIBE_BACKEND_MASK_ALL; } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); } extern "C" void transcribe_speaker_segment_init(struct transcribe_speaker_segment * p) { - if (p == nullptr) { - return; - } - std::memset(p, 0, sizeof(*p)); - p->struct_size = sizeof(*p); + transcribe::init_sized(p); } -// Whisper telemetry + run-extension init and chunk-trace accessors live in -// arch/whisper/public.cpp so this dispatcher stays family-agnostic. Whisper -// run knobs are in transcribe_whisper_run_ext (via run_params::family); -// sensevoice/funasr_nano use_itn and canary pnc use the generic run_params -// itn/pnc enums. - // Extension helpers extern "C" transcribe_status transcribe_ext_check(const struct transcribe_ext * ext, @@ -871,74 +457,16 @@ extern "C" bool transcribe_model_supports(const struct transcribe_model * model, namespace { -// Minimum struct_size accepted on each caller-owned input/output struct. -// Sized to the prefix the library currently relies on: any field the -// library writes on a given call path must lie inside this prefix. New -// fields appended at the end of the public struct without growing the -// library-side prefix do NOT raise this value. -#define TRANSCRIBE_FIELD_END(type, field) (offsetof(type, field) + sizeof(((type *) 0)->field)) - -constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); -constexpr size_t k_min_context_params_size = TRANSCRIBE_FIELD_END(transcribe_session_params, kv_type); -// run_params is the one 0.2.0 exception to the append-only rule: `diarize` -// was inserted mid-struct, shifting every field from `language` on by 8 -// bytes. A 0.1-layout caller's sizeof (64) equals FIELD_END(family) in the -// NEW layout, so a `family`-prefix minimum would admit stale callers and -// read their pointers as enums. Require through spec_k_drafts so every -// pre-0.2 caller that bypasses the SONAME/abihash checks (dlopen by path, -// stale static link, hand-rolled FFI) gets BAD_STRUCT_SIZE instead. -constexpr size_t k_min_run_params_size = TRANSCRIBE_FIELD_END(transcribe_run_params, spec_k_drafts); -constexpr size_t k_min_stream_params_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, family); -constexpr size_t k_stream_params_commit_policy_size = TRANSCRIBE_FIELD_END(transcribe_stream_params, commit_policy); -constexpr size_t k_stream_params_agreement_n_size = - TRANSCRIBE_FIELD_END(transcribe_stream_params, stable_prefix_agreement_n); -constexpr size_t k_min_stream_update_size = TRANSCRIBE_FIELD_END(transcribe_stream_update, buffered_ms); -constexpr size_t k_stream_update_committed_changed_size = - TRANSCRIBE_FIELD_END(transcribe_stream_update, committed_changed); -constexpr size_t k_stream_update_tentative_changed_size = - TRANSCRIBE_FIELD_END(transcribe_stream_update, tentative_changed); -constexpr size_t k_min_stream_text_size = TRANSCRIBE_FIELD_END(transcribe_stream_text, raw_tentative_start_bytes); -constexpr size_t k_min_capabilities_size = TRANSCRIBE_FIELD_END(transcribe_capabilities, supports_streaming); -constexpr size_t k_min_session_limits_size = TRANSCRIBE_FIELD_END(transcribe_session_limits, max_kv_bytes); -constexpr size_t k_min_segment_size = TRANSCRIBE_FIELD_END(transcribe_segment, text); -constexpr size_t k_min_word_size = TRANSCRIBE_FIELD_END(transcribe_word, text); -constexpr size_t k_min_token_size = TRANSCRIBE_FIELD_END(transcribe_token, text); -constexpr size_t k_min_speaker_segment_size = TRANSCRIBE_FIELD_END(transcribe_speaker_segment, p); -constexpr size_t k_min_timings_size = TRANSCRIBE_FIELD_END(transcribe_timings, decode_ms); -constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); +constexpr size_t k_min_model_params_size = TRANSCRIBE_FIELD_END(transcribe_model_load_params, device); +constexpr size_t k_min_device_info_size = TRANSCRIBE_FIELD_END(transcribe_device_info, kind); + constexpr size_t k_min_backend_init_params_size = TRANSCRIBE_FIELD_END(transcribe_backend_init_params, allowed_backends); -// k_min_whisper_chunk_trace_size lives in arch/whisper/public.cpp with -// the chunk-trace accessor that uses it. - -#undef TRANSCRIBE_FIELD_END -// Size-aware ABI helpers (check_struct_size / check_input_struct_size / -// copy_out_prefix) live in transcribe-abi.h so per-family public -// accessors (arch/whisper/public.cpp) share one definition. Pull them -// into this TU's unqualified scope so existing call sites are unchanged. using transcribe::check_input_struct_size; using transcribe::check_struct_size; using transcribe::copy_out_prefix; -static bool has_field(uint64_t struct_size, size_t field_end) { - return struct_size >= static_cast(field_end); -} - -// Takes the RAW integer, not the enum: a C caller can store any int in the -// struct's enum-typed field, and in C++ loading an out-of-range value through -// the enum lvalue is UB (UBSan trips on it). Callers memcpy the bytes into an -// int and validate before the first enum-typed load. -static bool valid_stream_commit_policy(int policy) { - switch (policy) { - case TRANSCRIBE_STREAM_COMMIT_AUTO: - case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: - case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: - return true; - } - return false; -} - } // namespace // Backend modules and device discovery @@ -1424,425 +952,88 @@ static bool transcribe_backend_available_impl(int raw) { return false; } -namespace { - -enum class StreamStablePrefixImpl { - GenericTextAgreement, - FamilyTokenAgreement, - FamilyNativeCommit, -}; - -static StreamStablePrefixImpl stream_stable_prefix_impl_for_arch(const transcribe::Arch * arch) { - // Single source of truth for the stable-prefix implementation used by - // AUTO and explicit STABLE_PREFIX. Add new streaming families here - // deliberately; otherwise they use the generic text-agreement fallback. - struct Entry { - const char * arch_name; - StreamStablePrefixImpl impl; - }; - - static constexpr Entry k_defaults[] = { - // Parakeet publishes native committed chunks; agreement_n does not - // add useful evidence on top of the family-provided boundary. - { "parakeet", StreamStablePrefixImpl::FamilyNativeCommit }, - // Moonshine re-decodes the full prefix and its family boundary is - // token-id agreement; the family applies stable_prefix_agreement_n. - { "moonshine_streaming", StreamStablePrefixImpl::FamilyTokenAgreement }, - }; +// Lifecycle - const char * name = arch != nullptr && arch->name != nullptr ? arch->name : ""; - for (const Entry & entry : k_defaults) { - if (std::strcmp(name, entry.arch_name) == 0) { - return entry.impl; - } +static transcribe_status transcribe_model_load_file_impl(const char * path, + const struct transcribe_model_load_params * params, + struct transcribe_model ** out_model) { + // Set up ggml's timer before any family's load-timing ggml_time_us() runs + // (Windows/MSVC ÷0 otherwise — see ensure_ggml_time_init). + ensure_ggml_time_init(); + if (out_model == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; } - return StreamStablePrefixImpl::GenericTextAgreement; -} - -static size_t utf8_floor_boundary(const std::string & s, size_t pos) { - if (pos >= s.size()) { - return s.size(); + *out_model = nullptr; + if (path == nullptr) { + return TRANSCRIBE_ERR_INVALID_ARG; } - while (pos > 0 && (static_cast(s[pos]) & 0xC0u) == 0x80u) { - --pos; + // NULL params means "all defaults". This is the version-proof + // spelling of defaults — it carries no struct_size, so it always + // matches the running library. A zeroed struct ({0}) is a different + // thing: it claims struct_size == 0 and is rejected below. + struct transcribe_model_load_params params_defaults; + transcribe_model_load_params_init(¶ms_defaults); + if (params == nullptr) { + params = ¶ms_defaults; } - return pos; -} - -static size_t common_prefix_bytes(const std::string & a, const std::string & b) { - const size_t n = std::min(a.size(), b.size()); - size_t i = 0; - while (i < n && a[i] == b[i]) { - ++i; + if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { + return st; } - return utf8_floor_boundary(a, i); -} -static bool starts_with_bytes(const std::string & s, const std::string & prefix) { - return s.size() >= prefix.size() && s.compare(0, prefix.size(), prefix) == 0; -} + // device is validated where the registry is available, in + // load_common::init_backends. NULL means automatic selection; a non-NULL + // handle selects that exact registered device or fails. -static size_t token_prefix_raw_bytes(const transcribe_session * session, int n_tokens) { - if (session == nullptr || n_tokens <= 0 || session->tokens.empty()) { - return 0; - } - const int capped = std::min(n_tokens, static_cast(session->tokens.size())); - std::string prefix; - prefix.reserve(session->full_text.size()); - for (int i = 0; i < capped; ++i) { - prefix += session->tokens[static_cast(i)].text; + // Raw-validate the backend request before the families' first + // enum-typed load of it (see enum_field_raw). init_backends re-checks + // as defense in depth; this boundary check is what makes every + // downstream `params->backend` read defined. + switch (enum_field_raw(¶ms->backend)) { + case TRANSCRIBE_BACKEND_AUTO: + case TRANSCRIBE_BACKEND_CPU: + case TRANSCRIBE_BACKEND_METAL: + case TRANSCRIBE_BACKEND_VULKAN: + case TRANSCRIBE_BACKEND_CPU_ACCEL: + case TRANSCRIBE_BACKEND_CUDA: + case TRANSCRIBE_BACKEND_ROCM: + break; + default: + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: invalid backend request %d", + enum_field_raw(¶ms->backend)); + return TRANSCRIBE_ERR_INVALID_ARG; } - if (starts_with_bytes(session->full_text, prefix)) { - return prefix.size(); + + // Format sniff by reading the first four bytes (not the file extension, + // which HF/users mislabel): GGUF magic 0x46554747 is the canonical path, + // ggml magic 0x67676d6c is a legacy whisper.cpp .bin. Public contract: + // missing path -> ERR_FILE_NOT_FOUND; every other failure -> ERR_GGUF. + if (!transcribe::path_is_present(path)) { + return TRANSCRIBE_ERR_FILE_NOT_FOUND; } - if (!prefix.empty() && prefix.front() == ' ' && (session->full_text.empty() || session->full_text.front() != ' ')) { - const std::string normalized_prefix = prefix.substr(1); - if (starts_with_bytes(session->full_text, normalized_prefix)) { - return normalized_prefix.size(); + uint32_t magic = 0; + { + std::ifstream fin(transcribe::path_from_utf8(path), std::ios::binary); + if (!fin) { + // The existence pre-check said the path is reachable but + // the open failed — permissions / type issue. Treat as a generic + // load failure rather than FILE_NOT_FOUND. + return TRANSCRIBE_ERR_GGUF; + } + fin.read(reinterpret_cast(&magic), sizeof(magic)); + if (!fin || fin.gcount() != sizeof(magic)) { + transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, + "transcribe_model_load_file: short read on file " + "magic for %s", + path); + return TRANSCRIBE_ERR_GGUF; } - return common_prefix_bytes(session->full_text, normalized_prefix); } - return common_prefix_bytes(session->full_text, prefix); -} -static size_t family_candidate_raw_prefix_bytes(const transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - if (session->n_committed_tokens > 0) { - return token_prefix_raw_bytes(session, session->n_committed_tokens); - } - if (session->n_committed_words > 0 && session->n_committed_words >= static_cast(session->words.size())) { - return session->full_text.size(); - } - if (session->n_committed_segments > 0 && - session->n_committed_segments >= static_cast(session->segments.size())) { - return session->full_text.size(); - } - return 0; -} - -static size_t generic_text_stable_prefix_candidate_raw_bytes(transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - // Library default when the caller leaves stable_prefix_agreement_n at - // 0; documented as "currently 3" on transcribe_stream_params. - static constexpr uint32_t k_default_stable_prefix_agreement_n = 3; - uint32_t agreement_n = session->stream_stable_prefix_agreement_n; - if (agreement_n == 0) { - agreement_n = k_default_stable_prefix_agreement_n; - } - if (agreement_n <= 1) { - return session->full_text.size(); - } - - session->stream_raw_history.push_back(session->full_text); - while (session->stream_raw_history.size() > agreement_n) { - session->stream_raw_history.pop_front(); - } - if (session->stream_raw_history.size() < agreement_n) { - return 0; - } - - size_t prefix_n = session->stream_raw_history.front().size(); - for (const auto & text : session->stream_raw_history) { - prefix_n = std::min(prefix_n, common_prefix_bytes(session->stream_raw_history.front(), text)); - } - return utf8_floor_boundary(session->full_text, prefix_n); -} - -static size_t selected_stable_prefix_candidate_raw_bytes(transcribe_session * session) { - if (session == nullptr) { - return 0; - } - const transcribe::Arch * arch = session->model != nullptr ? session->model->arch : nullptr; - switch (stream_stable_prefix_impl_for_arch(arch)) { - case StreamStablePrefixImpl::FamilyTokenAgreement: - case StreamStablePrefixImpl::FamilyNativeCommit: - return family_candidate_raw_prefix_bytes(session); - case StreamStablePrefixImpl::GenericTextAgreement: - return generic_text_stable_prefix_candidate_raw_bytes(session); - } - return 0; -} - -static bool append_committed_raw_prefix(transcribe_session * session, size_t candidate_bytes) { - if (session == nullptr || !session->has_result) { - return false; - } - const std::string & raw = session->full_text; - const size_t old_boundary = utf8_floor_boundary( - raw, std::min(static_cast(session->stream_raw_tentative_start_bytes), raw.size())); - const size_t candidate = utf8_floor_boundary(raw, std::min(candidate_bytes, raw.size())); - if (candidate <= old_boundary) { - return false; - } - if (old_boundary != session->stream_committed_text.size()) { - return false; - } - if (raw.compare(0, old_boundary, session->stream_committed_text.data(), old_boundary) != 0) { - return false; - } - session->stream_committed_text.append(raw.data() + old_boundary, candidate - old_boundary); - session->stream_raw_tentative_start_bytes = static_cast(candidate); - return true; -} - -static bool finalize_committed_text(transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return false; - } - if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE || - session->stream_committed_text.empty()) { - const bool changed = session->stream_committed_text != session->full_text; - session->stream_committed_text = session->full_text; - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return changed; - } - if (!starts_with_bytes(session->full_text, session->stream_committed_text)) { - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return false; - } - const size_t old_n = session->stream_committed_text.size(); - if (session->full_text.size() > old_n) { - session->stream_committed_text.append(session->full_text.data() + old_n, session->full_text.size() - old_n); - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return true; - } - session->stream_raw_tentative_start_bytes = static_cast(session->full_text.size()); - return false; -} - -struct StreamTextDelta { - bool committed_changed = false; - bool tentative_changed = false; -}; - -static StreamTextDelta apply_stream_text_policy(transcribe_session * session, bool is_finalize) { - StreamTextDelta delta; - if (session == nullptr) { - return delta; - } - const std::string prev_committed = session->stream_committed_text; - const std::string prev_tentative = session->stream_tentative_text; - - if (is_finalize) { - delta.committed_changed = finalize_committed_text(session); - session->stream_tentative_text.clear(); - } else if (session->has_result) { - size_t candidate = 0; - switch (session->stream_commit_policy) { - case TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE: - candidate = 0; - break; - case TRANSCRIBE_STREAM_COMMIT_STABLE_PREFIX: - case TRANSCRIBE_STREAM_COMMIT_AUTO: - default: - candidate = selected_stable_prefix_candidate_raw_bytes(session); - break; - } - delta.committed_changed = append_committed_raw_prefix(session, candidate); - - const size_t boundary = utf8_floor_boundary( - session->full_text, std::min(static_cast(session->stream_raw_tentative_start_bytes), - session->full_text.size())); - session->stream_tentative_text.assign(session->full_text.data() + boundary, - session->full_text.size() - boundary); - } else { - session->stream_tentative_text.clear(); - } - - delta.committed_changed = delta.committed_changed || session->stream_committed_text != prev_committed; - delta.tentative_changed = session->stream_tentative_text != prev_tentative; - return delta; -} - -static void publish_stream_update_tail(transcribe_stream_update * update, - bool committed_changed, - bool tentative_changed) { - if (update == nullptr) { - return; - } - if (has_field(update->struct_size, k_stream_update_committed_changed_size)) { - update->committed_changed = committed_changed; - } - if (has_field(update->struct_size, k_stream_update_tentative_changed_size)) { - update->tentative_changed = tentative_changed; - } -} - -// Compute the observable-change verdict, advance the revision counter -// when the result moved, and publish revision / result_changed / -// committed_changed / tentative_changed onto the caller's update. Shared -// verbatim by feed and finalize so the result_changed semantics live in -// exactly one place; the path-specific work (ON_FINALIZE counter zeroing -// and audio_committed_ms on feed, is_final and the lifecycle transition -// on finalize) stays in the callers. -static void publish_observable_delta(transcribe_session * session, - transcribe_stream_update * update, - int32_t prev_revision, - const std::string & prev_full_text, - bool prev_has_result, - const StreamTextDelta & text_delta) { - const bool raw_changed = session->has_result != prev_has_result || session->full_text != prev_full_text; - const bool observable_changed = raw_changed || text_delta.committed_changed || text_delta.tentative_changed || - (update != nullptr && update->result_changed); - if (observable_changed && session->stream_revision == prev_revision) { - session->stream_revision += 1; - } - if (update != nullptr) { - update->result_changed = - update->result_changed || observable_changed || session->stream_revision != prev_revision; - update->revision = session->stream_revision; - publish_stream_update_tail(update, text_delta.committed_changed, text_delta.tentative_changed); - } -} - -// Advisory pnc/itn warning. Emits a WARN when a non-DEFAULT request hits a -// model that does not support runtime control of that axis, then returns so -// the dispatcher proceeds with best-effort semantics. The reserved -// TRANSCRIBE_ERR_UNSUPPORTED_PNC / _ITN codes are NOT returned today -// (placeholders for a future opt-in strict mode). The message includes the -// arch + variant strings to pinpoint which model dropped the request. -void warn_unsupported_advisory(const struct transcribe_model * model, const struct transcribe_run_params * rp) { - if (model == nullptr || rp == nullptr) { - return; - } - const char * arch_name = (model->arch != nullptr && model->arch->name != nullptr) ? model->arch->name : "(unknown)"; - const char * variant = model->variant.c_str(); - if (variant == nullptr || variant[0] == '\0') { - variant = "(unknown)"; - } - - // Defense in depth: keep raw reads here even though every dispatcher - // validates before warning. A future call-site reorder must not turn a - // malformed C enum into a typed C++ load. - const int pnc_raw = enum_field_raw(&rp->pnc); - const int itn_raw = enum_field_raw(&rp->itn); - const int diarize_raw = enum_field_raw(&rp->diarize); - - char buf[512]; - if (pnc_raw != TRANSCRIBE_PNC_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_PNC)) { - const char * req = (pnc_raw == TRANSCRIBE_PNC_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested pnc=%s but model '%s' " - "(variant '%s') does not support pnc control; output will use " - "the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_PNC) to " - "pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } - if (itn_raw != TRANSCRIBE_ITN_MODE_DEFAULT && !transcribe::has_feature(model, TRANSCRIBE_FEATURE_ITN)) { - const char * req = (itn_raw == TRANSCRIBE_ITN_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested itn=%s but model '%s' " - "(variant '%s') does not support itn control; output will use " - "the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_ITN) to " - "pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } - if (diarize_raw != TRANSCRIBE_DIARIZE_MODE_DEFAULT && - !transcribe::has_feature(model, TRANSCRIBE_FEATURE_DIARIZATION)) { - const char * req = (diarize_raw == TRANSCRIBE_DIARIZE_MODE_ON) ? "ON" : "OFF"; - std::snprintf(buf, sizeof(buf), - "transcribe_run: caller requested diarize=%s but model '%s' " - "(variant '%s') does not support diarization control; output " - "will use the model's default behavior. Use " - "transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION) " - "to pre-check.", - req, arch_name, variant); - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_WARN, buf); - } -} - -} // namespace - -// Lifecycle - -static transcribe_status transcribe_model_load_file_impl(const char * path, - const struct transcribe_model_load_params * params, - struct transcribe_model ** out_model) { - // Set up ggml's timer before any family's load-timing ggml_time_us() runs - // (Windows/MSVC ÷0 otherwise — see ensure_ggml_time_init). - ensure_ggml_time_init(); - if (out_model == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_model = nullptr; - if (path == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults". This is the version-proof - // spelling of defaults — it carries no struct_size, so it always - // matches the running library. A zeroed struct ({0}) is a different - // thing: it claims struct_size == 0 and is rejected below. - struct transcribe_model_load_params params_defaults; - transcribe_model_load_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_model_params_size); st != TRANSCRIBE_OK) { - return st; - } - - // device is validated where the registry is available, in - // load_common::init_backends. NULL means automatic selection; a non-NULL - // handle selects that exact registered device or fails. - - // Raw-validate the backend request before the families' first - // enum-typed load of it (see enum_field_raw). init_backends re-checks - // as defense in depth; this boundary check is what makes every - // downstream `params->backend` read defined. - switch (enum_field_raw(¶ms->backend)) { - case TRANSCRIBE_BACKEND_AUTO: - case TRANSCRIBE_BACKEND_CPU: - case TRANSCRIBE_BACKEND_METAL: - case TRANSCRIBE_BACKEND_VULKAN: - case TRANSCRIBE_BACKEND_CPU_ACCEL: - case TRANSCRIBE_BACKEND_CUDA: - case TRANSCRIBE_BACKEND_ROCM: - break; - default: - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, "transcribe_model_load_file: invalid backend request %d", - enum_field_raw(¶ms->backend)); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Format sniff by reading the first four bytes (not the file extension, - // which HF/users mislabel): GGUF magic 0x46554747 is the canonical path, - // ggml magic 0x67676d6c is a legacy whisper.cpp .bin. Public contract: - // missing path -> ERR_FILE_NOT_FOUND; every other failure -> ERR_GGUF. - if (!transcribe::path_is_present(path)) { - return TRANSCRIBE_ERR_FILE_NOT_FOUND; - } - uint32_t magic = 0; - { - std::ifstream fin(transcribe::path_from_utf8(path), std::ios::binary); - if (!fin) { - // The existence pre-check said the path is reachable but - // the open failed — permissions / type issue. Treat as a generic - // load failure rather than FILE_NOT_FOUND. - return TRANSCRIBE_ERR_GGUF; - } - fin.read(reinterpret_cast(&magic), sizeof(magic)); - if (!fin || fin.gcount() != sizeof(magic)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_model_load_file: short read on file " - "magic for %s", - path); - return TRANSCRIBE_ERR_GGUF; - } - } - - // 0x67676d6c ("ggml" little-endian): legacy whisper.cpp .bin. Hand off - // to the whisper .bin adapter, which validates the hparams as - // whisper-shaped (rejecting unrelated ggml-magic files like Silero VAD). - if (magic == 0x67676d6cu) { - return transcribe::whisper::load_from_bin(path, params, out_model); + // 0x67676d6c ("ggml" little-endian): legacy whisper.cpp .bin. Hand off + // to the whisper .bin adapter, which validates the hparams as + // whisper-shaped (rejecting unrelated ggml-magic files like Silero VAD). + if (magic == 0x67676d6cu) { + return transcribe::whisper::load_from_bin(path, params, out_model); } // Header-only GGUF inspection. The Loader is stack-allocated; if @@ -1877,1005 +1068,24 @@ static transcribe_status transcribe_model_load_file_impl(const char * // here instead of in every per-family handler. `variant` is owned by the // family (it may default it when stt.variant was absent) and stays on its // own accessor; this map is the generic general.* / display surface. - if (st == TRANSCRIBE_OK && out_model != nullptr && *out_model != nullptr) { + if (st == TRANSCRIBE_OK && *out_model != nullptr) { (*out_model)->meta = loader.meta(); } return st; } -static void transcribe_model_free_impl(struct transcribe_model * model) { - // Polymorphic delete: the base has a virtual destructor (anchored - // in transcribe-model.cpp), so this dispatches to the per-family - // subclass destructor and that destructor frees gguf_context, - // weights, etc. Passing NULL is a no-op per the public contract. - delete model; -} - -static transcribe_status transcribe_session_init_impl(struct transcribe_model * model, - const struct transcribe_session_params * params, - struct transcribe_session ** out_session) { - if (out_session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_session = nullptr; - if (model == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults" (see transcribe_model_load_file). - struct transcribe_session_params params_defaults; - transcribe_session_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_context_params_size); st != TRANSCRIBE_OK) { - return st; - } - // n_threads contract from include/transcribe.h: 0 means "library - // picks a sensible default", positive means "use this many." A - // negative value is undefined input, not a documented sentinel — - // reject it here so individual family handlers don't have to - // re-check the same condition. - if (params->n_threads < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // Raw-validate kv_type before the families' first enum-typed load of it - // (see enum_field_raw): an unknown value from a C caller is a clean - // error here, never UB downstream. - switch (enum_field_raw(¶ms->kv_type)) { - case TRANSCRIBE_KV_TYPE_AUTO: - case TRANSCRIBE_KV_TYPE_F32: - case TRANSCRIBE_KV_TYPE_F16: - break; - default: - return TRANSCRIBE_ERR_INVALID_ARG; - } - // n_ctx is a trailing field (appended after kv_type), so only read it - // when the caller's struct_size actually covers it; an older caller's - // smaller struct leaves it at the default 0 = "model max". A negative - // value is undefined input, not a documented sentinel. The family that - // honors n_ctx clamps a too-large value down to the model maximum, so - // only the negative case is rejected here. See include/transcribe.h. - if (has_field(params->struct_size, offsetof(struct transcribe_session_params, n_ctx) + sizeof(params->n_ctx)) && - params->n_ctx < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // The model carries its own arch dispatch pointer. A model that - // came from a load() call always has arch set; the null check is - // defensive against a hypothetical malformed model object. - if (model->arch == nullptr || model->arch->init_context == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - return model->arch->init_context(model, params, out_session); -} - -static void transcribe_session_free_impl(struct transcribe_session * session) { - // Free the session and, if it owns its model (transcribe_open path), - // free the model too. The owns_model flag is set only by - // transcribe_open; sessions created via the two-step - // transcribe_session_init borrow the model and leave it alone here. - // - // Capture before the session is destroyed; reading session->model - // after delete would be use-after-free. - if (session == nullptr) { - return; - } - struct transcribe_model * owned = session->owns_model ? session->model : nullptr; - delete session; - transcribe_model_free(owned); // NULL is a no-op -} - -// Convenience: open / close / get_model -// -// transcribe_open bundles load + session_init and hands back a session that -// owns its model (owns_model = true). transcribe_close is a thin alias for -// transcribe_session_free; both honor owns_model and free the owned model -// after the session. - -static transcribe_status transcribe_open_impl(const char * path, - const struct transcribe_model_load_params * load_params, - const struct transcribe_session_params * session_params, - struct transcribe_session ** out_session) { - if (out_session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - *out_session = nullptr; - - // NULL params are forwarded as-is: the callees already treat NULL as - // "all defaults", so no substitution is needed here. - struct transcribe_model * model = nullptr; - if (const auto st = transcribe_model_load_file(path, load_params, &model); st != TRANSCRIBE_OK) { - return st; - } - - struct transcribe_session * session = nullptr; - if (const auto st = transcribe_session_init(model, session_params, &session); st != TRANSCRIBE_OK) { - // Load succeeded but session init failed: own the cleanup so the - // caller sees the same all-or-nothing contract as the two-step - // API. No partial state leaks. - transcribe_model_free(model); - return st; - } - - // The convenience session owns the model it loaded; either - // transcribe_session_free or transcribe_close (its alias) will free - // both. This is the only place owns_model is set. - session->owns_model = true; - *out_session = session; - return TRANSCRIBE_OK; -} - -extern "C" void transcribe_close(struct transcribe_session * session) { - // Alias for transcribe_session_free, kept for source compatibility - // with the prior split-API shape. Both honor owns_model. - transcribe_session_free(session); -} - -extern "C" const struct transcribe_model * transcribe_get_model(const struct transcribe_session * session) { - if (session == nullptr) { - return nullptr; - } - return session->model; -} - -extern "C" void transcribe_set_abort_callback(struct transcribe_session * session, - transcribe_abort_callback cb, - void * user_data) { - if (session == nullptr) { - return; - } - session->abort_cb = cb; - session->abort_userdata = user_data; -} - -extern "C" bool transcribe_was_aborted(const struct transcribe_session * session) { - if (session == nullptr) { - return false; - } - return session->was_aborted; -} - -extern "C" bool transcribe_was_truncated(const struct transcribe_session * session) { - if (session == nullptr) { - return false; - } - return session->was_truncated; -} - -// Streaming dispatcher -// -// State transitions are managed entirely here; hooks see ACTIVE on entry to -// begin/feed/finalize and never observe transitions. Hooks own the -// per-utterance result data (tokens/words/segments, committed counts, audio -// cursors, stream_revision); the dispatcher owns lifecycle state, -// last_status, and the public committed/tentative text view. clear_result() -// wipes the snapshot but never touches stream_state. - -static transcribe_status transcribe_stream_begin_impl(struct transcribe_session * session, - const struct transcribe_run_params * run_params, - const struct transcribe_stream_params * stream_params) { - if (session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL run/stream params each mean "all defaults". - struct transcribe_run_params run_params_defaults; - transcribe_run_params_init(&run_params_defaults); - struct transcribe_stream_params stream_params_defaults; - transcribe_stream_params_init(&stream_params_defaults); - if (run_params == nullptr) { - run_params = &run_params_defaults; - } - if (stream_params == nullptr) { - stream_params = &stream_params_defaults; - } - if (const auto st = check_input_struct_size(run_params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - // Full-size view (see normalize_run_params); the family hook gets a - // further copy whose strings the library owns, built below. - struct transcribe_run_params run_params_view; - normalize_run_params(run_params, &run_params_view); - run_params = &run_params_view; - if (const auto st = check_input_struct_size(stream_params->struct_size, k_min_stream_params_size); - st != TRANSCRIBE_OK) { - return st; - } - // Read the caller's commit_policy as raw bytes and validate BEFORE the - // first load through the enum-typed lvalue: C callers can legally store - // any int there, and an out-of-range enum load is UB in C++. - int commit_policy_raw = TRANSCRIBE_STREAM_COMMIT_AUTO; - uint32_t stable_prefix_agreement_n = 0; - if (has_field(stream_params->struct_size, k_stream_params_commit_policy_size)) { - static_assert(sizeof(stream_params->commit_policy) == sizeof(int), - "commit_policy must be int-sized for the raw read"); - std::memcpy(&commit_policy_raw, &stream_params->commit_policy, sizeof(commit_policy_raw)); - } - if (!valid_stream_commit_policy(commit_policy_raw)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const transcribe_stream_commit_policy commit_policy = - static_cast(commit_policy_raw); - if (has_field(stream_params->struct_size, k_stream_params_agreement_n_size)) { - stable_prefix_agreement_n = stream_params->stable_prefix_agreement_n; - if (stable_prefix_agreement_n > 32) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - // Caller must finalize or reset before starting a new stream. - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->model == nullptr || session->model->arch == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - // Capability gate: the model must advertise streaming AND the - // family must wire the full required hook set. begin / feed / - // finalize come as a triple — a partially-wired family that - // accepts begin would otherwise let the caller enter ACTIVE and - // then get stuck on NOT_IMPLEMENTED at the first feed. stream_reset - // remains optional; the dispatcher's clear_result + state wipe is - // sufficient when a family does not need to release per-utterance - // buffers explicitly. - if (!session->model->caps.supports_streaming || session->model->arch->stream_begin == nullptr || - session->model->arch->stream_feed == nullptr || session->model->arch->stream_finalize == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - if (const transcribe_status st = validate_run_params_common(session, run_params); st != TRANSCRIBE_OK) { - return st; - } - - // v1 rejects TRANSLATE unconditionally for streaming. A future - // family that supports streaming translate would loosen this in - // its stream_begin hook, but the central dispatcher refuses - // upfront so partially-wired callers fail fast. - if (run_params->task == TRANSCRIBE_TASK_TRANSLATE || run_params->task == TRANSCRIBE_TASK_INSTRUCT) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - // A prefix is forced decoder text for one utterance's opening; a stream - // has no fixed opening to force it onto. - if (transcribe::prompting::has_text(run_params->prefix)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_stream_begin: a transcript prefix is not accepted " - "for streaming"); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - if (stream_params->family != nullptr) { - if (stream_params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_STREAM, - stream_params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - - // Advisory warn for pnc/itn requests against models that don't - // expose the corresponding runtime toggle. Emitted before - // clear_result so the pre-hook "snapshot preserved on rejection" - // contract is undisturbed. - warn_unsupported_advisory(session->model, run_params); - if (const transcribe_status st = prepare_prompting(session->model, &run_params_view); st != TRANSCRIBE_OK) { - return st; - } - - // Optional family preflight: validates extension field values - // (e.g. parakeet's (L, C, R) menu) without mutating state. On - // non-OK return the previous snapshot and lifecycle are preserved - // so a caller-side typo does not destroy the prior utterance. - if (session->model->arch->stream_validate != nullptr) { - if (const transcribe_status st = session->model->arch->stream_validate(session, run_params, stream_params); - st != TRANSCRIBE_OK) { - return st; - } - } - - // All checks pass — clear the previous snapshot and hand off. - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - session->stream_state = TRANSCRIBE_STREAM_ACTIVE; - session->stream_commit_policy = commit_policy; - session->stream_stable_prefix_agreement_n = stable_prefix_agreement_n; - - // Hand the family hook a params view whose pointers the LIBRARY owns. - // Families may capture `*run_params` for the stream's lifetime - // (parakeet re-reads .language on every feed), and the public contract - // lets the caller free every params pointer once begin returns — so the - // strings are copied into session storage here and the view repointed - // at it. `family` (the run-slot extension) is nulled in the view: no - // family reads it on the stream path today, and per the ext copy-out - // contract a retained pointer to it would dangle; a future family that - // wants a run-slot ext at stream begin must plumb it deliberately. - session->stream_language_owned = run_params->language != nullptr ? run_params->language : ""; - session->stream_target_language_owned = run_params->target_language != nullptr ? run_params->target_language : ""; - // Copied from the normalized view (never the caller's struct, whose - // allocation may end before the trailing fields); the view keeps the - // caller's struct_size, so has_field() gating still sees its layout. - struct transcribe_run_params run_params_owned = run_params_view; - run_params_owned.language = run_params->language != nullptr ? session->stream_language_owned.c_str() : nullptr; - run_params_owned.target_language = - run_params->target_language != nullptr ? session->stream_target_language_owned.c_str() : nullptr; - run_params_owned.family = nullptr; - // Generic prompting strings, same ownership rule. Only non-empty terms - // are kept, so the view's count matches the owned array. - session->stream_vocabulary_owned = transcribe::prompting::terms(run_params); - session->stream_vocabulary_ptrs.clear(); - for (const std::string & term : session->stream_vocabulary_owned) { - session->stream_vocabulary_ptrs.push_back(term.c_str()); - } - session->stream_prompt_owned = run_params->prompt != nullptr ? run_params->prompt : ""; - run_params_owned.vocabulary = - session->stream_vocabulary_ptrs.empty() ? nullptr : session->stream_vocabulary_ptrs.data(); - run_params_owned.n_vocabulary = static_cast(session->stream_vocabulary_ptrs.size()); - run_params_owned.prompt = run_params->prompt != nullptr ? session->stream_prompt_owned.c_str() : nullptr; - run_params_owned.prefix = nullptr; - - const transcribe_status st = session->model->arch->stream_begin(session, &run_params_owned, stream_params); - if (st != TRANSCRIBE_OK) { - // Family hook rejected the begin (config it does not understand, - // memory allocation failure, etc.). Roll lifecycle back to - // FAILED so the caller can read last_status; leave the cleared - // result snapshot in place. - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - } - return st; -} - -// Helper: validate the caller's stream_update struct_size and reset -// every field except struct_size to a clean state before handing the -// update buffer to the family hook. Returns OK when update is NULL -// (the update buffer is optional) or when the size passes validation. -static transcribe_status prepare_stream_update(struct transcribe_stream_update * update) { - if (update == nullptr) { - return TRANSCRIBE_OK; - } - if (const auto st = check_struct_size(update->struct_size, k_min_stream_update_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = update->struct_size; - transcribe_stream_update staged{}; - staged.struct_size = caller_size; - copy_out_prefix(update, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -static transcribe_status transcribe_stream_feed_impl(struct transcribe_session * session, - const float * pcm, - int n_samples, - struct transcribe_stream_update * update) { - if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { - return st; - } - if (session == nullptr || pcm == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // Zero-length feed is rejected: feed exists to consume audio, - // and callers that just want to inspect state use the stream - // accessors. n_samples < 0 is plain garbage. - if (n_samples <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // begin already confirmed model/arch/hook; we re-check defensively - // so a malformed context (never happens in practice from a real - // begin) does not deref a null function pointer. - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->stream_feed == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - const int32_t prev_revision = session->stream_revision; - const std::string prev_full_text = session->full_text; - const bool prev_has_result = session->has_result; - - const transcribe_status st = session->model->arch->stream_feed(session, pcm, n_samples, update); - if (st != TRANSCRIBE_OK) { - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - return st; - } - const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/false); - if (session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { - session->n_committed_tokens = 0; - session->n_committed_words = 0; - session->n_committed_segments = 0; - } - publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); - if (update != nullptr && session->stream_commit_policy == TRANSCRIBE_STREAM_COMMIT_ON_FINALIZE) { - update->audio_committed_ms = 0; - } - return st; -} - -static transcribe_status transcribe_stream_finalize_impl(struct transcribe_session * session, - struct transcribe_stream_update * update) { - if (const auto st = prepare_stream_update(update); st != TRANSCRIBE_OK) { - return st; - } - if (session == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->stream_state != TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (session->model == nullptr || session->model->arch == nullptr || - session->model->arch->stream_finalize == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - const int32_t prev_revision = session->stream_revision; - const std::string prev_full_text = session->full_text; - const bool prev_has_result = session->has_result; - - const transcribe_status st = session->model->arch->stream_finalize(session, update); - if (update != nullptr) { - // Force is_final true regardless of family hook return; the - // marker describes the call site, not the result. Set after - // the hook so the family cannot accidentally clear it. - update->is_final = true; - } - if (st != TRANSCRIBE_OK) { - session->stream_state = TRANSCRIBE_STREAM_FAILED; - session->stream_last_status = st; - return st; - } - const StreamTextDelta text_delta = apply_stream_text_policy(session, /*is_finalize=*/true); - publish_observable_delta(session, update, prev_revision, prev_full_text, prev_has_result, text_delta); - session->stream_state = TRANSCRIBE_STREAM_FINISHED; - return TRANSCRIBE_OK; -} - -static void transcribe_stream_reset_impl(struct transcribe_session * session) { - if (session == nullptr) { - return; - } - // Family hook releases per-utterance state and clears any buffered - // audio contents while keeping the allocations. A family without - // streaming just has no hook installed; reset becomes a pure - // dispatcher state wipe. - if (session->model != nullptr && session->model->arch != nullptr && session->model->arch->stream_reset != nullptr) { - session->model->arch->stream_reset(session); - } - session->clear_result(); - session->stream_state = TRANSCRIBE_STREAM_IDLE; - // was_aborted is per-stream; reset re-arms it the same way begin - // does so a caller that resets after an abort starts clean. - session->was_aborted = false; -} - -extern "C" enum transcribe_stream_state transcribe_stream_get_state(const struct transcribe_session * session) { - if (session == nullptr) { - return TRANSCRIBE_STREAM_IDLE; - } - return session->stream_state; -} - -extern "C" int transcribe_stream_revision(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->stream_revision; -} - -extern "C" int transcribe_stream_n_committed_segments(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_segments; -} - -extern "C" int transcribe_stream_n_committed_words(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_words; -} - -extern "C" int transcribe_stream_n_committed_tokens(const struct transcribe_session * session) { - if (session == nullptr) { - return 0; - } - return session->n_committed_tokens; -} - -extern "C" transcribe_status transcribe_stream_last_status(const struct transcribe_session * session) { - if (session == nullptr) { - return TRANSCRIBE_OK; - } - return session->stream_last_status; -} - -extern "C" transcribe_status transcribe_stream_get_text(const struct transcribe_session * session, - struct transcribe_stream_text * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_stream_text_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_stream_text staged{}; - staged.struct_size = caller_size; - if (session != nullptr && session->has_result) { - staged.full_text = session->full_text.c_str(); - staged.full_text_bytes = static_cast(session->full_text.size()); - staged.committed_text = session->stream_committed_text.c_str(); - staged.committed_text_bytes = static_cast(session->stream_committed_text.size()); - staged.tentative_text = session->stream_tentative_text.c_str(); - staged.tentative_text_bytes = static_cast(session->stream_tentative_text.size()); - staged.raw_tentative_start_bytes = - std::min(session->stream_raw_tentative_start_bytes, staged.full_text_bytes); - } else { - staged.full_text = ""; - staged.committed_text = ""; - staged.tentative_text = ""; - } - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -// Scope guard that calls transcribe_session::release_scratch on exit once -// armed. Both offline entry points use it so release also happens when a -// family hook throws and the api_guard unwinds the stack, including the path -// where memory pressure matters most. release_scratch is noexcept, so running -// it during unwinding is safe. -namespace { - -struct scratch_release_guard { - transcribe_session * session = nullptr; - bool armed = false; - - ~scratch_release_guard() { - if (armed && session != nullptr) { - session->release_scratch(); - } - } -}; - -} // namespace - -// Shared one-utterance run body. Does NOT touch session->batch_results, so -// the batch dispatcher can call it once per utterance inside a loop without -// erasing already-accumulated entries; the public transcribe_run wrapper -// below clears batch_results once before delegating here. Every early -// return preserves the previous result snapshot exactly as the original -// transcribe_run contract documented (see the inline comments). -// `committed` (optional) is set true once the call passes the pre-clear -// gates and commits to replacing the result. The caller uses it to decide -// whether compute scratch needs releasing. -static transcribe_status run_one_inner(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params, - bool * committed = nullptr) { - // Parameter-shape validation runs first and does not touch session - // state. A caller that passes NULL pointers or a non-positive sample - // count gets ERR_INVALID_ARG back without any visible side effect - // — including the session's result fields, which are preserved - // across a malformed call. This is the narrower half of the - // "transcribe_run replaces the previous result" contract: the - // replacement happens when the call is well-formed enough that - // the dispatcher is willing to dereference session. A call that - // isn't well-formed isn't considered to have "run". - if (session == nullptr || pcm == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // n_samples must be strictly positive, matching transcribe_stream_feed. - // There is no meaningful "transcribe zero samples" operation; a - // zero-length batch is treated as a caller error, not an empty run. - if (n_samples <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // NULL params means "all defaults" (transcribe vs translate, no - // timestamps, etc.). A well-formed default run is not a malformed - // call, so it proceeds and replaces the previous result. - struct transcribe_run_params params_defaults; - transcribe_run_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - // Everything downstream reads this full-size view, never the caller's - // struct: an older caller's allocation may end before the prompting - // fields. Strings stay caller-owned; the call is synchronous. - struct transcribe_run_params params_view; - normalize_run_params(params, ¶ms_view); - params = ¶ms_view; - // A run cannot replace an active stream's results — that would - // strand the in-flight stream's per-family state. Caller must - // finalize or reset first. FINISHED and FAILED both fall through; - // the run path below resets stream_state to IDLE. - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Pre-clear family-extension SHAPE validation: a run ext with a bad - // transcribe_ext header size or a kind the model does not accept must - // NOT wipe the previous result snapshot (mirrors the - // transcribe_stream_begin contract). When model is null we skip the - // pre-clear check and let the post-clear NOT_IMPLEMENTED path handle it. - // - // This guarantee covers ext shape (size/kind) and the run-param checks - // below. A family that defers deeper value validation to run() (whisper - // does this for prompt semantics) can still reject a correctly-shaped - // ext after the snapshot is cleared; full pre-clear safety requires - // validating values in run_validate. - if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { - if (params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - - // Validate before advisory warning so typed enum reads are proven safe. - // A rejection preserves the previous snapshot. Skipped when model is - // null — the post-clear NOT_IMPLEMENTED path signals that directly. - if (session->model != nullptr) { - // Pre-clear run-param validation. A caller-side param bug - // (out-of-range enum, timestamp granularity finer than the - // model's max, unsupported language, or TRANSLATE against a - // model that doesn't support it) must NOT wipe the previous - // result snapshot — the same contract transcribe_stream_begin - // honors for its run params. These checks read model->caps and - // are therefore guarded by model != null; the degenerate - // model-null case falls through to the post-clear - // NOT_IMPLEMENTED path, which keeps its existing snapshot-wipe - // behavior. - if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { - return st; - } - warn_unsupported_advisory(session->model, params); - if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { - return st; - } - - // Family run-ext validation (the _RUN analogue of stream_validate), - // the final pre-clear gate. Runs AFTER the run-param checks above, - // mirroring transcribe_stream_begin's ordering (dispatcher param - // checks, then the family validate hook, then clear). The family - // alone knows its per-kind minimum struct size; whisper enforces - // that here. A non-OK return preserves the previous snapshot. A - // family with no _RUN ext leaves this hook NULL. Called regardless - // of whether params->family is set (whisper's hook treats a NULL - // family as "defaults", returning OK). - if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { - if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { - return st; - } - } - } - - // Result-replacement contract: everything past this point either - // succeeds and writes a fresh result, or fails and leaves the - // context in the documented "no result" sentinel state. Clear - // eagerly so the one remaining downstream rejection path — - // NOT_IMPLEMENTED on an incomplete arch — inherits the sentinel - // without having to remember to call clear_result() itself. - // Caller-param rejections (enum range, timestamp ceiling, - // language, TRANSLATE support) are validated ABOVE, before this - // clear, so a malformed call preserves the previous snapshot - // exactly as transcribe_stream_begin does. - // - // Per-family run() handlers call clear_result() again after - // their own front-matter checks succeed; that call is now - // redundant but idempotent, and removing it is a refactor - // deferred to a later pass. - if (committed != nullptr) { - *committed = true; - } - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - // Force stream_state to IDLE: clear_result deliberately preserves - // lifecycle state, but a well-formed transcribe_run subsumes any - // prior FINISHED/FAILED stream — after a one-shot run the context - // is no longer meaningfully in a streaming lifecycle. - session->stream_state = TRANSCRIBE_STREAM_IDLE; - - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - return session->model->arch->run(session, pcm, n_samples, params); -} - -static transcribe_status transcribe_run_impl(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params) { - // A well-formed single run has the same result view as a one-item batch: - // reset batch_results so the transcribe_batch_* accessors fall back to - // reading the scratch slot as utterance 0. - // - // Guard the deref with the SAME pre-deref conditions run_one_inner uses - // (non-null session + non-null pcm + positive n_samples). A malformed - // call must not touch the session at all — the API smoke test probes - // this with a fake (session *)0x1 and a NULL pcm / non-positive - // n_samples, expecting INVALID_ARG with no dereference. - if (session != nullptr && pcm != nullptr && n_samples > 0) { - session->batch_results.clear(); - } - // run_one_inner arms the guard at its commit point, so a pre-clear - // rejection never releases and everything after (family error, abort, - // throw) always does. - scratch_release_guard scratch_release; - scratch_release.session = session; - return run_one_inner(session, pcm, n_samples, params, &scratch_release.armed); -} - -// Batch run (offline) -// -// transcribe_run_batch validates the shared run_params ONCE, then either -// delegates to the family's batched run_batch() fast path or falls back to -// run_one_inner() per utterance. Either way the result is N entries in -// session->batch_results read back via the transcribe_batch_* accessors. - -namespace { - -// Pad batch_results out to `n` entries with explicit aborted failures. -// Called only on an aborted batch: synthesized slots carry -// TRANSCRIBE_ERR_ABORTED ("did not complete because the batch was aborted"), -// so the result-set view has one slot per input utterance -// (transcribe_batch_n_results() == n) regardless of family or path. -void pad_batch_results_aborted(transcribe_session * s, int n) { - while (s->batch_results.size() < static_cast(n)) { - transcribe_session::ResultSet rs; - rs.status = TRANSCRIBE_ERR_ABORTED; - s->batch_results.push_back(std::move(rs)); - } -} - -// Restore the scratch slot from a ResultSet so the single-result accessors -// stay coherent after a batch run (they reflect utterance 0). This is the -// inverse of transcribe_session::capture_result minus the per-run timing -// accumulators — keep the result fields in sync with that method when adding -// one (raw_text was once missed here). -void restore_scratch_from_result(transcribe_session * s, const transcribe_session::ResultSet & rs) { - s->tokens = rs.tokens; - s->words = rs.words; - s->segments = rs.segments; - s->speaker_segments = rs.speaker_segments; - s->full_text = rs.full_text; - s->raw_text = rs.raw_text; - s->detected_language = rs.detected_language; - s->result_kind = rs.result_kind; - s->has_result = rs.has_result; -} - -} // namespace - -static transcribe_status transcribe_run_batch_impl(struct transcribe_session * session, - const float * const * pcm, - const int * n_samples, - int n, - const struct transcribe_run_params * params) { - // Top-level argument shape. The arrays themselves must be present and - // n strictly positive; an individual malformed utterance (pcm[i] NULL - // or n_samples[i] <= 0) is a per-utterance failure handled in the loop, - // not a whole-batch error. As with transcribe_run, these checks do not - // mutate the session, so a malformed call preserves the prior result. - if (session == nullptr || pcm == nullptr || n_samples == nullptr || n <= 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - struct transcribe_run_params params_defaults; - transcribe_run_params_init(¶ms_defaults); - if (params == nullptr) { - params = ¶ms_defaults; - } - if (const auto st = check_input_struct_size(params->struct_size, k_min_run_params_size); st != TRANSCRIBE_OK) { - return st; - } - struct transcribe_run_params params_view; - normalize_run_params(params, ¶ms_view); - params = ¶ms_view; - if (session->stream_state == TRANSCRIBE_STREAM_ACTIVE) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - // One shared params across different audio: a transcript prefix can - // only describe one of them. - if (transcribe::prompting::has_text(params->prefix)) { - transcribe::log_msg(TRANSCRIBE_LOG_LEVEL_ERROR, - "transcribe_run_batch: a transcript prefix is per-utterance " - "and is not accepted in a batch"); - return TRANSCRIBE_ERR_INVALID_ARG; - } - - // Shared-param validation, ONCE, mirroring transcribe_run's pre-clear - // gates (ext shape/kind, pnc/itn advisory, enum range, timestamp - // ceiling, language, TRANSLATE support, run_validate). A rejection here - // preserves the previous result snapshot — nothing has been cleared. - if (params->family != nullptr && session->model != nullptr && session->model->arch != nullptr) { - if (params->family->size < sizeof(struct transcribe_ext)) { - return TRANSCRIBE_ERR_BAD_STRUCT_SIZE; - } - if (!transcribe_model_accepts_ext_kind(session->model, TRANSCRIBE_EXT_SLOT_RUN, params->family->kind)) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - } - if (session->model != nullptr) { - if (const transcribe_status st = validate_run_params_common(session, params); st != TRANSCRIBE_OK) { - return st; - } - warn_unsupported_advisory(session->model, params); - if (params->task == TRANSCRIBE_TASK_TRANSLATE && !session->model->caps.supports_translate) { - return TRANSCRIBE_ERR_UNSUPPORTED_TASK; - } - if (const transcribe_status st = prepare_prompting(session->model, ¶ms_view); st != TRANSCRIBE_OK) { - return st; - } - if (session->model->arch != nullptr && session->model->arch->run_validate != nullptr) { - if (const transcribe_status st = session->model->arch->run_validate(session, params); st != TRANSCRIBE_OK) { - return st; - } - } - } - - if (session->model == nullptr || session->model->arch == nullptr || session->model->arch->run == nullptr) { - return TRANSCRIBE_ERR_NOT_IMPLEMENTED; - } - - // Past this point we commit to producing a fresh batch result. - session->clear_result(); - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; - session->was_aborted = false; - session->was_truncated = false; - session->stopped_on_repetition = false; - session->stream_state = TRANSCRIBE_STREAM_IDLE; - session->batch_results.clear(); - - // Release the compute scratch once the batch has run, whichever path it - // took (see transcribe_session::release_scratch). Once per call, not per - // utterance, so the serial fallback keeps its workspace across the loop. - scratch_release_guard scratch_release; - scratch_release.session = session; - scratch_release.armed = true; - - // Fast path: a family with a batched compute graph owns the whole loop. - if (session->model->arch->run_batch != nullptr) { - const transcribe_status st = session->model->arch->run_batch(session, pcm, n_samples, n, params); - // On abort the hook may retain only completed results; synthesize any - // missing slots so the result-set view always exposes n entries. - if (st == TRANSCRIBE_ERR_ABORTED) { - pad_batch_results_aborted(session, n); - } - // Keep the single accessors coherent with utterance 0. - if (!session->batch_results.empty()) { - restore_scratch_from_result(session, session->batch_results.front()); - } - return st; - } - - // Generic serial fallback: run each utterance in turn and snapshot it. - // Correct for every family; only the per-dispatch device throughput of - // a real run_batch() is forgone. run_one_inner re-validates the shared - // params (idempotent) before the family run(). - session->batch_results.reserve(static_cast(n)); - const transcribe_status batch_status = transcribe::run_batch_serial( - session, pcm, n_samples, n, [&](const float * p, int ns) { return run_one_inner(session, p, ns, params); }); - - // On abort the loop can break early, leaving fewer than n entries; - // synthesize any missing slots so the result-set view always exposes n - // entries (same invariant the fast path holds). - if (batch_status == TRANSCRIBE_ERR_ABORTED) { - pad_batch_results_aborted(session, n); - } - - // Restore the scratch slot to mirror utterance 0 for the single-result - // accessors (n >= 1 guarantees at least one entry, but be defensive). - if (!session->batch_results.empty()) { - restore_scratch_from_result(session, session->batch_results.front()); - } - return batch_status; +static void transcribe_model_free_impl(struct transcribe_model * model) { + // Polymorphic delete: the base has a virtual destructor (anchored + // in transcribe-model.cpp), so this dispatches to the per-family + // subclass destructor and that destructor frees gguf_context, + // weights, etc. Passing NULL is a no-op per the public contract. + delete model; } // Model introspection // // These accessors read directly from the base struct; per-family load() -// fills caps / variant / backend before returning success. - -extern "C" transcribe_status transcribe_model_get_capabilities(const struct transcribe_model * model, - struct transcribe_capabilities * out_caps) { - if (model == nullptr || out_caps == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out_caps->struct_size, k_min_capabilities_size); st != TRANSCRIBE_OK) { - return st; - } - // Preserve the caller-declared size, then write only the prefix - // that fits in both the caller's buffer and this library's view; - // any tail bytes stay as the caller initialized them (zero, by - // the init function's zero-fill contract). - // - // Pointer fields written into the caller buffer (e.g. languages) - // remain model-owned and valid until transcribe_model_free(). - const uint64_t caller_size = out_caps->struct_size; - transcribe_capabilities staged = model->caps; - staged.struct_size = caller_size; - copy_out_prefix(out_caps, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_session_get_limits(const struct transcribe_session * session, - struct transcribe_session_limits * out) { - if (session == nullptr || out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_session_limits_size); st != TRANSCRIBE_OK) { - return st; - } - - transcribe_session_limits staged; - transcribe_session_limits_init(&staged); - - const transcribe_model * model = session->model; - const auto & lb = model->limits; - - if (lb.has_context_cap && lb.model_max_ctx > 0) { - // effective_n_ctx = model max, lowered (never raised) by the session - // n_ctx cap. This mirrors the per-family context_ceiling helpers. - int32_t eff = lb.model_max_ctx; - if (session->n_ctx > 0 && session->n_ctx < eff) { - eff = session->n_ctx; - } - staged.effective_n_ctx = eff; - - // effective_max_audio_ms: for families whose audio tokens consume the - // decoder context, invert the input gate at the effective ceiling so - // the audio bound tracks n_ctx. For families whose audio bound is the - // encoder (audio_from_caps), the audio limit is independent of the - // decoder context, so report caps.max_audio_ms unchanged. Either way - // it is advisory (representative prompt), not an exact per-call bound. - if (lb.audio_from_caps) { - staged.effective_max_audio_ms = model->caps.max_audio_ms; - } else { - const int64_t audio_tokens = (int64_t) eff - lb.prompt_overhead - lb.gen_reserve; - staged.effective_max_audio_ms = (audio_tokens > 0 && lb.ms_per_audio_token > 0.0) ? - (int64_t) ((double) audio_tokens * lb.ms_per_audio_token) : - 0; - } - - // max_kv_bytes: worst-case single-utterance KV allocation at the - // effective ceiling, exact for the session's kv_type. The families - // resolve AUTO (and F16) to f16 for the KV cache and use f32 only for - // an explicit F32 request, so the byte size is 4/elem for F32 and - // 2/elem otherwise. This is the ceiling for one utterance, not the - // per-run allocation (the cache grows to fit input); transcribe_run_batch - // allocates roughly batch_size x this. - const int64_t kv_bytes_per_elem = (session->kv_type == TRANSCRIBE_KV_TYPE_F32) ? 4 : 2; - staged.max_kv_bytes = lb.kv_elems_per_ctx_token * (int64_t) eff * kv_bytes_per_elem; - } else { - // Unbounded / soft-window family: no decoder context cap. Report the - // soft window (if any) from caps.max_audio_ms, independent of n_ctx. - staged.effective_n_ctx = 0; - staged.effective_max_audio_ms = model->caps.max_audio_ms; - staged.max_kv_bytes = 0; - } - - const uint64_t caller_size = out->struct_size; - staged.struct_size = caller_size; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} +// fills variant / backend / meta before returning success. extern "C" const char * transcribe_model_arch_string(const struct transcribe_model * model) { // The dispatch token's name field has static lifetime and is the @@ -2905,44 +1115,8 @@ extern "C" const char * transcribe_model_meta_val_str(const struct transcribe_mo return it != model->meta.end() ? it->second.c_str() : ""; } -static int transcribe_tokenize_impl(const struct transcribe_model * model, - const char * text, - int32_t * tokens, - size_t n_max) { - if (model == nullptr || text == nullptr) { - return INT_MIN; - } - const transcribe::Tokenizer * tok = model->tokenizer(); - if (tok == nullptr) { - return INT_MIN; - } - - std::vector ids; - if (tok->encode(std::string(text), ids) != TRANSCRIBE_OK) { - return INT_MIN; - } - - const size_t n = ids.size(); - if (n > static_cast(INT_MAX)) { - return INT_MIN; - } - if (n > n_max) { - // Negative-of-N retry contract (mirrors whisper.cpp). Guard - // against -n overflowing int range for pathologically large - // vocabularies; fall back to INT_MIN + 1 so the caller still - // gets a hard-failure code rather than a stale pointer. - if (n > static_cast(INT_MAX)) { - return INT_MIN; - } - return -static_cast(n); - } - if (tokens == nullptr && n > 0) { - return INT_MIN; - } - for (size_t i = 0; i < n; ++i) { - tokens[i] = ids[i]; - } - return static_cast(n); +extern "C" uint32_t transcribe_model_roles(const struct transcribe_model * model) { + return model != nullptr ? model->roles : 0; } extern "C" const char * transcribe_model_backend(const struct transcribe_model * model) { @@ -2965,554 +1139,6 @@ static transcribe_device_t transcribe_model_device_impl(const struct transcribe_ return reinterpret_cast(dev); } -// Timings -// -// Accumulators live on the base classes (transcribe_model::t_load_us and -// transcribe_session::t_{mel,encode,decode}_us), populated by per-family -// load() / run(). These accessors convert microseconds to milliseconds. - -extern "C" transcribe_status transcribe_get_timings(const struct transcribe_session * session, - struct transcribe_timings * out_timings) { - if (session == nullptr || out_timings == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out_timings->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out_timings->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(session->t_mel_us) / 1000.0f; - staged.encode_ms = static_cast(session->t_encode_us) / 1000.0f; - staged.decode_ms = static_cast(session->t_decode_us) / 1000.0f; - copy_out_prefix(out_timings, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" void transcribe_print_timings(const struct transcribe_session * session) { - if (session == nullptr) { - return; - } - struct transcribe_timings t; - transcribe_timings_init(&t); - (void) transcribe_get_timings(session, &t); - char buf[256]; - - std::snprintf(buf, sizeof(buf), - "timings: load=%.2f ms mel=%.2f ms " - "encode=%.2f ms decode=%.2f ms", - t.load_ms, t.mel_ms, t.encode_ms, t.decode_ms); - // Surfaced even in the common dev case (transcribe-cli without an - // installed sink) via the stderr fallback; an explicit - // transcribe_log_set(NULL, ...) silences it like everything else. - transcribe_log_emit_or_stderr(TRANSCRIBE_LOG_LEVEL_INFO, buf); -} - -extern "C" void transcribe_reset_timings(struct transcribe_session * session) { - if (session == nullptr) { - return; - } - // load_us is intentionally left alone — it's a model-scoped fact, - // not a per-call accumulator. - session->t_mel_us = 0; - session->t_encode_us = 0; - session->t_decode_us = 0; -} - -// Result accessors -// -// These read from the base context's result storage (tokens / words / -// segments / full_text / result_kind / has_result), populated by the -// per-family run() driver. Out-of-range indices and pre-run access return -// safe sentinels per the public contract. - -extern "C" const char * transcribe_full_text(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->full_text.c_str(); -} - -extern "C" const char * transcribe_raw_text(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->raw_text.c_str(); -} - -extern "C" const char * transcribe_detected_language(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return ""; - } - return session->detected_language.c_str(); -} - -extern "C" transcribe_timestamp_kind transcribe_returned_timestamp_kind(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return TRANSCRIBE_TIMESTAMPS_NONE; - } - return session->result_kind; -} - -extern "C" int transcribe_n_segments(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->segments.size()); -} - -extern "C" int transcribe_n_words(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->words.size()); -} - -extern "C" int transcribe_n_tokens(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->tokens.size()); -} - -extern "C" int transcribe_n_speaker_segments(const struct transcribe_session * session) { - if (session == nullptr || !session->has_result) { - return 0; - } - return static_cast(session->speaker_segments.size()); -} - -// Result accessors - per-item rows -// -// Copy-out accessors backed by the context's segments / words / tokens -// vectors, writing only the prefix that fits the caller's struct_size. -// Out-of-range index or pre-run access leaves the caller's struct zero-init -// and still returns OK, so callers can use text!=NULL as the "row present" -// signal without branching on status. -// -// `text` pointers in the staged structs alias the context's std::string -// storage; the library treats that as session-owned, valid until the next -// transcribe_run / transcribe_stream_begin / transcribe_stream_reset / -// transcribe_session_free on the same context (see the public header). - -extern "C" transcribe_status transcribe_get_segment(const struct transcribe_session * session, - int i, - struct transcribe_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->segments.size()) { - return TRANSCRIBE_OK; - } - const auto & s = session->segments[static_cast(i)]; - transcribe_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.first_word = s.first_word; - staged.n_words = s.n_words; - staged.first_token = s.first_token; - staged.n_tokens = s.n_tokens; - staged.text = s.text.c_str(); - staged.speaker_id = s.speaker_id; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_word(const struct transcribe_session * session, - int i, - struct transcribe_word * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_word zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->words.size()) { - return TRANSCRIBE_OK; - } - const auto & w = session->words[static_cast(i)]; - transcribe_word staged{}; - staged.struct_size = caller_size; - staged.t0_ms = w.t0_ms; - staged.t1_ms = w.t1_ms; - staged.seg_index = w.seg_index; - staged.first_token = w.first_token; - staged.n_tokens = w.n_tokens; - staged.text = w.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_token(const struct transcribe_session * session, - int i, - struct transcribe_token * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_token zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || static_cast(i) >= session->tokens.size()) { - return TRANSCRIBE_OK; - } - const auto & t = session->tokens[static_cast(i)]; - transcribe_token staged{}; - staged.struct_size = caller_size; - staged.id = t.id; - staged.p = t.p; - staged.t0_ms = t.t0_ms; - staged.t1_ms = t.t1_ms; - staged.seg_index = t.seg_index; - staged.word_index = t.word_index; - staged.text = t.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_get_speaker_segment(const struct transcribe_session * session, - int i, - struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - if (session == nullptr || !session->has_result || i < 0 || - static_cast(i) >= session->speaker_segments.size()) { - return TRANSCRIBE_OK; - } - const auto & s = session->speaker_segments[static_cast(i)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -// Batch result accessors -// -// These index session->batch_results when a batch run populated it, and -// otherwise synthesize utterance 0 from the scratch slot after -// transcribe_run. They share the staging shape of the single-result -// accessors above; only the source vector differs. - -namespace { - -// Non-owning view of one utterance's result. valid == false means "no such -// utterance / empty result" and the accessors return their safe sentinels. -struct BatchResultView { - const std::vector * tokens = nullptr; - const std::vector * words = nullptr; - const std::vector * segments = nullptr; - const std::vector * speaker_segments = nullptr; - const std::string * full_text = nullptr; - const std::string * raw_text = nullptr; - const std::string * detected_language = nullptr; - transcribe_timestamp_kind result_kind = TRANSCRIBE_TIMESTAMPS_NONE; - bool valid = false; -}; - -int batch_result_count(const transcribe_session * s) { - if (s == nullptr) { - return 0; - } - if (!s->batch_results.empty()) { - return static_cast(s->batch_results.size()); - } - return s->has_result ? 1 : 0; -} - -BatchResultView batch_result_view(const transcribe_session * s, int i) { - BatchResultView v; - if (s == nullptr || i < 0) { - return v; - } - if (!s->batch_results.empty()) { - if (static_cast(i) >= s->batch_results.size()) { - return v; - } - const auto & rs = s->batch_results[static_cast(i)]; - if (!rs.has_result) { - return v; // individually-failed / empty utterance - } - v.tokens = &rs.tokens; - v.words = &rs.words; - v.segments = &rs.segments; - v.speaker_segments = &rs.speaker_segments; - v.full_text = &rs.full_text; - v.raw_text = &rs.raw_text; - v.detected_language = &rs.detected_language; - v.result_kind = rs.result_kind; - v.valid = true; - return v; - } - if (i == 0 && s->has_result) { - v.tokens = &s->tokens; - v.words = &s->words; - v.segments = &s->segments; - v.speaker_segments = &s->speaker_segments; - v.full_text = &s->full_text; - v.raw_text = &s->raw_text; - v.detected_language = &s->detected_language; - v.result_kind = s->result_kind; - v.valid = true; - } - return v; -} - -} // namespace - -extern "C" int transcribe_batch_n_results(const struct transcribe_session * session) { - return batch_result_count(session); -} - -extern "C" transcribe_status transcribe_batch_status(const struct transcribe_session * session, int i) { - if (session == nullptr || i < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (!session->batch_results.empty()) { - if (static_cast(i) >= session->batch_results.size()) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - return session->batch_results[static_cast(i)].status; - } - if (i == 0 && session->has_result) { - return TRANSCRIBE_OK; - } - return TRANSCRIBE_ERR_INVALID_ARG; -} - -extern "C" const char * transcribe_batch_full_text(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.full_text->c_str() : ""; -} - -extern "C" const char * transcribe_batch_raw_text(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.raw_text->c_str() : ""; -} - -extern "C" const char * transcribe_batch_detected_language(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.detected_language->c_str() : ""; -} - -extern "C" transcribe_timestamp_kind transcribe_batch_returned_timestamp_kind(const struct transcribe_session * session, - int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? v.result_kind : TRANSCRIBE_TIMESTAMPS_NONE; -} - -extern "C" int transcribe_batch_n_segments(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.segments->size()) : 0; -} - -extern "C" int transcribe_batch_n_words(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.words->size()) : 0; -} - -extern "C" int transcribe_batch_n_tokens(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.tokens->size()) : 0; -} - -extern "C" transcribe_status transcribe_batch_get_segment(const struct transcribe_session * session, - int i, - int j, - struct transcribe_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.segments->size()) { - return TRANSCRIBE_OK; - } - const auto & s = (*v.segments)[static_cast(j)]; - transcribe_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.first_word = s.first_word; - staged.n_words = s.n_words; - staged.first_token = s.first_token; - staged.n_tokens = s.n_tokens; - staged.text = s.text.c_str(); - staged.speaker_id = s.speaker_id; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_word(const struct transcribe_session * session, - int i, - int j, - struct transcribe_word * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_word_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_word zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.words->size()) { - return TRANSCRIBE_OK; - } - const auto & w = (*v.words)[static_cast(j)]; - transcribe_word staged{}; - staged.struct_size = caller_size; - staged.t0_ms = w.t0_ms; - staged.t1_ms = w.t1_ms; - staged.seg_index = w.seg_index; - staged.first_token = w.first_token; - staged.n_tokens = w.n_tokens; - staged.text = w.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_token(const struct transcribe_session * session, - int i, - int j, - struct transcribe_token * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_token_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_token zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.tokens->size()) { - return TRANSCRIBE_OK; - } - const auto & t = (*v.tokens)[static_cast(j)]; - transcribe_token staged{}; - staged.struct_size = caller_size; - staged.id = t.id; - staged.p = t.p; - staged.t0_ms = t.t0_ms; - staged.t1_ms = t.t1_ms; - staged.seg_index = t.seg_index; - staged.word_index = t.word_index; - staged.text = t.text.c_str(); - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" int transcribe_batch_n_speaker_segments(const struct transcribe_session * session, int i) { - const BatchResultView v = batch_result_view(session, i); - return v.valid ? static_cast(v.speaker_segments->size()) : 0; -} - -extern "C" transcribe_status transcribe_batch_get_speaker_segment(const struct transcribe_session * session, - int i, - int j, - struct transcribe_speaker_segment * out) { - if (out == nullptr) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_speaker_segment_size); st != TRANSCRIBE_OK) { - return st; - } - const uint64_t caller_size = out->struct_size; - transcribe_speaker_segment zero{}; - zero.struct_size = caller_size; - copy_out_prefix(out, &zero, caller_size, sizeof(zero)); - const BatchResultView v = batch_result_view(session, i); - if (!v.valid || j < 0 || static_cast(j) >= v.speaker_segments->size()) { - return TRANSCRIBE_OK; - } - const auto & s = (*v.speaker_segments)[static_cast(j)]; - transcribe_speaker_segment staged{}; - staged.struct_size = caller_size; - staged.t0_ms = s.t0_ms; - staged.t1_ms = s.t1_ms; - staged.speaker_id = s.speaker_id; - staged.p = s.p; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - -extern "C" transcribe_status transcribe_batch_get_timings(const struct transcribe_session * session, - int i, - struct transcribe_timings * out) { - if (session == nullptr || out == nullptr || i < 0) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - if (const auto st = check_struct_size(out->struct_size, k_min_timings_size); st != TRANSCRIBE_OK) { - return st; - } - int64_t mel_us = 0, enc_us = 0, dec_us = 0; - if (!session->batch_results.empty()) { - if (static_cast(i) >= session->batch_results.size()) { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const auto & rs = session->batch_results[static_cast(i)]; - mel_us = rs.t_mel_us; - enc_us = rs.t_encode_us; - dec_us = rs.t_decode_us; - } else if (i == 0) { - mel_us = session->t_mel_us; - enc_us = session->t_encode_us; - dec_us = session->t_decode_us; - } else { - return TRANSCRIBE_ERR_INVALID_ARG; - } - const uint64_t caller_size = out->struct_size; - transcribe_timings staged{}; - staged.struct_size = caller_size; - if (session->model != nullptr) { - staged.load_ms = static_cast(session->model->t_load_us) / 1000.0f; - } - staged.mel_ms = static_cast(mel_us) / 1000.0f; - staged.encode_ms = static_cast(enc_us) / 1000.0f; - staged.decode_ms = static_cast(dec_us) / 1000.0f; - copy_out_prefix(out, &staged, caller_size, sizeof(staged)); - return TRANSCRIBE_OK; -} - // C ABI forwarders. Entry points that allocate, reach ggml/driver state, // or transfer ownership route through api_guard_* here. Other public symbols // above are kept nothrow by construction: pure POD/c_str reads, prefix @@ -3561,8 +1187,10 @@ extern "C" bool transcribe_backend_available(transcribe_backend_request kind) { extern "C" transcribe_status transcribe_model_load_file(const char * path, const struct transcribe_model_load_params * params, struct transcribe_model ** out_model) { - const transcribe_status st = api_guard_status( - "transcribe_model_load_file", [&] { return transcribe_model_load_file_impl(path, params, out_model); }); + const transcribe_status st = api_guard_status("transcribe_model_load_file", [&] { + const transcribe_status lst = transcribe_model_load_file_impl(path, params, out_model); + return lst == TRANSCRIBE_OK && *out_model != nullptr ? transcribe::resolve_roles(*out_model) : lst; + }); // Boundary-owned postcondition: failure => *out_model == NULL. if (st != TRANSCRIBE_OK && out_model != nullptr && *out_model != nullptr) { transcribe_model_free(*out_model); @@ -3575,87 +1203,7 @@ extern "C" void transcribe_model_free(struct transcribe_model * model) { api_guard_void("transcribe_model_free", [&] { transcribe_model_free_impl(model); }); } -extern "C" transcribe_status transcribe_session_init(struct transcribe_model * model, - const struct transcribe_session_params * params, - struct transcribe_session ** out_session) { - const transcribe_status st = api_guard_status( - "transcribe_session_init", [&] { return transcribe_session_init_impl(model, params, out_session); }); - // Boundary-owned postcondition: failure => *out_session == NULL. - if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { - transcribe_session_free(*out_session); - *out_session = nullptr; - } - return st; -} - -extern "C" void transcribe_session_free(struct transcribe_session * session) { - api_guard_void("transcribe_session_free", [&] { transcribe_session_free_impl(session); }); -} - -extern "C" transcribe_status transcribe_open(const char * path, - const struct transcribe_model_load_params * load_params, - const struct transcribe_session_params * session_params, - struct transcribe_session ** out_session) { - const transcribe_status st = api_guard_status( - "transcribe_open", [&] { return transcribe_open_impl(path, load_params, session_params, out_session); }); - // Boundary-owned postcondition: failure => *out_session == NULL. - if (st != TRANSCRIBE_OK && out_session != nullptr && *out_session != nullptr) { - transcribe_session_free(*out_session); - *out_session = nullptr; - } - return st; -} - -extern "C" transcribe_status transcribe_run(struct transcribe_session * session, - const float * pcm, - int n_samples, - const struct transcribe_run_params * params) { - return api_guard_status("transcribe_run", [&] { return transcribe_run_impl(session, pcm, n_samples, params); }); -} - -extern "C" transcribe_status transcribe_run_batch(struct transcribe_session * session, - const float * const * pcm, - const int * n_samples, - int n, - const struct transcribe_run_params * params) { - return api_guard_status("transcribe_run_batch", - [&] { return transcribe_run_batch_impl(session, pcm, n_samples, n, params); }); -} - -extern "C" transcribe_status transcribe_stream_begin(struct transcribe_session * session, - const struct transcribe_run_params * run_params, - const struct transcribe_stream_params * stream_params) { - return api_guard_status("transcribe_stream_begin", - [&] { return transcribe_stream_begin_impl(session, run_params, stream_params); }); -} - -extern "C" transcribe_status transcribe_stream_feed(struct transcribe_session * session, - const float * pcm, - int n_samples, - struct transcribe_stream_update * update) { - return api_guard_status("transcribe_stream_feed", - [&] { return transcribe_stream_feed_impl(session, pcm, n_samples, update); }); -} - -extern "C" transcribe_status transcribe_stream_finalize(struct transcribe_session * session, - struct transcribe_stream_update * update) { - return api_guard_status("transcribe_stream_finalize", - [&] { return transcribe_stream_finalize_impl(session, update); }); -} - -extern "C" void transcribe_stream_reset(struct transcribe_session * session) { - api_guard_void("transcribe_stream_reset", [&] { transcribe_stream_reset_impl(session); }); -} - extern "C" transcribe_device_t transcribe_model_device(const struct transcribe_model * model) { return api_guard_value("transcribe_model_device", static_cast(nullptr), [&] { return transcribe_model_device_impl(model); }); } - -extern "C" int transcribe_tokenize(const struct transcribe_model * model, - const char * text, - int32_t * tokens, - size_t n_max) { - return api_guard_value("transcribe_tokenize", INT_MIN, - [&] { return transcribe_tokenize_impl(model, text, tokens, n_max); }); -} diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3ec7871f9..2b60c01d0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -341,6 +341,19 @@ transcribe_apply_warnings(transcribe_run_dispatch_unit) add_test(NAME transcribe_run_dispatch_unit COMMAND transcribe_run_dispatch_unit) +# ----------------------------------------------------------------------------- +# Dispatcher units against fake arches: roles, host thread pool, non-finite +# input, throwing stream hooks, DIARIZE role +# ----------------------------------------------------------------------------- + +foreach(_t role_resolve run_on_threads nonfinite_input stream_hook_throw diarize_dispatch) + add_executable(transcribe_${_t}_unit ${_t}_unit.cpp) + target_link_libraries(transcribe_${_t}_unit PRIVATE transcribe) + target_include_directories(transcribe_${_t}_unit PRIVATE ${CMAKE_SOURCE_DIR}/src) + transcribe_apply_warnings(transcribe_${_t}_unit) + add_test(NAME transcribe_${_t}_unit COMMAND transcribe_${_t}_unit) +endforeach() + # ----------------------------------------------------------------------------- # Dispatcher-level streaming entry-point behavior # ----------------------------------------------------------------------------- @@ -1333,24 +1346,19 @@ if(TRANSCRIBE_BUILD_REAL_MODEL_TESTS) set_tests_properties(transcribe_qwen3_asr_e2e_smoke PROPERTIES SKIP_RETURN_CODE 77) - # Sortformer streaming operating-point run ext (SFST, RUN slot). - # Kind/slot probe, init stamping, pre-clear rejection with snapshot - # preservation, and ext-vs-env preset parity on the committed 2-spk - # oracle mix. Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). - add_executable(transcribe_sortformer_stream_ext_unit - sortformer_stream_ext_unit.cpp) - - target_link_libraries(transcribe_sortformer_stream_ext_unit + # Sortformer on the DIARIZE role: golden segments per preset and pre-clear + # rejection. + # Gated by TRANSCRIBE_SORTFORMER_GGUF (RC 77 skip). + add_executable(transcribe_sortformer_diarize_unit + sortformer_diarize_unit.cpp) + target_link_libraries(transcribe_sortformer_diarize_unit PRIVATE transcribe transcribe-common-example) - - target_compile_definitions(transcribe_sortformer_stream_ext_unit PRIVATE + target_compile_definitions(transcribe_sortformer_diarize_unit PRIVATE "TRANSCRIBE_TEST_SAMPLES_DIR=\"${CMAKE_SOURCE_DIR}/samples\"") - - transcribe_apply_warnings(transcribe_sortformer_stream_ext_unit) - - add_test(NAME transcribe_sortformer_stream_ext_unit - COMMAND transcribe_sortformer_stream_ext_unit) - set_tests_properties(transcribe_sortformer_stream_ext_unit PROPERTIES + transcribe_apply_warnings(transcribe_sortformer_diarize_unit) + add_test(NAME transcribe_sortformer_diarize_unit + COMMAND transcribe_sortformer_diarize_unit) + set_tests_properties(transcribe_sortformer_diarize_unit PROPERTIES SKIP_RETURN_CODE 77) endif() @@ -1531,4 +1539,13 @@ if(TRANSCRIBE_BUILD_EXAMPLES) -DCLI=$ -DWAV=${CMAKE_SOURCE_DIR}/samples/jfk.wav -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_device_arg_smoke.cmake) + + add_test(NAME transcribe_cli_diarize_smoke + COMMAND ${CMAKE_COMMAND} + -DCLI=$ + -DWAV=${CMAKE_SOURCE_DIR}/samples/sortformer-2spk-mix.wav + -DTEST_DIR=${CMAKE_CURRENT_BINARY_DIR}/cli-diarize-smoke + -P ${CMAKE_CURRENT_SOURCE_DIR}/cli_diarize_smoke.cmake) + set_tests_properties(transcribe_cli_diarize_smoke PROPERTIES + SKIP_REGULAR_EXPRESSION "SKIP:") endif() diff --git a/tests/api_smoke.c b/tests/api_smoke.c index 0fb680a08..dbdb6dca0 100644 --- a/tests/api_smoke.c +++ b/tests/api_smoke.c @@ -77,6 +77,7 @@ static void test_status_string(void) { TRANSCRIBE_ERR_INPUT_TOO_LONG, TRANSCRIBE_ERR_OUTPUT_TRUNCATED, TRANSCRIBE_ERR_OUTPUT_REPETITION, + TRANSCRIBE_ERR_UNSUPPORTED_ROLE, }; for (size_t i = 0; i < sizeof(all) / sizeof(all[0]); ++i) { const char * s = transcribe_status_string(all[i]); @@ -524,6 +525,8 @@ static void test_model_introspection_null(void) { false); CHECK(transcribe_model_accepts_ext_kind(NULL, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_WHISPER_RUN) == false); + CHECK(transcribe_model_roles(NULL) == 0); + /* The feature probe is also NULL-safe and returns false for every * known feature value plus any out-of-range enum. */ CHECK(transcribe_model_supports(NULL, TRANSCRIBE_FEATURE_INITIAL_PROMPT) == false); diff --git a/tests/cli_diarize_smoke.cmake b/tests/cli_diarize_smoke.cmake new file mode 100644 index 000000000..f0e326a8b --- /dev/null +++ b/tests/cli_diarize_smoke.cmake @@ -0,0 +1,56 @@ +# transcribe-cli routes a DIARIZE model to the diarize path and prints speaker +# segments for the 2-speaker oracle mix, writes them with -o, honours --repeat, +# and refuses --batch and --stream-chunk-ms. Gated on TRANSCRIBE_SORTFORMER_GGUF (reported as skipped). + +if("$ENV{TRANSCRIBE_SORTFORMER_GGUF}" STREQUAL "") + message("SKIP: TRANSCRIBE_SORTFORMER_GGUF not set") + return() +endif() + +file(REMOVE_RECURSE "${TEST_DIR}") +file(MAKE_DIRECTORY "${TEST_DIR}") +set(_output "${TEST_DIR}/segments.txt") + +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" -o "${_output}" "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(NOT rc EQUAL 0) + message(FATAL_ERROR "transcribe-cli exited ${rc}\n${out}\n${err}") +endif() +if(NOT out MATCHES "speaker segments: [1-9].*S2") + message(FATAL_ERROR "unexpected output:\n${out}") +endif() +file(READ "${_output}" written) +if(NOT written MATCHES "^\\[ +0\\.[0-9]+ -> +[0-9.]+\\] S[0-9].*S2\n$") + message(FATAL_ERROR "-o wrote unexpected segments:\n${written}") +endif() + +file(WRITE "${TEST_DIR}/list.txt" "${WAV}\n") +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --batch "${TEST_DIR}/list.txt" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(rc EQUAL 0 OR NOT err MATCHES "--batch is ASR-only") + message(FATAL_ERROR "--batch on a diarize model: rc=${rc}\n${err}") +endif() + +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --repeat 2 "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(NOT rc EQUAL 0 OR NOT out MATCHES "speaker segments: [1-9].*S2") + message(FATAL_ERROR "--repeat 2 on a diarize model: rc=${rc}\n${out}\n${err}") +endif() + +execute_process( + COMMAND "${CLI}" -q --backend cpu -m "$ENV{TRANSCRIBE_SORTFORMER_GGUF}" --stream-chunk-ms 100 "${WAV}" + RESULT_VARIABLE rc + OUTPUT_VARIABLE out + ERROR_VARIABLE err) +if(rc EQUAL 0 OR NOT err MATCHES "no streaming entry point") + message(FATAL_ERROR "--stream-chunk-ms on a diarize model: rc=${rc}\n${err}") +endif() diff --git a/tests/diarize_dispatch_unit.cpp b/tests/diarize_dispatch_unit.cpp new file mode 100644 index 000000000..7c8d9ecc7 --- /dev/null +++ b/tests/diarize_dispatch_unit.cpp @@ -0,0 +1,223 @@ +// diarize_dispatch_unit.cpp - DIARIZE role dispatcher (transcribe-diarize.cpp) +// against a fake arch: role checks, pre-clear validation, probs -> segments, +// accessors. + +#include "transcribe-arch.h" +#include "transcribe-diarize.h" +#include "transcribe-model.h" +#include "transcribe.h" +#include "transcribe/diarize.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +constexpr uint32_t k_fake_kind = 0x4B414644; // 'DFAK' + +int g_run_calls = 0; +transcribe_status g_run_status = TRANSCRIBE_OK; +transcribe_status g_validate_st = TRANSCRIBE_OK; + +int fake_max_speakers(const transcribe_model *) { + return 3; +} + +transcribe_diarize_session * fake_new_session() { + return new transcribe_diarize_session(); +} + +transcribe_status fake_run_validate(const transcribe_diarize_params *) { + return g_validate_st; +} + +// Frames at 80 ms: speaker 1 active in frames 1-2, speaker 2 in frames 2-3. +transcribe_status fake_run(transcribe_diarize_session *, + const float *, + int, + const transcribe_diarize_params *, + transcribe::DiarizeProbs & out) { + ++g_run_calls; + if (g_run_status != TRANSCRIBE_OK) { + return g_run_status; + } + out.probs = { 0.1f, 0.0f, 0.9f, 0.0f, 0.8f, 0.7f, 0.2f, 0.6f }; + out.n_frames = 4; + out.n_speakers = 2; + out.frame_ms = 80.0; + return TRANSCRIBE_OK; +} + +bool fake_accepts(const transcribe_model *, transcribe_ext_slot slot, uint32_t kind) { + return slot == TRANSCRIBE_EXT_SLOT_DIARIZE_RUN && kind == k_fake_kind; +} + +const transcribe::DiarizeOps k_ops = { fake_max_speakers, fake_new_session, fake_run_validate, fake_run }; + +const transcribe::Arch k_arch = { + /* .name = */ "fake-diarize", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ nullptr, + /* .run_batch = */ nullptr, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ nullptr, + /* .stream_feed = */ nullptr, + /* .stream_finalize = */ nullptr, + /* .stream_reset = */ nullptr, + /* .accepts_ext_kind = */ fake_accepts, + /* .run_validate = */ nullptr, + /* .diarize = */ &k_ops, +}; + +struct Fixture { + transcribe_model model; + transcribe_diarize_session * session = nullptr; + std::vector pcm = std::vector(1600, 0.0f); + + Fixture() { + model.arch = &k_arch; + model.roles = TRANSCRIBE_ROLE_DIARIZE; + CHECK(transcribe_diarize_session_init(&model, nullptr, &session) == TRANSCRIBE_OK); + } + + ~Fixture() { transcribe_diarize_session_free(session); } + + transcribe_status run(const transcribe_diarize_params * p = nullptr) { + return transcribe_diarize_run(session, pcm.data(), static_cast(pcm.size()), p); + } +}; + +void test_probs_to_segments() { + // [frames=5, speakers=2]; speaker 2 still active at the end. + const float probs[] = { 0.6f, 0.0f, 0.6f, 0.0f, 0.4f, 0.0f, 0.9f, 0.51f, 0.0f, 0.7f }; + std::vector out; + transcribe::probs_to_segments(probs, 5, 2, 80.0, out); + CHECK(out.size() == 3); + CHECK(out[0].speaker_id == 1 && out[0].t0_ms == 0 && out[0].t1_ms == 160); + CHECK(out[1].speaker_id == 1 && out[1].t0_ms == 240 && out[1].t1_ms == 320); + CHECK(out[2].speaker_id == 2 && out[2].t0_ms == 240 && out[2].t1_ms == 400); + CHECK(std::isnan(out[0].p)); +} + +void test_role_checks() { + transcribe_model model; + model.arch = &k_arch; + model.roles = TRANSCRIBE_ROLE_ASR; // no DIARIZE bit + + transcribe_diarize_session * s = reinterpret_cast(0x1); + CHECK(transcribe_diarize_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(s == nullptr); + + transcribe_diarize_info info; + transcribe_diarize_info_init(&info); + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + + model.roles = TRANSCRIBE_ROLE_DIARIZE; + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_OK); + CHECK(info.sample_rate == 16000 && info.max_speakers == 3); + CHECK(transcribe_diarize_get_info(nullptr, &info) == TRANSCRIBE_ERR_INVALID_ARG); + info.struct_size = 0; + CHECK(transcribe_diarize_get_info(&model, &info) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + + transcribe_diarize_session_params sp; + transcribe_diarize_session_params_init(&sp); + sp.n_threads = -1; + CHECK(transcribe_diarize_session_init(&model, &sp, &s) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(s == nullptr); + sp.n_threads = 2; + CHECK(transcribe_diarize_session_init(&model, &sp, &s) == TRANSCRIBE_OK); + CHECK(s != nullptr); + transcribe_diarize_session_free(s); + transcribe_diarize_session_free(nullptr); +} + +void test_run_and_accessors() { + Fixture f; + CHECK(transcribe_diarize_n_segments(f.session) == 0); + CHECK(f.run() == TRANSCRIBE_OK); + CHECK(transcribe_diarize_n_segments(f.session) == 2); + + transcribe_speaker_segment row; + transcribe_speaker_segment_init(&row); + CHECK(transcribe_diarize_get_segment(f.session, 1, &row) == TRANSCRIBE_OK); + CHECK(row.speaker_id == 2 && row.t0_ms == 160 && row.t1_ms == 320); + CHECK(transcribe_diarize_get_segment(f.session, 2, &row) == TRANSCRIBE_OK); // out of range: zeroed + CHECK(row.speaker_id == 0 && row.t0_ms == 0 && row.t1_ms == 0); + CHECK(transcribe_diarize_get_segment(f.session, 0, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + + transcribe_timings tm; + transcribe_timings_init(&tm); + CHECK(transcribe_diarize_get_timings(f.session, &tm) == TRANSCRIBE_OK); +} + +// Malformed input and family-rejected params leave the previous result alone +// and never reach the family run. +void test_pre_clear_rejections_preserve_result() { + g_validate_st = TRANSCRIBE_OK; + g_run_status = TRANSCRIBE_OK; + Fixture f; + CHECK(f.run() == TRANSCRIBE_OK); + g_run_calls = 0; + + std::vector bad(1600, 0.0f); + bad[7] = std::numeric_limits::quiet_NaN(); + CHECK(transcribe_diarize_run(f.session, bad.data(), 1600, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_run(f.session, nullptr, 1600, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_run(f.session, f.pcm.data(), 0, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + + transcribe_ext ext{ sizeof(transcribe_ext), 0x12345678u }; + transcribe_diarize_params p; + transcribe_diarize_params_init(&p); + p.family = &ext; + CHECK(f.run(&p) == TRANSCRIBE_ERR_INVALID_ARG); // kind not accepted on DIARIZE_RUN + ext.size = 4; + CHECK(f.run(&p) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + ext = { sizeof(transcribe_ext), k_fake_kind }; + g_validate_st = TRANSCRIBE_ERR_INVALID_ARG; + CHECK(f.run(&p) == TRANSCRIBE_ERR_INVALID_ARG); + p.struct_size = 0; + CHECK(f.run(&p) == TRANSCRIBE_ERR_BAD_STRUCT_SIZE); + + CHECK(g_run_calls == 0); + CHECK(transcribe_diarize_n_segments(f.session) == 2); + + // A family failure after the commit point clears the result. + g_validate_st = TRANSCRIBE_OK; + g_run_status = TRANSCRIBE_ERR_ABORTED; + p.struct_size = sizeof(p); + CHECK(f.run(&p) == TRANSCRIBE_ERR_ABORTED); + CHECK(transcribe_diarize_n_segments(f.session) == 0); + g_run_status = TRANSCRIBE_OK; +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); + + test_probs_to_segments(); + test_role_checks(); + test_run_and_accessors(); + test_pre_clear_rejections_preserve_result(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("diarize_dispatch_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/nonfinite_input_unit.cpp b/tests/nonfinite_input_unit.cpp new file mode 100644 index 000000000..169e8bb2e --- /dev/null +++ b/tests/nonfinite_input_unit.cpp @@ -0,0 +1,173 @@ +// nonfinite_input_unit.cpp - every ASR PCM entry point rejects NaN / Inf +// samples with INVALID_ARG before any result or stream state is modified +// and before the family hook runs (D16a). + +#include "transcribe-arch.h" +#include "transcribe-model.h" +#include "transcribe-session.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +int g_run_calls = 0; +int g_run_batch_calls = 0; +int g_feed_calls = 0; + +transcribe_status fake_run(transcribe_session * s, const float *, int, const transcribe_run_params *) { + ++g_run_calls; + s->full_text = "fresh"; + s->has_result = true; + return TRANSCRIBE_OK; +} + +transcribe_status fake_run_batch(transcribe_session * s, + const float * const *, + const int *, + int n, + const transcribe_run_params *) { + ++g_run_batch_calls; + for (int i = 0; i < n; ++i) { + s->batch_results.push_back(s->capture_result(TRANSCRIBE_OK)); + } + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_begin(transcribe_session *, + const transcribe_run_params *, + const transcribe_stream_params *) { + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_feed(transcribe_session *, const float *, int, transcribe_stream_update *) { + ++g_feed_calls; + return TRANSCRIBE_OK; +} + +transcribe_status fake_stream_finalize(transcribe_session *, transcribe_stream_update *) { + return TRANSCRIBE_OK; +} + +const transcribe::Arch k_arch = { + /* .name = */ "fake-nonfinite", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ fake_run, + /* .run_batch = */ fake_run_batch, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ fake_stream_begin, + /* .stream_feed = */ fake_stream_feed, + /* .stream_finalize = */ fake_stream_finalize, +}; + +const float k_nan = std::numeric_limits::quiet_NaN(); +const float k_inf = std::numeric_limits::infinity(); +const float k_bads[] = { k_nan, k_inf, -k_inf }; + +// A session holding a prior single result and a prior batch result. +void seed(transcribe_session & s, transcribe_model & m) { + s.model = &m; + s.full_text = "previous"; + s.has_result = true; + s.batch_results.clear(); + s.batch_results.push_back(s.capture_result(TRANSCRIBE_OK)); +} + +bool prior_intact(const transcribe_session & s) { + return s.full_text == "previous" && s.has_result && s.batch_results.size() == 1; +} + +void test_run_rejects_nonfinite() { + transcribe_model model; + model.arch = &k_arch; + for (float bad : k_bads) { + transcribe_session s; + seed(s, model); + std::vector pcm(1600, 0.1f); + pcm[799] = bad; + g_run_calls = 0; + CHECK(transcribe_run(&s, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_run_calls == 0); + CHECK(prior_intact(s)); + } +} + +void test_batch_rejects_nonfinite_whole_batch() { + transcribe_model model; + model.arch = &k_arch; + for (float bad : k_bads) { + transcribe_session s; + seed(s, model); + std::vector a(800, 0.1f), b(800, 0.1f), c(800, 0.1f); + b[400] = bad; + const float * pcm[] = { a.data(), b.data(), c.data() }; + const int n_samples[] = { 800, 800, 800 }; + g_run_calls = 0; + g_run_batch_calls = 0; + CHECK(transcribe_run_batch(&s, pcm, n_samples, 3, nullptr) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_run_calls == 0); + CHECK(g_run_batch_calls == 0); + CHECK(prior_intact(s)); + } +} + +void test_stream_feed_rejects_nonfinite() { + transcribe_model model; + model.arch = &k_arch; + model.caps.supports_streaming = true; + transcribe_session s; + s.model = &model; + CHECK(transcribe_stream_begin(&s, nullptr, nullptr) == TRANSCRIBE_OK); + const int revision = transcribe_stream_revision(&s); + + for (float bad : k_bads) { + std::vector pcm(1600, 0.1f); + pcm[1000] = bad; + g_feed_calls = 0; + transcribe_stream_update upd; + transcribe_stream_update_init(&upd); + CHECK(transcribe_stream_feed(&s, pcm.data(), static_cast(pcm.size()), &upd) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(g_feed_calls == 0); + // The stream stays usable: rejected before the hook, so no FAILED. + CHECK(transcribe_stream_get_state(&s) == TRANSCRIBE_STREAM_ACTIVE); + CHECK(transcribe_stream_revision(&s) == revision); + } + + std::vector ok(1600, 0.0f); + g_feed_calls = 0; + CHECK(transcribe_stream_feed(&s, ok.data(), static_cast(ok.size()), nullptr) == TRANSCRIBE_OK); + CHECK(g_feed_calls == 1); + CHECK(transcribe_stream_finalize(&s, nullptr) == TRANSCRIBE_OK); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); + + test_run_rejects_nonfinite(); + test_batch_rejects_nonfinite_whole_batch(); + test_stream_feed_rejects_nonfinite(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("nonfinite_input_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/role_resolve_unit.cpp b/tests/role_resolve_unit.cpp new file mode 100644 index 000000000..e39917b19 --- /dev/null +++ b/tests/role_resolve_unit.cpp @@ -0,0 +1,141 @@ +// role_resolve_unit.cpp - load-time role validation (transcribe::resolve_roles) and the public role API (transcribe_model_roles, UNSUPPORTED_ROLE). + +#include "transcribe-arch.h" +#include "transcribe-diarize.h" +#include "transcribe-model.h" +#include "transcribe.h" + +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +transcribe_status fake_init_context(transcribe_model *, const transcribe_session_params *, transcribe_session **) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; +} + +transcribe_status fake_run(transcribe_session *, const float *, int, const transcribe_run_params *) { + return TRANSCRIBE_ERR_NOT_IMPLEMENTED; +} + +const transcribe::Arch k_asr_arch = { + /* .name = */ "fake_asr", + /* .load = */ nullptr, + /* .init_context = */ fake_init_context, + /* .run = */ fake_run, +}; + +// init_context without run: not a usable ASR arch. +const transcribe::Arch k_half_asr_arch = { + /* .name = */ "fake_half_asr", + /* .load = */ nullptr, + /* .init_context = */ fake_init_context, +}; + +const transcribe::Arch k_no_role_arch = { + /* .name = */ "fake_no_role", +}; + +// Diarize-only: a diarize ops table and no ASR hooks. resolve_roles only +// checks the table is present, so an empty one suffices. +const transcribe::DiarizeOps k_diarize_ops = {}; + +transcribe::Arch make_diarize_arch() { + transcribe::Arch a = {}; + a.name = "fake_diarize"; + a.diarize = &k_diarize_ops; + return a; +} + +const transcribe::Arch k_diarize_arch = make_diarize_arch(); + +void test_resolve_roles() { + constexpr uint32_t ASR = TRANSCRIBE_ROLE_ASR; + constexpr uint32_t DIA = TRANSCRIBE_ROLE_DIARIZE; + + struct Case { + const transcribe::Arch * arch; + uint32_t roles_in; + transcribe_status want; + uint32_t roles_out; // checked on OK only + }; + + const Case cases[] = { + { &k_asr_arch, 0, TRANSCRIBE_OK, ASR }, // defaults to ASR + { &k_asr_arch, ASR, TRANSCRIBE_OK, ASR }, + { &k_diarize_arch, DIA, TRANSCRIBE_OK, DIA }, + { &k_diarize_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, // no default for non-ASR roles + { &k_no_role_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_half_asr_arch, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_half_asr_arch, ASR, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, ASR | DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_diarize_arch, ASR | DIA, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + { &k_asr_arch, ASR | (1u << 31), TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, // unknown bit + { nullptr, 0, TRANSCRIBE_ERR_NOT_IMPLEMENTED, 0 }, + }; + for (const Case & c : cases) { + transcribe_model model; + model.arch = c.arch; + model.roles = c.roles_in; + const transcribe_status st = transcribe::resolve_roles(&model); + CHECK(st == c.want); + if (st == TRANSCRIBE_OK) { + CHECK(model.roles == c.roles_out); + } + } +} + +// ASR entry points refuse a model that does not serve ASR, before +// touching params or the arch; capabilities leave the caller's struct alone. +void test_asr_entry_points_check_role() { + transcribe_model model; + model.arch = &k_asr_arch; + model.roles = TRANSCRIBE_ROLE_DIARIZE; // not ASR + + transcribe_session * s = reinterpret_cast(0x1); + CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(s == nullptr); + + transcribe_capabilities caps; + transcribe_capabilities_init(&caps); + caps.max_audio_ms = 1234; // sentinel: must survive the rejection + CHECK(transcribe_model_get_capabilities(&model, &caps) == TRANSCRIBE_ERR_UNSUPPORTED_ROLE); + CHECK(caps.max_audio_ms == 1234); + + CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_DIARIZE); + + // Same model with the ASR bit: capabilities succeed; session_init gets + // past the role check to the fake arch's init_context. + model.roles = TRANSCRIBE_ROLE_ASR; + CHECK(transcribe_model_get_capabilities(&model, &caps) == TRANSCRIBE_OK); + CHECK(transcribe_session_init(&model, nullptr, &s) == TRANSCRIBE_ERR_NOT_IMPLEMENTED); + CHECK(s == nullptr); + CHECK(transcribe_model_roles(&model) == TRANSCRIBE_ROLE_ASR); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); // the rejections log at ERROR + + test_resolve_roles(); + test_asr_entry_points_check_role(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("role_resolve_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/run_on_threads_unit.cpp b/tests/run_on_threads_unit.cpp new file mode 100644 index 000000000..98c898ca1 --- /dev/null +++ b/tests/run_on_threads_unit.cpp @@ -0,0 +1,140 @@ +// run_on_threads_unit.cpp - transcribe::run_on_threads joins every launched +// thread on every path and hands worker / launch exceptions to the caller +// instead of calling std::terminate. + +#include "transcribe-batch-util.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +// std::thread wrapper whose construction fails on the k-th launch, like a +// system out of threads. Counts joins so the test can see every launched +// thread was joined before the error surfaced. +struct FlakyThread { + static int fail_at; // 0-based launch index that throws; -1 = never + static int launches; + static std::atomic joins; + + std::thread th; + + template explicit FlakyThread(F && f, A &&... a) { + if (launches++ == fail_at) { + throw std::system_error(std::make_error_code(std::errc::resource_unavailable_try_again)); + } + th = std::thread(std::forward(f), std::forward(a)...); + } + + void join() { + th.join(); + joins.fetch_add(1); + } +}; + +int FlakyThread::fail_at = -1; +int FlakyThread::launches = 0; +std::atomic FlakyThread::joins{ 0 }; + +void reset_flaky(int fail_at) { + FlakyThread::fail_at = fail_at; + FlakyThread::launches = 0; + FlakyThread::joins = 0; +} + +void test_every_tid_runs_once() { + for (int n : { 0, 1, 2, 7 }) { + std::vector> hits(static_cast(n > 0 ? n : 1)); + transcribe::run_on_threads(n, [&](int tid) { hits[static_cast(tid)].fetch_add(1); }); + for (auto & h : hits) { + CHECK(h.load() == 1); + } + } +} + +void test_worker_throw_rethrown_after_join() { + std::atomic done{ 0 }; + bool caught = false; + try { + transcribe::run_on_threads(4, [&](int tid) { + if (tid == 2) { + throw std::bad_alloc(); + } + done.fetch_add(1); + }); + } catch (const std::bad_alloc &) { + caught = true; // exception type preserved across the thread boundary + } + CHECK(caught); + CHECK(done.load() == 3); // every other thread ran to completion and was joined +} + +void test_launch_failure_joins_launched_threads() { + // Launch index 0 is tid 1. Failing at index 2 (tid 3) means tids 1 and 2 + // were launched and must be joined; tid 0 must not run. + reset_flaky(2); + std::atomic ran{ 0 }; + bool tid0_ran = false; + bool caught = false; + try { + transcribe::run_on_threads(5, [&](int tid) { + if (tid == 0) { + tid0_ran = true; + } + ran.fetch_add(1); + }); + } catch (const std::system_error &) { + caught = true; + } + CHECK(caught); + CHECK(!tid0_ran); + CHECK(FlakyThread::joins.load() == 2); + CHECK(ran.load() == 2); +} + +void test_parallel_for_all_propagates() { + bool caught = false; + try { + (void) transcribe::parallel_for_all(16, 4, [](int i) -> bool { + if (i == 9) { + throw std::runtime_error("work"); + } + return true; + }); + } catch (const std::runtime_error &) { + caught = true; + } + CHECK(caught); +} + +} // namespace + +int main() { + test_every_tid_runs_once(); + test_worker_throw_rethrown_after_join(); + test_launch_failure_joins_launched_threads(); + test_parallel_for_all_propagates(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("run_on_threads_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/sortformer_diarize_unit.cpp b/tests/sortformer_diarize_unit.cpp new file mode 100644 index 000000000..cc582757b --- /dev/null +++ b/tests/sortformer_diarize_unit.cpp @@ -0,0 +1,144 @@ +// sortformer_diarize_unit.cpp - Sortformer on the DIARIZE role, against a +// real GGUF (TRANSCRIBE_SORTFORMER_GGUF; RC 77 skip when unset). + +#include "transcribe.h" +#include "transcribe/diarize.h" +#include "transcribe/sortformer.h" +#include "wav.h" + +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +struct Row { + int64_t t0, t1; + int32_t spk; + + bool operator==(const Row & o) const { return t0 == o.t0 && t1 == o.t1 && spk == o.spk; } +}; + +std::vector diarize_rows(const transcribe_diarize_session * s) { + std::vector rows; + for (int i = 0; i < transcribe_diarize_n_segments(s); ++i) { + transcribe_speaker_segment r; + transcribe_speaker_segment_init(&r); + transcribe_diarize_get_segment(s, i, &r); + rows.push_back({ r.t0_ms, r.t1_ms, r.speaker_id }); + } + return rows; +} + +transcribe_model * load_cpu(const char * path) { + transcribe_model_load_params mp; + transcribe_model_load_params_init(&mp); + mp.backend = TRANSCRIBE_BACKEND_CPU; + transcribe_model * model = nullptr; + return transcribe_model_load_file(path, &mp, &model) == TRANSCRIBE_OK ? model : nullptr; +} + +std::vector diarize(transcribe_diarize_session * s, + const std::vector & pcm, + transcribe_sortformer_preset preset) { + transcribe_sortformer_diarize_ext ext; + transcribe_sortformer_diarize_ext_init(&ext); + ext.preset = preset; + transcribe_diarize_params dp; + transcribe_diarize_params_init(&dp); + dp.family = &ext.ext; + CHECK(transcribe_diarize_run(s, pcm.data(), static_cast(pcm.size()), &dp) == TRANSCRIBE_OK); + return diarize_rows(s); +} + +} // namespace + +int main() { + const char * path = std::getenv("TRANSCRIBE_SORTFORMER_GGUF"); + if (path == nullptr || path[0] == '\0') { + std::fprintf(stderr, "sortformer_diarize_unit: TRANSCRIBE_SORTFORMER_GGUF not set; skipping.\n"); + return 77; + } + std::vector pcm; + std::string err; + if (!transcribe_cli::load_wav_mono_16k(std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/sortformer-2spk-mix.wav", pcm, + err)) { + std::fprintf(stderr, "sortformer_diarize_unit: wav: %s\n", err.c_str()); + return 77; + } + ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); + + transcribe_model * model = load_cpu(path); + if (model == nullptr) { + std::fprintf(stderr, "FAIL: model load\n"); + return EXIT_FAILURE; + } + + // 1. Roles, info, extension probe. + CHECK(transcribe_model_roles(model) == TRANSCRIBE_ROLE_DIARIZE); + CHECK(!transcribe_model_supports(model, TRANSCRIBE_FEATURE_DIARIZATION)); + CHECK(transcribe_model_supports(model, TRANSCRIBE_FEATURE_CANCELLATION)); + transcribe_diarize_info info; + transcribe_diarize_info_init(&info); + CHECK(transcribe_diarize_get_info(model, &info) == TRANSCRIBE_OK); + CHECK(info.sample_rate == 16000 && info.max_speakers == 4); + CHECK(transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_DIARIZE_RUN, + TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); + CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_DIARIZE)); + + transcribe_diarize_session * dia = nullptr; + CHECK(transcribe_diarize_session_init(model, nullptr, &dia) == TRANSCRIBE_OK); + + // 2. Golden segments (ms, speaker) recorded on CPU, and ext == env preset. + const std::vector golden_default = { + { 320, 2400, 1 }, + { 7360, 9360, 1 }, + { 10240, 10640, 1 }, + { 4240, 6640, 2 }, + { 9760, 12000, 2 }, + }; + const std::vector golden_low_latency = { + { 320, 2480, 1 }, + { 7360, 9360, 1 }, + { 10240, 10640, 1 }, + { 4160, 6640, 2 }, + { 9760, 12000, 2 }, + }; + CHECK(diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_DEFAULT) == golden_default); + CHECK(diarize(dia, pcm, TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY) == golden_low_latency); + ::setenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET", "low_latency", 1); + CHECK(transcribe_diarize_run(dia, pcm.data(), static_cast(pcm.size()), nullptr) == TRANSCRIBE_OK); + CHECK(diarize_rows(dia) == golden_low_latency); + ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); + + // 3. Pre-clear rejection of a bad preset keeps the previous result. + const int n_before = transcribe_diarize_n_segments(dia); + transcribe_sortformer_diarize_ext bad; + transcribe_sortformer_diarize_ext_init(&bad); + bad.preset = static_cast(99); + transcribe_diarize_params dp; + transcribe_diarize_params_init(&dp); + dp.family = &bad.ext; + CHECK(transcribe_diarize_run(dia, pcm.data(), static_cast(pcm.size()), &dp) == TRANSCRIBE_ERR_INVALID_ARG); + CHECK(transcribe_diarize_n_segments(dia) == n_before); + + transcribe_diarize_session_free(dia); + transcribe_model_free(model); + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("sortformer_diarize_unit: ok\n"); + return EXIT_SUCCESS; +} diff --git a/tests/sortformer_stream_ext_unit.cpp b/tests/sortformer_stream_ext_unit.cpp deleted file mode 100644 index 31bd1001c..000000000 --- a/tests/sortformer_stream_ext_unit.cpp +++ /dev/null @@ -1,188 +0,0 @@ -// sortformer_stream_ext_unit.cpp - Sortformer streaming operating-point run -// extension (TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM, RUN slot). -// -// Covers, against a real GGUF (env-gated, RC 77 skip): -// -// 1. transcribe_model_accepts_ext_kind: SFST accepted on _RUN only; -// foreign kinds rejected. -// 2. transcribe_sortformer_stream_ext_init stamps size/kind/preset. -// 3. Pre-clear rejection: a wrong-kind ext and an out-of-range preset -// both fail with INVALID_ARG and PRESERVE the previous result -// (run_validate fires before clear_result). -// 4. Ext preset == env preset parity: running the committed 2-speaker -// oracle mix with ext=VERY_HIGH_LATENCY produces exactly the same -// speaker-segment rows as TRANSCRIBE_SORTFORMER_STREAM_PRESET= -// very_high_latency (the DER-validated env path), and a different -// geometry (LOW_LATENCY) is accepted and produces rows. -// -// Gated by TRANSCRIBE_SORTFORMER_GGUF (same pattern as the other -// real-model tests) because the preset resolution runs inside run() and -// needs real weights to produce comparable segments. - -#include "transcribe.h" -#include "transcribe/sortformer.h" -#include "wav.h" - -#include - -#include -#include -#include -#include - -namespace { - -int g_failures = 0; - -#define CHECK(cond) \ - do { \ - if (!(cond)) { \ - std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ - ++g_failures; \ - } \ - } while (0) - -bool file_exists(const std::string & path) { - struct stat st{}; - return ::stat(path.c_str(), &st) == 0; -} - -std::vector read_segments(const transcribe_session * session) { - std::vector rows; - const int n = transcribe_n_speaker_segments(session); - for (int i = 0; i < n; ++i) { - transcribe_speaker_segment row; - transcribe_speaker_segment_init(&row); - if (transcribe_get_speaker_segment(session, i, &row) == TRANSCRIBE_OK) { - rows.push_back(row); - } - } - return rows; -} - -bool same_segments(const std::vector & a, - const std::vector & b) { - if (a.size() != b.size()) { - return false; - } - for (size_t i = 0; i < a.size(); ++i) { - if (a[i].t0_ms != b[i].t0_ms || a[i].t1_ms != b[i].t1_ms || a[i].speaker_id != b[i].speaker_id) { - return false; - } - } - return true; -} - -} // namespace - -int main() { - const char * env = std::getenv("TRANSCRIBE_SORTFORMER_GGUF"); - if (env == nullptr || env[0] == '\0') { - std::fprintf(stderr, - "sortformer_stream_ext_unit: TRANSCRIBE_SORTFORMER_GGUF not set; skipping.\n" - "Re-run with TRANSCRIBE_SORTFORMER_GGUF=models/diar_streaming_sortformer_4spk-v2.1/" - "diar_streaming_sortformer_4spk-v2.1-F32.gguf\n"); - return 77; - } - const std::string gguf = env; - if (!file_exists(gguf)) { - std::fprintf(stderr, "sortformer_stream_ext_unit: file not found: %s\n", gguf.c_str()); - return 77; - } - const std::string wav_path = std::string(TRANSCRIBE_TEST_SAMPLES_DIR) + "/sortformer-2spk-mix.wav"; - std::vector pcm; - std::string wav_err; - if (!transcribe_cli::load_wav_mono_16k(wav_path, pcm, wav_err)) { - std::fprintf(stderr, "sortformer_stream_ext_unit: wav load: %s\n", wav_err.c_str()); - return 77; - } - - // The parity leg compares the ext path against the env path, so the - // environment must start clean of the validation overrides. - ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); - - transcribe_model_load_params mp; - transcribe_model_load_params_init(&mp); - mp.backend = TRANSCRIBE_BACKEND_CPU; // deterministic float order for the parity leg - struct transcribe_model * model = nullptr; - if (transcribe_model_load_file(gguf.c_str(), &mp, &model) != TRANSCRIBE_OK || model == nullptr) { - std::fprintf(stderr, "FAIL: model load\n"); - return EXIT_FAILURE; - } - - // 1. Kind+slot probe. - CHECK(transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); - CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_STREAM, TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM)); - CHECK(!transcribe_model_accepts_ext_kind(model, TRANSCRIBE_EXT_SLOT_RUN, 0x4E524857u /* WHRN */)); - - // 2. Init function stamps the header + default. - transcribe_sortformer_stream_ext ext; - transcribe_sortformer_stream_ext_init(&ext); - CHECK(ext.ext.size == sizeof(ext)); - CHECK(ext.ext.kind == TRANSCRIBE_EXT_KIND_SORTFORMER_STREAM); - CHECK(ext.preset == TRANSCRIBE_SORTFORMER_PRESET_DEFAULT); - - struct transcribe_session * session = nullptr; - if (transcribe_session_init(model, nullptr, &session) != TRANSCRIBE_OK || session == nullptr) { - std::fprintf(stderr, "FAIL: session create\n"); - transcribe_model_free(model); - return EXIT_FAILURE; - } - - transcribe_run_params rp; - transcribe_run_params_init(&rp); - - // Baseline run (env preset path, the DER-validated harness route). - ::setenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET", "very_high_latency", 1); - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - const std::vector env_rows = read_segments(session); - ::unsetenv("TRANSCRIBE_SORTFORMER_STREAM_PRESET"); - CHECK(!env_rows.empty()); // the oracle mix has two speakers - - // 3. Pre-clear rejection preserves the previous result. - { - transcribe_sortformer_stream_ext bad; - transcribe_sortformer_stream_ext_init(&bad); - bad.ext.kind = 0x4E524857u; // WHRN: wrong family kind - rp.family = &bad.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(same_segments(read_segments(session), env_rows)); // snapshot intact - - transcribe_sortformer_stream_ext oor; - transcribe_sortformer_stream_ext_init(&oor); - oor.preset = static_cast(99); - rp.family = &oor.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_ERR_INVALID_ARG); - CHECK(same_segments(read_segments(session), env_rows)); - } - - // 4a. Ext preset == env preset parity (same operating point, same rows). - { - transcribe_sortformer_stream_ext vh; - transcribe_sortformer_stream_ext_init(&vh); - vh.preset = TRANSCRIBE_SORTFORMER_PRESET_VERY_HIGH_LATENCY; - rp.family = &vh.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - CHECK(same_segments(read_segments(session), env_rows)); - } - - // 4b. A different geometry is accepted and produces speaker rows. - { - transcribe_sortformer_stream_ext lo; - transcribe_sortformer_stream_ext_init(&lo); - lo.preset = TRANSCRIBE_SORTFORMER_PRESET_LOW_LATENCY; - rp.family = &lo.ext; - CHECK(transcribe_run(session, pcm.data(), static_cast(pcm.size()), &rp) == TRANSCRIBE_OK); - CHECK(!read_segments(session).empty()); - } - - transcribe_session_free(session); - transcribe_model_free(model); - - if (g_failures != 0) { - std::fprintf(stderr, "sortformer_stream_ext_unit: %d failure(s)\n", g_failures); - return EXIT_FAILURE; - } - std::printf("sortformer_stream_ext_unit: OK\n"); - return 0; -} diff --git a/tests/stream_hook_throw_unit.cpp b/tests/stream_hook_throw_unit.cpp new file mode 100644 index 000000000..ea159ed02 --- /dev/null +++ b/tests/stream_hook_throw_unit.cpp @@ -0,0 +1,157 @@ +// stream_hook_throw_unit.cpp - a family stream hook that throws ends the +// stream: begin / feed / finalize leave it FAILED with last_status matching +// the status the call returned (OOM for bad_alloc, BACKEND otherwise), and +// reset still ends IDLE. + +#include "transcribe-arch.h" +#include "transcribe-model.h" +#include "transcribe-session.h" +#include "transcribe.h" + +#include +#include +#include +#include +#include + +namespace { + +int g_failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++g_failures; \ + } \ + } while (0) + +enum class Throw { None, BadAlloc, Runtime }; + +Throw g_begin = Throw::None; +Throw g_feed = Throw::None; +Throw g_finalize = Throw::None; +Throw g_reset = Throw::None; + +void maybe_throw(Throw t) { + if (t == Throw::BadAlloc) { + throw std::bad_alloc(); + } + if (t == Throw::Runtime) { + throw std::runtime_error("hook failed"); + } +} + +transcribe_status fake_begin(transcribe_session *, const transcribe_run_params *, const transcribe_stream_params *) { + maybe_throw(g_begin); + return TRANSCRIBE_OK; +} + +transcribe_status fake_feed(transcribe_session *, const float *, int, transcribe_stream_update *) { + maybe_throw(g_feed); + return TRANSCRIBE_OK; +} + +transcribe_status fake_finalize(transcribe_session *, transcribe_stream_update *) { + maybe_throw(g_finalize); + return TRANSCRIBE_OK; +} + +void fake_reset(transcribe_session *) { + maybe_throw(g_reset); +} + +const transcribe::Arch k_arch = { + /* .name = */ "fake-stream-throw", + /* .load = */ nullptr, + /* .init_context = */ nullptr, + /* .run = */ nullptr, + /* .run_batch = */ nullptr, + /* .stream_validate = */ nullptr, + /* .stream_begin = */ fake_begin, + /* .stream_feed = */ fake_feed, + /* .stream_finalize = */ fake_finalize, + /* .stream_reset = */ fake_reset, +}; + +void reset_globals() { + g_begin = g_feed = g_finalize = g_reset = Throw::None; +} + +struct Fixture { + transcribe_model model; + transcribe_session session; + std::vector pcm = std::vector(1600, 0.0f); + + Fixture() { + model.arch = &k_arch; + model.caps.supports_streaming = true; + session.model = &model; + } + + transcribe_status feed() { + return transcribe_stream_feed(&session, pcm.data(), static_cast(pcm.size()), nullptr); + } +}; + +void expect_failed(const transcribe_session & s, transcribe_status st, Throw t) { + const transcribe_status want = t == Throw::BadAlloc ? TRANSCRIBE_ERR_OOM : TRANSCRIBE_ERR_BACKEND; + CHECK(st == want); + CHECK(transcribe_stream_get_state(&s) == TRANSCRIBE_STREAM_FAILED); + CHECK(transcribe_stream_last_status(&s) == want); +} + +// Each of begin / feed / finalize, throwing bad_alloc or a runtime_error, +// leaves the stream FAILED; a failed stream accepts a fresh begin. +void test_hook_throw_fails_stream() { + Throw * hooks[] = { &g_begin, &g_feed, &g_finalize }; + for (Throw * hook : hooks) { + for (Throw t : { Throw::BadAlloc, Throw::Runtime }) { + reset_globals(); + Fixture f; + if (hook != &g_begin) { + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + } + *hook = t; + const transcribe_status st = hook == &g_begin ? transcribe_stream_begin(&f.session, nullptr, nullptr) : + hook == &g_feed ? f.feed() : + transcribe_stream_finalize(&f.session, nullptr); + expect_failed(f.session, st, t); + *hook = Throw::None; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + } + } +} + +void test_reset_throw_still_idles() { + reset_globals(); + Fixture f; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); + f.session.full_text = "partial"; + f.session.has_result = true; + f.session.was_aborted = true; + g_reset = Throw::Runtime; + transcribe_stream_reset(&f.session); // void; the guard swallows the exception + CHECK(transcribe_stream_get_state(&f.session) == TRANSCRIBE_STREAM_IDLE); + CHECK(!f.session.has_result); + CHECK(f.session.full_text.empty()); + CHECK(!transcribe_was_aborted(&f.session)); + g_reset = Throw::None; + CHECK(transcribe_stream_begin(&f.session, nullptr, nullptr) == TRANSCRIBE_OK); +} + +} // namespace + +int main() { + transcribe_log_set(nullptr, nullptr); // the guard logs each caught exception + + test_hook_throw_fails_stream(); + test_reset_throw_still_idles(); + + if (g_failures != 0) { + std::fprintf(stderr, "%d failure(s)\n", g_failures); + return EXIT_FAILURE; + } + std::printf("stream_hook_throw_unit: ok\n"); + return EXIT_SUCCESS; +}